diff options
Diffstat (limited to 'lib/CodeGen')
21 files changed, 726 insertions, 118 deletions
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index b544ff1..bfee679 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1228,9 +1228,9 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) { /// EmitLLVMUsedList - For targets that define a MAI::UsedDirective, mark each /// global in the specified llvm.used list for which emitUsedDirectiveFor /// is true, as being used with this directive. -void AsmPrinter::EmitLLVMUsedList(Constant *List) { +void AsmPrinter::EmitLLVMUsedList(const Constant *List) { // Should be an array of 'i8*'. - ConstantArray *InitList = dyn_cast<ConstantArray>(List); + const ConstantArray *InitList = dyn_cast<ConstantArray>(List); if (InitList == 0) return; for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) { @@ -1243,11 +1243,11 @@ void AsmPrinter::EmitLLVMUsedList(Constant *List) { /// EmitXXStructorList - Emit the ctor or dtor list. This just prints out the /// function pointers, ignoring the init priority. -void AsmPrinter::EmitXXStructorList(Constant *List) { +void AsmPrinter::EmitXXStructorList(const Constant *List) { // Should be an array of '{ int, void ()* }' structs. The first value is the // init priority, which we ignore. if (!isa<ConstantArray>(List)) return; - ConstantArray *InitList = cast<ConstantArray>(List); + const ConstantArray *InitList = cast<ConstantArray>(List); for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) if (ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i))){ if (CS->getNumOperands() != 2) return; // Not array of 2-element structs. @@ -1533,6 +1533,13 @@ static void EmitGlobalConstantVector(const ConstantVector *CV, unsigned AddrSpace, AsmPrinter &AP) { for (unsigned i = 0, e = CV->getType()->getNumElements(); i != e; ++i) EmitGlobalConstantImpl(CV->getOperand(i), AddrSpace, AP); + + const TargetData &TD = *AP.TM.getTargetData(); + unsigned Size = TD.getTypeAllocSize(CV->getType()); + unsigned EmittedSize = TD.getTypeAllocSize(CV->getType()->getElementType()) * + CV->getType()->getNumElements(); + if (unsigned Padding = Size - EmittedSize) + AP.OutStreamer.EmitZeros(Padding, AddrSpace); } static void EmitGlobalConstantStruct(const ConstantStruct *CS, diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp index 967a278..1f992fa 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp @@ -512,6 +512,8 @@ void DwarfException::EmitExceptionTable() { SizeAlign = 0; } + bool VerboseAsm = Asm->OutStreamer.isVerboseAsm(); + // SjLj Exception handling if (IsSJLJ) { Asm->EmitEncodingByte(dwarf::DW_EH_PE_udata4, "Call site"); @@ -525,14 +527,30 @@ void DwarfException::EmitExceptionTable() { I = CallSites.begin(), E = CallSites.end(); I != E; ++I, ++idx) { const CallSiteEntry &S = *I; + if (VerboseAsm) { + // Emit comments that decode the call site. + Asm->OutStreamer.AddComment(Twine(">> Call Site ") + + llvm::utostr(idx) + " <<"); + Asm->OutStreamer.AddComment(Twine(" On exception at call site ") + + llvm::utostr(idx)); + + if (S.Action == 0) + Asm->OutStreamer.AddComment(" Action: cleanup"); + else + Asm->OutStreamer.AddComment(Twine(" Action: ") + + llvm::utostr((S.Action - 1) / 2 + 1)); + + Asm->OutStreamer.AddBlankLine(); + } + // Offset of the landing pad, counted in 16-byte bundles relative to the // @LPStart address. - Asm->EmitULEB128(idx, "Landing pad"); + Asm->EmitULEB128(idx); // Offset of the first associated action record, relative to the start of // the action table. This value is biased by 1 (1 indicates the start of // the action table), and 0 indicates that there are no actions. - Asm->EmitULEB128(S.Action, "Action"); + Asm->EmitULEB128(S.Action); } } else { // DWARF Exception handling @@ -562,6 +580,7 @@ void DwarfException::EmitExceptionTable() { // Add extra padding if it wasn't added to the TType base offset. Asm->EmitULEB128(CallSiteTableLength, "Call site table length", SizeAlign); + unsigned Entry = 0; for (SmallVectorImpl<CallSiteEntry>::const_iterator I = CallSites.begin(), E = CallSites.end(); I != E; ++I) { const CallSiteEntry &S = *I; @@ -576,19 +595,38 @@ void DwarfException::EmitExceptionTable() { if (EndLabel == 0) EndLabel = Asm->GetTempSymbol("eh_func_end", Asm->getFunctionNumber()); + if (VerboseAsm) { + // Emit comments that decode the call site. + Asm->OutStreamer.AddComment(Twine(">> Call Site ") + + llvm::utostr(++Entry) + " <<"); + Asm->OutStreamer.AddComment(Twine(" Call between ") + + BeginLabel->getName() + " and " + + EndLabel->getName()); + + if (!S.PadLabel) { + Asm->OutStreamer.AddComment(" has no landing pad"); + } else { + Asm->OutStreamer.AddComment(Twine(" jumps to ") + + S.PadLabel->getName()); + + if (S.Action == 0) + Asm->OutStreamer.AddComment(" On action: cleanup"); + else + Asm->OutStreamer.AddComment(Twine(" On action: ") + + llvm::utostr((S.Action - 1) / 2 + 1)); + } + + Asm->OutStreamer.AddBlankLine(); + } + // Offset of the call site relative to the previous call site, counted in // number of 16-byte bundles. The first call site is counted relative to // the start of the procedure fragment. - Asm->OutStreamer.AddComment("Region start"); Asm->EmitLabelDifference(BeginLabel, EHFuncBeginSym, 4); - - Asm->OutStreamer.AddComment("Region length"); Asm->EmitLabelDifference(EndLabel, BeginLabel, 4); - // Offset of the landing pad, counted in 16-byte bundles relative to the // @LPStart address. - Asm->OutStreamer.AddComment("Landing pad"); if (!S.PadLabel) Asm->OutStreamer.EmitIntValue(0, 4/*size*/, 0/*addrspace*/); else @@ -597,45 +635,63 @@ void DwarfException::EmitExceptionTable() { // Offset of the first associated action record, relative to the start of // the action table. This value is biased by 1 (1 indicates the start of // the action table), and 0 indicates that there are no actions. - Asm->EmitULEB128(S.Action, "Action"); + Asm->EmitULEB128(S.Action); } } // Emit the Action Table. - if (Actions.size() != 0) { - Asm->OutStreamer.AddComment("-- Action Record Table --"); - Asm->OutStreamer.AddBlankLine(); - } - + int Entry = 0; for (SmallVectorImpl<ActionEntry>::const_iterator I = Actions.begin(), E = Actions.end(); I != E; ++I) { const ActionEntry &Action = *I; - Asm->OutStreamer.AddComment("Action Record"); - Asm->OutStreamer.AddBlankLine(); + + if (VerboseAsm) { + // Emit comments that decode the action table. + Asm->OutStreamer.AddComment(Twine(">> Action Record ") + + llvm::utostr(++Entry) + " <<"); + if (Action.ValueForTypeID >= 0) + Asm->OutStreamer.AddComment(Twine(" Catch TypeInfo ") + + llvm::itostr(Action.ValueForTypeID)); + else + Asm->OutStreamer.AddComment(Twine(" Filter TypeInfo ") + + llvm::itostr(Action.ValueForTypeID)); + + if (Action.NextAction == 0) { + Asm->OutStreamer.AddComment(" No further actions"); + } else { + unsigned NextAction = Entry + (Action.NextAction + 1) / 2; + Asm->OutStreamer.AddComment(Twine(" Continue to action ") + + llvm::utostr(NextAction)); + } + + Asm->OutStreamer.AddBlankLine(); + } // Type Filter // // Used by the runtime to match the type of the thrown exception to the // type of the catch clauses or the types in the exception specification. - Asm->EmitSLEB128(Action.ValueForTypeID, " TypeInfo index"); + Asm->EmitSLEB128(Action.ValueForTypeID); // Action Record // // Self-relative signed displacement in bytes of the next action record, // or 0 if there is no next action record. - Asm->EmitSLEB128(Action.NextAction, " Next action"); + Asm->EmitSLEB128(Action.NextAction); } // Emit the Catch TypeInfos. - if (!TypeInfos.empty()) { - Asm->OutStreamer.AddComment("-- Catch TypeInfos --"); + if (VerboseAsm && !TypeInfos.empty()) { + Asm->OutStreamer.AddComment(">> Catch TypeInfos <<"); Asm->OutStreamer.AddBlankLine(); + Entry = TypeInfos.size(); } + for (std::vector<const GlobalVariable *>::const_reverse_iterator I = TypeInfos.rbegin(), E = TypeInfos.rend(); I != E; ++I) { const GlobalVariable *GV = *I; - - Asm->OutStreamer.AddComment("TypeInfo"); + if (VerboseAsm) + Asm->OutStreamer.AddComment(Twine("TypeInfo ") + llvm::utostr(Entry--)); if (GV) Asm->EmitReference(GV, TTypeEncoding); else @@ -644,14 +700,21 @@ void DwarfException::EmitExceptionTable() { } // Emit the Exception Specifications. - if (!FilterIds.empty()) { - Asm->OutStreamer.AddComment("-- Filter IDs --"); + if (VerboseAsm && !FilterIds.empty()) { + Asm->OutStreamer.AddComment(">> Filter TypeInfos <<"); Asm->OutStreamer.AddBlankLine(); + Entry = 0; } for (std::vector<unsigned>::const_iterator I = FilterIds.begin(), E = FilterIds.end(); I < E; ++I) { unsigned TypeID = *I; - Asm->EmitULEB128(TypeID, TypeID != 0 ? "Exception specification" : 0); + if (VerboseAsm) { + --Entry; + if (TypeID != 0) + Asm->OutStreamer.AddComment(Twine("FilterInfo ") + llvm::itostr(Entry)); + } + + Asm->EmitULEB128(TypeID); } Asm->EmitAlignment(2); diff --git a/lib/CodeGen/ELFWriter.cpp b/lib/CodeGen/ELFWriter.cpp index fa2319b..d977651 100644 --- a/lib/CodeGen/ELFWriter.cpp +++ b/lib/CodeGen/ELFWriter.cpp @@ -659,11 +659,11 @@ bool ELFWriter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) { /// EmitXXStructorList - Emit the ctor or dtor list. This just emits out the /// function pointers, ignoring the init priority. -void ELFWriter::EmitXXStructorList(Constant *List, ELFSection &Xtor) { +void ELFWriter::EmitXXStructorList(const Constant *List, ELFSection &Xtor) { // Should be an array of '{ i32, void ()* }' structs. The first value is the // init priority, which we ignore. if (List->isNullValue()) return; - ConstantArray *InitList = cast<ConstantArray>(List); + const ConstantArray *InitList = cast<ConstantArray>(List); for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) { if (InitList->getOperand(i)->isNullValue()) continue; diff --git a/lib/CodeGen/ELFWriter.h b/lib/CodeGen/ELFWriter.h index b8bac55..6f7fbac 100644 --- a/lib/CodeGen/ELFWriter.h +++ b/lib/CodeGen/ELFWriter.h @@ -232,7 +232,7 @@ namespace llvm { void EmitGlobalDataRelocation(const GlobalValue *GV, unsigned Size, ELFSection &GblS, int64_t Offset = 0); bool EmitSpecialLLVMGlobal(const GlobalVariable *GV); - void EmitXXStructorList(Constant *List, ELFSection &Xtor); + void EmitXXStructorList(const Constant *List, ELFSection &Xtor); void EmitRelocations(); void EmitRelocation(BinaryObject &RelSec, ELFRelocation &Rel, bool HasRelA); void EmitSectionHeader(BinaryObject &SHdrTab, const ELFSection &SHdr); diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index 613f0c4..c0f71d2 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -768,6 +768,9 @@ MachineBasicBlock::findDebugLoc(MachineBasicBlock::iterator &MBBI) { /// getSuccWeight - Return weight of the edge from this block to MBB. /// uint32_t MachineBasicBlock::getSuccWeight(MachineBasicBlock *succ) { + if (Weights.empty()) + return 0; + succ_iterator I = std::find(Successors.begin(), Successors.end(), succ); return *getWeightIterator(I); } diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp index 65ebdf8..0f27dfc 100644 --- a/lib/CodeGen/RegAllocFast.cpp +++ b/lib/CodeGen/RegAllocFast.cpp @@ -86,7 +86,7 @@ namespace { // that is currently available in a physical register. LiveRegMap LiveVirtRegs; - DenseMap<unsigned, MachineInstr *> LiveDbgValueMap; + DenseMap<unsigned, SmallVector<MachineInstr *, 4> > LiveDbgValueMap; // RegState - Track the state of a physical register. enum RegState { @@ -272,7 +272,9 @@ void RAFast::spillVirtReg(MachineBasicBlock::iterator MI, // If this register is used by DBG_VALUE then insert new DBG_VALUE to // identify spilled location as the place to find corresponding variable's // value. - if (MachineInstr *DBG = LiveDbgValueMap.lookup(LRI->first)) { + SmallVector<MachineInstr *, 4> &LRIDbgValues = LiveDbgValueMap[LRI->first]; + for (unsigned li = 0, le = LRIDbgValues.size(); li != le; ++li) { + MachineInstr *DBG = LRIDbgValues[li]; const MDNode *MDPtr = DBG->getOperand(DBG->getNumOperands()-1).getMetadata(); int64_t Offset = 0; @@ -291,9 +293,11 @@ void RAFast::spillVirtReg(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB = DBG->getParent(); MBB->insert(MI, NewDV); DEBUG(dbgs() << "Inserting debug info due to spill:" << "\n" << *NewDV); - LiveDbgValueMap[LRI->first] = NewDV; } } + // Now this register is spilled there is should not be any DBG_VALUE pointing + // to this register because they are all pointing to spilled value now. + LRIDbgValues.clear(); if (SpillKill) LR.LastUse = 0; // Don't kill register again } @@ -816,7 +820,7 @@ void RAFast::AllocateBasicBlock() { if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - LiveDbgValueMap[Reg] = MI; + LiveDbgValueMap[Reg].push_back(MI); LiveRegMap::iterator LRI = LiveVirtRegs.find(Reg); if (LRI != LiveVirtRegs.end()) setPhysReg(MI, i, LRI->second.PhysReg); diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp index 0818034..7a2ea6c 100644 --- a/lib/CodeGen/RegAllocLinearScan.cpp +++ b/lib/CodeGen/RegAllocLinearScan.cpp @@ -1440,8 +1440,12 @@ unsigned RALinScan::getFreePhysReg(LiveInterval* cur, if (TargetRegisterInfo::isVirtualRegister(physReg) && vrm_->hasPhys(physReg)) physReg = vrm_->getPhys(physReg); - ArrayRef<unsigned> Order = tri_->getRawAllocationOrder(RC, Hint.first, - physReg, *mf_); + ArrayRef<unsigned> Order; + if (Hint.first) + Order = tri_->getRawAllocationOrder(RC, Hint.first, physReg, *mf_); + else + Order = RegClassInfo.getOrder(RC); + assert(!Order.empty() && "No allocatable register in this register class!"); // Scan for the first available register. diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 605507f..49f8fb4 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -84,8 +84,8 @@ public: static char ID; /// Construct a PBQP register allocator. - RegAllocPBQP(std::auto_ptr<PBQPBuilder> b) - : MachineFunctionPass(ID), builder(b) { + RegAllocPBQP(std::auto_ptr<PBQPBuilder> b, char *cPassID=0) + : MachineFunctionPass(ID), builder(b), customPassID(cPassID) { initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); initializeRegisterCoalescerAnalysisGroup(*PassRegistry::getPassRegistry()); @@ -122,6 +122,8 @@ private: std::auto_ptr<PBQPBuilder> builder; + char *customPassID; + MachineFunction *mf; const TargetMachine *tm; const TargetRegisterInfo *tri; @@ -449,6 +451,8 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const { au.addRequired<LiveIntervals>(); //au.addRequiredID(SplitCriticalEdgesID); au.addRequired<RegisterCoalescer>(); + if (customPassID) + au.addRequiredID(*customPassID); au.addRequired<CalculateSpillWeights>(); au.addRequired<LiveStacks>(); au.addPreserved<LiveStacks>(); @@ -702,8 +706,9 @@ bool RegAllocPBQP::runOnMachineFunction(MachineFunction &MF) { } FunctionPass* llvm::createPBQPRegisterAllocator( - std::auto_ptr<PBQPBuilder> builder) { - return new RegAllocPBQP(builder); + std::auto_ptr<PBQPBuilder> builder, + char *customPassID) { + return new RegAllocPBQP(builder, customPassID); } FunctionPass* llvm::createDefaultPBQPRegisterAllocator() { diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 4ac590a..e3d3906 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -238,6 +238,9 @@ namespace { SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT); SDValue BuildSDIV(SDNode *N); SDValue BuildUDIV(SDNode *N); + SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, + bool DemandHighBits = true); + SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1); SDNode *MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL); SDValue ReduceLoadWidth(SDNode *N); SDValue ReduceLoadOpStoreWidth(SDNode *N); @@ -2512,6 +2515,244 @@ SDValue DAGCombiner::visitAND(SDNode *N) { return SDValue(); } +/// MatchBSwapHWord - Match (a >> 8) | (a << 8) as (bswap a) >> 16 +/// +SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, + bool DemandHighBits) { + if (!LegalOperations) + return SDValue(); + + EVT VT = N->getValueType(0); + if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16) + return SDValue(); + if (!TLI.isOperationLegal(ISD::BSWAP, VT)) + return SDValue(); + + // Recognize (and (shl a, 8), 0xff), (and (srl a, 8), 0xff00) + bool LookPassAnd0 = false; + bool LookPassAnd1 = false; + if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) + std::swap(N0, N1); + if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL) + std::swap(N0, N1); + if (N0.getOpcode() == ISD::AND) { + if (!N0.getNode()->hasOneUse()) + return SDValue(); + ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); + if (!N01C || N01C->getZExtValue() != 0xFF00) + return SDValue(); + N0 = N0.getOperand(0); + LookPassAnd0 = true; + } + + if (N1.getOpcode() == ISD::AND) { + if (!N1.getNode()->hasOneUse()) + return SDValue(); + ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1)); + if (!N11C || N11C->getZExtValue() != 0xFF) + return SDValue(); + N1 = N1.getOperand(0); + LookPassAnd1 = true; + } + + if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) + std::swap(N0, N1); + if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) + return SDValue(); + if (!N0.getNode()->hasOneUse() || + !N1.getNode()->hasOneUse()) + return SDValue(); + + ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); + ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1)); + if (!N01C || !N11C) + return SDValue(); + if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8) + return SDValue(); + + // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8) + SDValue N00 = N0->getOperand(0); + if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) { + if (!N00.getNode()->hasOneUse()) + return SDValue(); + ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1)); + if (!N001C || N001C->getZExtValue() != 0xFF) + return SDValue(); + N00 = N00.getOperand(0); + LookPassAnd0 = true; + } + + SDValue N10 = N1->getOperand(0); + if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) { + if (!N10.getNode()->hasOneUse()) + return SDValue(); + ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1)); + if (!N101C || N101C->getZExtValue() != 0xFF00) + return SDValue(); + N10 = N10.getOperand(0); + LookPassAnd1 = true; + } + + if (N00 != N10) + return SDValue(); + + // Make sure everything beyond the low halfword is zero since the SRL 16 + // will clear the top bits. + unsigned OpSizeInBits = VT.getSizeInBits(); + if (DemandHighBits && OpSizeInBits > 16 && + (!LookPassAnd0 || !LookPassAnd1) && + !DAG.MaskedValueIsZero(N10, APInt::getHighBitsSet(OpSizeInBits, 16))) + return SDValue(); + + SDValue Res = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT, N00); + if (OpSizeInBits > 16) + Res = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, Res, + DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT))); + return Res; +} + +/// isBSwapHWordElement - Return true if the specified node is an element +/// that makes up a 32-bit packed halfword byteswap. i.e. +/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8) +static bool isBSwapHWordElement(SDValue N, SmallVector<SDNode*,4> &Parts) { + if (!N.getNode()->hasOneUse()) + return false; + + unsigned Opc = N.getOpcode(); + if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL) + return false; + + ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N.getOperand(1)); + if (!N1C) + return false; + + unsigned Num; + switch (N1C->getZExtValue()) { + default: + return false; + case 0xFF: Num = 0; break; + case 0xFF00: Num = 1; break; + case 0xFF0000: Num = 2; break; + case 0xFF000000: Num = 3; break; + } + + // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00). + SDValue N0 = N.getOperand(0); + if (Opc == ISD::AND) { + if (Num == 0 || Num == 2) { + // (x >> 8) & 0xff + // (x >> 8) & 0xff0000 + if (N0.getOpcode() != ISD::SRL) + return false; + ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); + if (!C || C->getZExtValue() != 8) + return false; + } else { + // (x << 8) & 0xff00 + // (x << 8) & 0xff000000 + if (N0.getOpcode() != ISD::SHL) + return false; + ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); + if (!C || C->getZExtValue() != 8) + return false; + } + } else if (Opc == ISD::SHL) { + // (x & 0xff) << 8 + // (x & 0xff0000) << 8 + if (Num != 0 && Num != 2) + return false; + ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); + if (!C || C->getZExtValue() != 8) + return false; + } else { // Opc == ISD::SRL + // (x & 0xff00) >> 8 + // (x & 0xff000000) >> 8 + if (Num != 1 && Num != 3) + return false; + ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); + if (!C || C->getZExtValue() != 8) + return false; + } + + if (Parts[Num]) + return false; + + Parts[Num] = N0.getOperand(0).getNode(); + return true; +} + +/// MatchBSwapHWord - Match a 32-bit packed halfword bswap. That is +/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8) +/// => (rotl (bswap x), 16) +SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) { + if (!LegalOperations) + return SDValue(); + + EVT VT = N->getValueType(0); + if (VT != MVT::i32) + return SDValue(); + if (!TLI.isOperationLegal(ISD::BSWAP, VT)) + return SDValue(); + + SmallVector<SDNode*,4> Parts(4, (SDNode*)0); + // Look for either + // (or (or (and), (and)), (or (and), (and))) + // (or (or (or (and), (and)), (and)), (and)) + if (N0.getOpcode() != ISD::OR) + return SDValue(); + SDValue N00 = N0.getOperand(0); + SDValue N01 = N0.getOperand(1); + + if (N1.getOpcode() == ISD::OR) { + // (or (or (and), (and)), (or (and), (and))) + SDValue N000 = N00.getOperand(0); + if (!isBSwapHWordElement(N000, Parts)) + return SDValue(); + + SDValue N001 = N00.getOperand(1); + if (!isBSwapHWordElement(N001, Parts)) + return SDValue(); + SDValue N010 = N01.getOperand(0); + if (!isBSwapHWordElement(N010, Parts)) + return SDValue(); + SDValue N011 = N01.getOperand(1); + if (!isBSwapHWordElement(N011, Parts)) + return SDValue(); + } else { + // (or (or (or (and), (and)), (and)), (and)) + if (!isBSwapHWordElement(N1, Parts)) + return SDValue(); + if (!isBSwapHWordElement(N01, Parts)) + return SDValue(); + if (N00.getOpcode() != ISD::OR) + return SDValue(); + SDValue N000 = N00.getOperand(0); + if (!isBSwapHWordElement(N000, Parts)) + return SDValue(); + SDValue N001 = N00.getOperand(1); + if (!isBSwapHWordElement(N001, Parts)) + return SDValue(); + } + + // Make sure the parts are all coming from the same node. + if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3]) + return SDValue(); + + SDValue BSwap = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT, + SDValue(Parts[0],0)); + + // Result of the bswap should be rotated by 16. If it's not legal, than + // do (x << 16) | (x >> 16). + SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT)); + if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) + return DAG.getNode(ISD::ROTL, N->getDebugLoc(), VT, BSwap, ShAmt); + else if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) + return DAG.getNode(ISD::ROTR, N->getDebugLoc(), VT, BSwap, ShAmt); + return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, + DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, BSwap, ShAmt), + DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, BSwap, ShAmt)); +} + SDValue DAGCombiner::visitOR(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -2547,6 +2788,15 @@ SDValue DAGCombiner::visitOR(SDNode *N) { // fold (or x, c) -> c iff (x & ~c) == 0 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue())) return N1; + + // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16) + SDValue BSwap = MatchBSwapHWord(N, N0, N1); + if (BSwap.getNode() != 0) + return BSwap; + BSwap = MatchBSwapHWordLow(N, N0, N1); + if (BSwap.getNode() != 0) + return BSwap; + // reassociate or SDValue ROR = ReassociateOps(ISD::OR, N->getDebugLoc(), N0, N1); if (ROR.getNode() != 0) @@ -4606,6 +4856,16 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); return SDValue(N, 0); // Return N so it doesn't get rechecked! } + + // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16)) + if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) { + SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), + N0.getOperand(1), false); + if (BSwap.getNode() != 0) + return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, + BSwap, N1); + } + return SDValue(); } @@ -5231,7 +5491,8 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) { // fold (sint_to_fp c1) -> c1fp if (N0C && OpVT != MVT::ppcf128 && // ...but only if the target supports immediate floating-point values - (Level == llvm::Unrestricted || TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) + (Level == llvm::Unrestricted || + TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0); // If the input is a legal type, and SINT_TO_FP is not legal on this target, @@ -5255,7 +5516,8 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) { // fold (uint_to_fp c1) -> c1fp if (N0C && OpVT != MVT::ppcf128 && // ...but only if the target supports immediate floating-point values - (Level == llvm::Unrestricted || TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) + (Level == llvm::Unrestricted || + TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0); // If the input is a legal type, and UINT_TO_FP is not legal on this target, @@ -7208,7 +7470,7 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, const TargetData &TD = *TLI.getTargetData(); // Create a ConstantArray of the two constants. - Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts, 2); + Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts); SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(), TD.getPrefTypeAlignment(FPTy)); unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index da75b8a..e7c77dd 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -19,6 +19,7 @@ //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" +#include "llvm/DerivedTypes.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -191,10 +192,6 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { if (NOutVT.bitsEq(NInVT)) // The input promotes to the same size. Convert the promoted value. return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); - if (NInVT.isVector()) - // Promote vector element via memory load/store. - return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, - CreateStackStoreLoad(InOp, OutVT)); break; case TargetLowering::TypeSoftenFloat: // Promote the integer operand by hand. @@ -341,8 +338,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { // (eg: because the value being converted is too big), then the result of the // original operation was undefined anyway, so the assert is still correct. return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? - ISD::AssertZext : ISD::AssertSext, dl, - NVT, Res, DAG.getValueType(N->getValueType(0))); + ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, + DAG.getValueType(N->getValueType(0).getScalarType())); } SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) { @@ -372,7 +369,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, DAG.getValueType(N->getOperand(0).getValueType())); if (N->getOpcode() == ISD::ZERO_EXTEND) - return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); + return DAG.getZeroExtendInReg(Res, dl, + N->getOperand(0).getValueType().getScalarType()); assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); return Res; } @@ -522,20 +520,44 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Res; + SDValue InOp = N->getOperand(0); + DebugLoc dl = N->getDebugLoc(); - switch (getTypeAction(N->getOperand(0).getValueType())) { + switch (getTypeAction(InOp.getValueType())) { default: llvm_unreachable("Unknown type action!"); case TargetLowering::TypeLegal: case TargetLowering::TypeExpandInteger: - Res = N->getOperand(0); + Res = InOp; break; case TargetLowering::TypePromoteInteger: - Res = GetPromotedInteger(N->getOperand(0)); + Res = GetPromotedInteger(InOp); break; + case TargetLowering::TypeSplitVector: + EVT InVT = InOp.getValueType(); + assert(InVT.isVector() && "Cannot split scalar types"); + unsigned NumElts = InVT.getVectorNumElements(); + assert(NumElts == NVT.getVectorNumElements() && + "Dst and Src must have the same number of elements"); + EVT EltVT = InVT.getScalarType(); + assert(isPowerOf2_32(NumElts) && + "Promoted vector type must be a power of two"); + + EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts/2); + EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), + NumElts/2); + + SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, InOp, + DAG.getIntPtrConstant(0)); + SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, InOp, + DAG.getIntPtrConstant(NumElts/2)); + EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); + EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); + + return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); } // Truncate to NVT instead of VT - return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), NVT, Res); + return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); } SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { @@ -1072,6 +1094,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; case ISD::UADDO: case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; + case ISD::UMULO: + case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -2149,6 +2173,86 @@ void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, ReplaceValueWith(SDValue(N, 1), Ofl); } +void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, + SDValue &Lo, SDValue &Hi) { + EVT VT = N->getValueType(0); + const Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); + EVT PtrVT = TLI.getPointerTy(); + const Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); + DebugLoc dl = N->getDebugLoc(); + + // A divide for UMULO should be faster than a function call. + if (N->getOpcode() == ISD::UMULO) { + SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); + DebugLoc DL = N->getDebugLoc(); + + SDValue MUL = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS); + SplitInteger(MUL, Lo, Hi); + + // A divide for UMULO will be faster than a function call. Select to + // make sure we aren't using 0. + SDValue isZero = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), + RHS, DAG.getConstant(0, VT), ISD::SETNE); + SDValue NotZero = DAG.getNode(ISD::SELECT, dl, VT, isZero, + DAG.getConstant(1, VT), RHS); + SDValue DIV = DAG.getNode(ISD::UDIV, DL, LHS.getValueType(), MUL, NotZero); + SDValue Overflow; + Overflow = DAG.getSetCC(DL, N->getValueType(1), DIV, LHS, ISD::SETNE); + ReplaceValueWith(SDValue(N, 1), Overflow); + return; + } + + // Replace this with a libcall that will check overflow. + RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; + if (VT == MVT::i32) + LC = RTLIB::MULO_I32; + else if (VT == MVT::i64) + LC = RTLIB::MULO_I64; + else if (VT == MVT::i128) + LC = RTLIB::MULO_I128; + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); + + SDValue Temp = DAG.CreateStackTemporary(PtrVT); + // Temporary for the overflow value, default it to zero. + SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, + DAG.getConstant(0, PtrVT), Temp, + MachinePointerInfo(), false, false, 0); + + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + EVT ArgVT = N->getOperand(i).getValueType(); + const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); + Entry.Node = N->getOperand(i); + Entry.Ty = ArgTy; + Entry.isSExt = true; + Entry.isZExt = false; + Args.push_back(Entry); + } + + // Also pass the address of the overflow check. + Entry.Node = Temp; + Entry.Ty = PtrTy->getPointerTo(); + Entry.isSExt = true; + Entry.isZExt = false; + Args.push_back(Entry); + + SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); + std::pair<SDValue, SDValue> CallInfo = + TLI.LowerCallTo(Chain, RetTy, true, false, false, false, + 0, TLI.getLibcallCallingConv(LC), false, + true, Func, Args, DAG, dl); + + SplitInteger(CallInfo.first, Lo, Hi); + SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, + MachinePointerInfo(), false, false, 0); + SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, + DAG.getConstant(0, PtrVT), + ISD::SETNE); + // Use the overflow from the libcall everywhere. + ReplaceValueWith(SDValue(N, 1), Ofl); +} + void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT VT = N->getValueType(0); @@ -2641,18 +2745,18 @@ SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { SDValue InOp0 = N->getOperand(0); EVT InVT = InOp0.getValueType(); - EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); EVT OutVT = N->getValueType(0); EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); assert(NOutVT.isVector() && "This type must be promoted to a vector type"); - unsigned OutNumElems = N->getValueType(0).getVectorNumElements(); + unsigned OutNumElems = OutVT.getVectorNumElements(); EVT NOutVTElem = NOutVT.getVectorElementType(); DebugLoc dl = N->getDebugLoc(); SDValue BaseIdx = N->getOperand(1); SmallVector<SDValue, 8> Ops; + Ops.reserve(OutNumElems); for (unsigned i = 0; i != OutNumElems; ++i) { // Extract the element from the original vector. @@ -2684,18 +2788,13 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { SDValue V0 = GetPromotedInteger(N->getOperand(0)); SDValue V1 = GetPromotedInteger(N->getOperand(1)); - EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); + EVT OutVT = V0.getValueType(); - return DAG.getVectorShuffle(OutVT, dl, V0,V1, &NewMask[0]); + return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]); } SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { - - SDValue InOp0 = N->getOperand(0); - EVT InVT = InOp0.getValueType(); - EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); - EVT OutVT = N->getValueType(0); EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); assert(NOutVT.isVector() && "This type must be promoted to a vector type"); @@ -2705,6 +2804,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { DebugLoc dl = N->getDebugLoc(); SmallVector<SDValue, 8> Ops; + Ops.reserve(NumElems); for (unsigned i = 0; i != NumElems; ++i) { SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); Ops.push_back(Op); @@ -2717,10 +2817,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { DebugLoc dl = N->getDebugLoc(); - SDValue InOp0 = N->getOperand(0); - EVT InVT = InOp0.getValueType(); - EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); - assert(!InVT.isVector() && "Input must not be a scalar"); + assert(!N->getOperand(0).getValueType().isVector() && + "Input must be a scalar"); EVT OutVT = N->getValueType(0); EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); @@ -2733,12 +2831,6 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { } SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { - - SDValue InOp0 = N->getOperand(0); - EVT InVT = InOp0.getValueType(); - EVT InElVT = InVT.getVectorElementType(); - EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); - EVT OutVT = N->getValueType(0); EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); assert(NOutVT.isVector() && "This type must be promoted to a vector type"); @@ -2747,7 +2839,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { DebugLoc dl = N->getDebugLoc(); - SDValue ConvertedVector = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp0); + SDValue ConvertedVector = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, + N->getOperand(0)); SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(1)); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 06dc40f..4597ec9 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -318,6 +318,7 @@ private: void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandShiftByConstant(SDNode *N, unsigned Amt, SDValue &Lo, SDValue &Hi); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index f09b381..a827187 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -290,7 +290,17 @@ static void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, // Special handling for untyped values. These values can only come from // the expansion of custom DAG-to-DAG patterns. if (VT == MVT::untyped) { - unsigned Opcode = RegDefPos.GetNode()->getMachineOpcode(); + const SDNode *Node = RegDefPos.GetNode(); + unsigned Opcode = Node->getMachineOpcode(); + + if (Opcode == TargetOpcode::REG_SEQUENCE) { + unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue(); + const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx); + RegClass = RC->getID(); + Cost = 1; + return; + } + unsigned Idx = RegDefPos.GetIdx(); const TargetInstrDesc Desc = TII->get(Opcode); const TargetRegisterClass *RC = Desc.getRegClass(Idx, TRI); @@ -1833,8 +1843,6 @@ bool RegReductionPQBase::HighRegPressure(const SUnit *SU) const { } for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); RegDefPos.IsValid(); RegDefPos.Advance()) { - EVT VT = RegDefPos.GetValue(); - unsigned RCId, Cost; GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index 0d656ef..dbc623b 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -649,7 +649,7 @@ static void ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, // order number right after the N. MachineBasicBlock *BB = Emitter.getBlock(); MachineBasicBlock::iterator InsertPos = Emitter.getInsertPos(); - SmallVector<SDDbgValue*,2> &DVs = DAG->GetDbgValues(N); + ArrayRef<SDDbgValue*> DVs = DAG->GetDbgValues(N); for (unsigned i = 0, e = DVs.size(); i != e; ++i) { if (DVs[i]->isInvalidated()) continue; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 68eeb60..77ce54f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -598,7 +598,7 @@ void SelectionDAG::DeallocateNode(SDNode *N) { Ordering->remove(N); // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. - SmallVector<SDDbgValue*, 2> &DbgVals = DbgInfo->getSDDbgValues(N); + ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) DbgVals[i]->setIsInvalidated(); } @@ -5508,9 +5508,9 @@ void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { return; SDNode *FromNode = From.getNode(); SDNode *ToNode = To.getNode(); - SmallVector<SDDbgValue *, 2> &DVs = GetDbgValues(FromNode); + ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); SmallVector<SDDbgValue *, 2> ClonedDVs; - for (SmallVector<SDDbgValue *, 2>::iterator I = DVs.begin(), E = DVs.end(); + for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); I != E; ++I) { SDDbgValue *Dbg = *I; if (Dbg->getKind() == SDDbgValue::SDNODE) { diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index d79a5ae..19bfa33 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -475,17 +475,9 @@ static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL, PartVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { // Promoted vector extract - unsigned NumElts = ValueVT.getVectorNumElements(); - SmallVector<SDValue, 8> NewOps; - for (unsigned i = 0; i < NumElts; ++i) { - SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, - ValueVT.getScalarType(), Val ,DAG.getIntPtrConstant(i)); - SDValue Cast = DAG.getNode(ISD::ANY_EXTEND, - DL, PartVT.getScalarType(), Ext); - NewOps.push_back(Cast); - } - Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, - &NewOps[0], NewOps.size()); + bool Smaller = PartVT.bitsLE(ValueVT); + Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), + DL, PartVT, Val); } else{ // Vector -> scalar conversion. assert(ValueVT.getVectorNumElements() == 1 && diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index efbfaa4..474dd7a 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -81,6 +81,9 @@ static void InitLibcallNames(const char **Names) { Names[RTLIB::MUL_I32] = "__mulsi3"; Names[RTLIB::MUL_I64] = "__muldi3"; Names[RTLIB::MUL_I128] = "__multi3"; + Names[RTLIB::MULO_I32] = "__mulosi4"; + Names[RTLIB::MULO_I64] = "__mulodi4"; + Names[RTLIB::MULO_I128] = "__muloti4"; Names[RTLIB::SDIV_I8] = "__divqi3"; Names[RTLIB::SDIV_I16] = "__divhi3"; Names[RTLIB::SDIV_I32] = "__divsi3"; @@ -1914,7 +1917,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // comparisons. if (isa<ConstantSDNode>(N0.getNode())) return DAG.getSetCC(dl, VT, N1, N0, ISD::getSetCCSwappedOperands(Cond)); - + if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { const APInt &C1 = N1C->getAPIntValue(); @@ -2673,9 +2676,9 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops, SelectionDAG &DAG) const { - + if (Constraint.length() > 1) return; - + char ConstraintLetter = Constraint[0]; switch (ConstraintLetter) { default: break; @@ -2865,7 +2868,7 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints( report_fatal_error("Indirect operand for inline asm not a pointer!"); OpTy = PtrTy->getElementType(); } - + // Look for vector wrapped in a struct. e.g. { <16 x i8> }. if (const StructType *STy = dyn_cast<StructType>(OpTy)) if (STy->getNumElements() == 1) diff --git a/lib/CodeGen/ShadowStackGC.cpp b/lib/CodeGen/ShadowStackGC.cpp index 6ab0cb0..ff58b22 100644 --- a/lib/CodeGen/ShadowStackGC.cpp +++ b/lib/CodeGen/ShadowStackGC.cpp @@ -194,27 +194,31 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) { // Truncate the ShadowStackDescriptor if some metadata is null. unsigned NumMeta = 0; - SmallVector<Constant*,16> Metadata; + SmallVector<Constant*, 16> Metadata; for (unsigned I = 0; I != Roots.size(); ++I) { Constant *C = cast<Constant>(Roots[I].first->getArgOperand(1)); if (!C->isNullValue()) NumMeta = I + 1; Metadata.push_back(ConstantExpr::getBitCast(C, VoidPtr)); } + Metadata.resize(NumMeta); + const Type *Int32Ty = Type::getInt32Ty(F.getContext()); + Constant *BaseElts[] = { - ConstantInt::get(Type::getInt32Ty(F.getContext()), Roots.size(), false), - ConstantInt::get(Type::getInt32Ty(F.getContext()), NumMeta, false), + ConstantInt::get(Int32Ty, Roots.size(), false), + ConstantInt::get(Int32Ty, NumMeta, false), }; Constant *DescriptorElts[] = { - ConstantStruct::get(F.getContext(), BaseElts, 2, false), - ConstantArray::get(ArrayType::get(VoidPtr, NumMeta), - Metadata.begin(), NumMeta) + ConstantStruct::get(StructType::get(Int32Ty, Int32Ty, NULL), BaseElts), + ConstantArray::get(ArrayType::get(VoidPtr, NumMeta), Metadata) }; - Constant *FrameMap = ConstantStruct::get(F.getContext(), DescriptorElts, 2, - false); + Constant *FrameMap = + ConstantStruct::get(StructType::get(DescriptorElts[0]->getType(), + DescriptorElts[1]->getType(), NULL), + DescriptorElts); std::string TypeName("gc_map."); TypeName += utostr(NumMeta); diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp index 92970e4..c2565af 100644 --- a/lib/CodeGen/SjLjEHPrepare.cpp +++ b/lib/CodeGen/SjLjEHPrepare.cpp @@ -91,8 +91,7 @@ bool SjLjEHPass::doInitialization(Module &M) { Type::getInt8PtrTy(M.getContext()); const Type *Int32Ty = Type::getInt32Ty(M.getContext()); FunctionContextTy = - StructType::get(M.getContext(), - VoidPtrTy, // __prev + StructType::get(VoidPtrTy, // __prev Int32Ty, // call_site ArrayType::get(Int32Ty, 4), // __data VoidPtrTy, // __personality diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index 90cb72f..e5c23b3 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -96,6 +96,12 @@ namespace { bool TailDuplicateBlocks(MachineFunction &MF); bool shouldTailDuplicate(const MachineFunction &MF, MachineBasicBlock &TailBB); + bool isSimpleBB(MachineBasicBlock *TailBB); + bool canCompletelyDuplicateSimpleBB(MachineBasicBlock &BB); + bool duplicateSimpleBB(MachineBasicBlock *TailBB, + SmallVector<MachineBasicBlock*, 8> &TDBBs, + const DenseSet<unsigned> &RegsUsedByPhi, + SmallVector<MachineInstr*, 16> &Copies); bool TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF, SmallVector<MachineBasicBlock*, 8> &TDBBs, SmallVector<MachineInstr*, 16> &Copies); @@ -201,7 +207,7 @@ bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) { // TailBB's immediate successors are now successors of those predecessors // which duplicated TailBB. Add the predecessors as sources to the PHI // instructions. - bool isDead = MBB->pred_empty(); + bool isDead = MBB->pred_empty() && !MBB->hasAddressTaken(); if (PreRegAlloc) UpdateSuccessorsPHIs(MBB, isDead, TDBBs, Succs); @@ -242,6 +248,14 @@ bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) { MachineOperand &UseMO = UI.getOperand(); MachineInstr *UseMI = &*UI; ++UI; + if (UseMI->isDebugValue()) { + // SSAUpdate can replace the use with an undef. That creates + // a debug instruction that is a kill. + // FIXME: Should it SSAUpdate job to delete debug instructions + // instead of replacing the use with undef? + UseMI->eraseFromParent(); + continue; + } if (UseMI->getParent() == DefBB && !UseMI->isPHI()) continue; SSAUpdate.RewriteUse(UseMO); @@ -283,6 +297,8 @@ static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB, for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg), UE = MRI->use_end(); UI != UE; ++UI) { MachineInstr *UseMI = &*UI; + if (UseMI->isDebugValue()) + continue; if (UseMI->getParent() != BB) return true; } @@ -547,6 +563,141 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, return true; } +/// isSimpleBB - True if this BB has only one unconditional jump. +bool +TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) { + if (TailBB->succ_size() != 1) + return false; + MachineBasicBlock::iterator I = TailBB->begin(); + MachineBasicBlock::iterator E = TailBB->end(); + while (I != E && I->isDebugValue()) + ++I; + if (I == E) + return true; + return I->getDesc().isUnconditionalBranch(); +} + +static bool +bothUsedInPHI(const MachineBasicBlock &A, + SmallPtrSet<MachineBasicBlock*, 8> SuccsB) { + for (MachineBasicBlock::const_succ_iterator SI = A.succ_begin(), + SE = A.succ_end(); SI != SE; ++SI) { + MachineBasicBlock *BB = *SI; + if (SuccsB.count(BB) && !BB->empty() && BB->begin()->isPHI()) + return true; + } + + return false; +} + +bool +TailDuplicatePass::canCompletelyDuplicateSimpleBB(MachineBasicBlock &BB) { + SmallPtrSet<MachineBasicBlock*, 8> Succs(BB.succ_begin(), BB.succ_end()); + + for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(), + PE = BB.pred_end(); PI != PE; ++PI) { + MachineBasicBlock *PredBB = *PI; + if (PredBB->getLandingPadSuccessor()) + return false; + if (bothUsedInPHI(*PredBB, Succs)) + return false; + MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; + SmallVector<MachineOperand, 4> PredCond; + if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) + return false; + } + return true; +} + +bool +TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, + SmallVector<MachineBasicBlock*, 8> &TDBBs, + const DenseSet<unsigned> &UsedByPhi, + SmallVector<MachineInstr*, 16> &Copies) { + if (!canCompletelyDuplicateSimpleBB(*TailBB)) + return false; + + bool Changed = false; + SmallVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(), + TailBB->pred_end()); + for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), + PE = Preds.end(); PI != PE; ++PI) { + MachineBasicBlock *PredBB = *PI; + + MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; + SmallVector<MachineOperand, 4> PredCond; + bool NotAnalyzable = + TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true); + (void)NotAnalyzable; + assert(!NotAnalyzable && "Cannot duplicate this!"); + + DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB + << "From simple Succ: " << *TailBB); + + MachineBasicBlock *NewTarget = *TailBB->succ_begin(); + MachineBasicBlock *NextBB = llvm::next(MachineFunction::iterator(PredBB)); + + DenseMap<unsigned, unsigned> LocalVRMap; + SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos; + for (MachineBasicBlock::iterator I = TailBB->begin(); + I != TailBB->end() && I->isPHI();) { + MachineInstr *MI = &*I; + ++I; + ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true); + } + MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator(); + for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { + Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(), + TII->get(TargetOpcode::COPY), + CopyInfos[i].first).addReg(CopyInfos[i].second)); + } + + // Make PredFBB explicit. + if (PredCond.empty()) + PredFBB = PredTBB; + + // Make fall through explicit. + if (!PredTBB) + PredTBB = NextBB; + if (!PredFBB) + PredFBB = NextBB; + + // Redirect + if (PredFBB == TailBB) + PredFBB = NewTarget; + if (PredTBB == TailBB) + PredTBB = NewTarget; + + // Make the branch unconditional if possible + if (PredTBB == PredFBB) { + PredCond.clear(); + PredFBB = NULL; + } + + // Avoid adding fall through branches. + if (PredFBB == NextBB) + PredFBB = NULL; + if (PredTBB == NextBB && PredFBB == NULL) + PredTBB = NULL; + + TII->RemoveBranch(*PredBB); + + if (PredTBB) + TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc()); + + PredBB->removeSuccessor(TailBB); + unsigned NumSuccessors = PredBB->succ_size(); + assert(NumSuccessors <= 1); + if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget) + PredBB->addSuccessor(NewTarget); + + TDBBs.push_back(PredBB); + + Changed = true; + } + return Changed; +} + /// TailDuplicate - If it is profitable, duplicate TailBB's contents in each /// of its predecessors. bool @@ -558,14 +709,18 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF, DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n'); + DenseSet<unsigned> UsedByPhi; + getRegsUsedByPHIs(*TailBB, &UsedByPhi); + + if (isSimpleBB(TailBB)) + return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies); + // Iterate through all the unique predecessors and tail-duplicate this // block into them, if possible. Copying the list ahead of time also // avoids trouble with the predecessor list reallocating. bool Changed = false; SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(), TailBB->pred_end()); - DenseSet<unsigned> UsedByPhi; - getRegsUsedByPHIs(*TailBB, &UsedByPhi); for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), PE = Preds.end(); PI != PE; ++PI) { MachineBasicBlock *PredBB = *PI; diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 2da1bd4..df42b2c 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -487,8 +487,12 @@ void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx, // .comm doesn't support alignment before Leopard. Triple T(((LLVMTargetMachine&)TM).getTargetTriple()); - if (T.isMacOSX() && T.isMacOSXVersionLT(10, 5)) - CommDirectiveSupportsAlignment = false; + if (T.isMacOSX()) { + if (T.isMacOSXVersionLT(10, 5)) + CommDirectiveSupportsAlignment = false; + if (!T.isMacOSXVersionLT(10, 6)) + SupportsCompactUnwindInfo = true; + } TargetLoweringObjectFile::Initialize(Ctx, TM); @@ -605,6 +609,12 @@ void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx, // Exception Handling. LSDASection = getContext().getMachOSection("__TEXT", "__gcc_except_tab", 0, SectionKind::getReadOnlyWithRel()); + + CompactUnwindSection = + getContext().getMachOSection("__LD", "__compact_unwind", + MCSectionMachO::S_ATTR_DEBUG, + SectionKind::getReadOnly()); + // Debug Information. DwarfAbbrevSection = getContext().getMachOSection("__DWARF", "__debug_abbrev", diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index f54d879..3860e0b 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -1095,12 +1095,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { "two address instruction invalid"); unsigned regB = mi->getOperand(SrcIdx).getReg(); - TiedOperandMap::iterator OI = TiedOperands.find(regB); - if (OI == TiedOperands.end()) { - SmallVector<std::pair<unsigned, unsigned>, 4> TiedPair; - OI = TiedOperands.insert(std::make_pair(regB, TiedPair)).first; - } - OI->second.push_back(std::make_pair(SrcIdx, DstIdx)); + TiedOperands[regB].push_back(std::make_pair(SrcIdx, DstIdx)); } // Now iterate over the information collected above. |