From 4c5e43da7792f75567b693105cc53e3f1992ad98 Mon Sep 17 00:00:00 2001 From: Pirama Arumuga Nainar Date: Wed, 8 Apr 2015 08:55:49 -0700 Subject: Update aosp/master llvm for rebase to r233350 Change-Id: I07d935f8793ee8ec6b7da003f6483046594bca49 --- lib/Analysis/AliasAnalysis.cpp | 10 +- lib/Analysis/AliasAnalysisCounter.cpp | 3 +- lib/Analysis/AliasDebugger.cpp | 2 +- lib/Analysis/Analysis.cpp | 1 - lib/Analysis/Android.mk | 1 - lib/Analysis/BasicAliasAnalysis.cpp | 62 +- lib/Analysis/BranchProbabilityInfo.cpp | 1 + lib/Analysis/CFLAliasAnalysis.cpp | 239 ++- lib/Analysis/CMakeLists.txt | 1 - lib/Analysis/CodeMetrics.cpp | 1 + lib/Analysis/ConstantFolding.cpp | 259 ++- lib/Analysis/DependenceAnalysis.cpp | 62 +- lib/Analysis/IPA/CallGraphSCCPass.cpp | 41 +- lib/Analysis/IPA/GlobalsModRef.cpp | 12 +- lib/Analysis/IPA/InlineCost.cpp | 61 +- lib/Analysis/IVUsers.cpp | 9 +- lib/Analysis/InstructionSimplify.cpp | 120 +- lib/Analysis/JumpInstrTableInfo.cpp | 55 - lib/Analysis/LazyValueInfo.cpp | 110 +- lib/Analysis/LibCallAliasAnalysis.cpp | 6 +- lib/Analysis/LibCallSemantics.cpp | 12 - lib/Analysis/Lint.cpp | 378 ++-- lib/Analysis/Loads.cpp | 28 +- lib/Analysis/LoopAccessAnalysis.cpp | 417 ++-- lib/Analysis/LoopInfo.cpp | 1 + lib/Analysis/LoopPass.cpp | 1 + lib/Analysis/MemDerefPrinter.cpp | 5 +- lib/Analysis/MemoryBuiltins.cpp | 63 +- lib/Analysis/MemoryDependenceAnalysis.cpp | 67 +- lib/Analysis/ModuleDebugInfoPrinter.cpp | 62 +- lib/Analysis/NoAliasAnalysis.cpp | 7 +- lib/Analysis/PHITransAddr.cpp | 7 +- lib/Analysis/RegionPass.cpp | 25 +- lib/Analysis/ScalarEvolution.cpp | 629 +++--- lib/Analysis/ScalarEvolutionAliasAnalysis.cpp | 3 +- lib/Analysis/ScalarEvolutionExpander.cpp | 121 +- lib/Analysis/ScopedNoAliasAA.cpp | 7 +- lib/Analysis/TargetLibraryInfo.cpp | 487 ++--- lib/Analysis/TargetTransformInfo.cpp | 11 +- lib/Analysis/TypeBasedAliasAnalysis.cpp | 22 +- lib/Analysis/ValueTracking.cpp | 741 ++++--- lib/AsmParser/LLParser.cpp | 101 +- lib/AsmParser/Parser.cpp | 1 + lib/Bitcode/Reader/BitcodeReader.cpp | 447 ++++- lib/Bitcode/Reader/BitcodeReader.h | 369 ---- lib/Bitcode/Reader/BitstreamReader.cpp | 2 +- lib/Bitcode/Writer/BitcodeWriter.cpp | 9 +- lib/CMakeLists.txt | 1 + lib/CodeGen/Analysis.cpp | 9 +- lib/CodeGen/Android.mk | 4 +- lib/CodeGen/AsmPrinter/ARMException.cpp | 17 +- lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 199 +- lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp | 121 +- lib/CodeGen/AsmPrinter/AsmPrinterHandler.h | 4 + lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp | 37 +- lib/CodeGen/AsmPrinter/ByteStreamer.h | 29 + lib/CodeGen/AsmPrinter/DIE.cpp | 85 +- lib/CodeGen/AsmPrinter/DIEHash.cpp | 6 +- .../AsmPrinter/DbgValueHistoryCalculator.cpp | 1 + lib/CodeGen/AsmPrinter/DebugLocEntry.h | 39 +- lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp | 63 +- lib/CodeGen/AsmPrinter/DwarfAccelTable.h | 6 +- lib/CodeGen/AsmPrinter/DwarfCFIException.cpp | 51 +- lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp | 43 +- lib/CodeGen/AsmPrinter/DwarfCompileUnit.h | 24 +- lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 466 ++--- lib/CodeGen/AsmPrinter/DwarfDebug.h | 56 +- lib/CodeGen/AsmPrinter/DwarfException.h | 45 +- lib/CodeGen/AsmPrinter/DwarfExpression.cpp | 78 +- lib/CodeGen/AsmPrinter/DwarfExpression.h | 31 +- lib/CodeGen/AsmPrinter/DwarfFile.cpp | 25 +- lib/CodeGen/AsmPrinter/DwarfFile.h | 7 +- lib/CodeGen/AsmPrinter/DwarfStringPool.cpp | 2 +- lib/CodeGen/AsmPrinter/DwarfUnit.cpp | 48 +- lib/CodeGen/AsmPrinter/DwarfUnit.h | 11 +- lib/CodeGen/AsmPrinter/EHStreamer.cpp | 29 +- lib/CodeGen/AsmPrinter/EHStreamer.h | 10 - lib/CodeGen/AsmPrinter/Win64Exception.cpp | 14 +- lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp | 12 +- lib/CodeGen/AtomicExpandPass.cpp | 33 +- lib/CodeGen/BranchFolding.cpp | 61 +- lib/CodeGen/CMakeLists.txt | 2 - lib/CodeGen/CodeGen.cpp | 20 +- lib/CodeGen/CodeGenPrepare.cpp | 231 ++- lib/CodeGen/DwarfEHPrepare.cpp | 109 +- lib/CodeGen/ExecutionDepsFix.cpp | 35 +- lib/CodeGen/ForwardControlFlowIntegrity.cpp | 374 ---- lib/CodeGen/IfConversion.cpp | 18 +- lib/CodeGen/InterferenceCache.cpp | 3 +- lib/CodeGen/InterferenceCache.h | 4 +- lib/CodeGen/JumpInstrTables.cpp | 296 --- lib/CodeGen/LLVMTargetMachine.cpp | 79 +- lib/CodeGen/LatencyPriorityQueue.cpp | 13 - lib/CodeGen/LiveDebugVariables.cpp | 11 +- lib/CodeGen/LiveInterval.cpp | 13 +- lib/CodeGen/LiveIntervalAnalysis.cpp | 8 +- lib/CodeGen/LivePhysRegs.cpp | 1 + lib/CodeGen/LiveRangeCalc.cpp | 4 +- lib/CodeGen/LiveRangeCalc.h | 2 +- lib/CodeGen/LiveStackAnalysis.cpp | 6 +- lib/CodeGen/LiveVariables.cpp | 1 + lib/CodeGen/LocalStackSlotAllocation.cpp | 12 +- lib/CodeGen/MachineBasicBlock.cpp | 2 +- lib/CodeGen/MachineBlockPlacement.cpp | 431 +++-- lib/CodeGen/MachineCSE.cpp | 1 + lib/CodeGen/MachineCopyPropagation.cpp | 7 +- lib/CodeGen/MachineDominators.cpp | 66 + lib/CodeGen/MachineFunction.cpp | 18 +- lib/CodeGen/MachineInstr.cpp | 67 +- lib/CodeGen/MachineLICM.cpp | 61 + lib/CodeGen/MachineLoopInfo.cpp | 1 + lib/CodeGen/MachineRegisterInfo.cpp | 2 +- lib/CodeGen/MachineScheduler.cpp | 11 + lib/CodeGen/MachineVerifier.cpp | 64 +- lib/CodeGen/PHIElimination.cpp | 21 +- lib/CodeGen/Passes.cpp | 50 +- lib/CodeGen/PeepholeOptimizer.cpp | 6 +- lib/CodeGen/PrologEpilogInserter.cpp | 60 +- lib/CodeGen/PrologEpilogInserter.h | 78 - lib/CodeGen/RegAllocBase.cpp | 1 + lib/CodeGen/RegAllocGreedy.cpp | 3 +- lib/CodeGen/RegAllocPBQP.cpp | 83 +- lib/CodeGen/RegisterClassInfo.cpp | 7 +- lib/CodeGen/RegisterCoalescer.cpp | 170 +- lib/CodeGen/RegisterPressure.cpp | 2 + lib/CodeGen/ScheduleDAGInstrs.cpp | 107 +- lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 991 ++++++---- lib/CodeGen/SelectionDAG/FastISel.cpp | 10 +- lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp | 1 + lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 154 +- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp | 101 +- lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 10 + lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp | 16 +- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 76 +- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 735 +++---- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h | 45 +- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp | 1 + lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 79 +- lib/CodeGen/SelectionDAG/StatepointLowering.cpp | 67 +- lib/CodeGen/SelectionDAG/TargetLowering.cpp | 9 +- lib/CodeGen/ShadowStackGCLowering.cpp | 35 +- lib/CodeGen/SjLjEHPrepare.cpp | 3 +- lib/CodeGen/SlotIndexes.cpp | 2 +- lib/CodeGen/StackColoring.cpp | 2 +- lib/CodeGen/StackMapLivenessAnalysis.cpp | 64 +- lib/CodeGen/StackMaps.cpp | 186 +- lib/CodeGen/StackSlotColoring.cpp | 12 +- lib/CodeGen/TargetInstrInfo.cpp | 45 +- lib/CodeGen/TargetLoweringBase.cpp | 120 +- lib/CodeGen/TargetLoweringObjectFileImpl.cpp | 155 +- lib/CodeGen/TwoAddressInstructionPass.cpp | 63 + lib/CodeGen/VirtRegMap.cpp | 2 +- lib/CodeGen/WinEHPrepare.cpp | 1629 ++++++++++++---- lib/DebugInfo/DWARF/DWARFDebugFrame.cpp | 1 + lib/DebugInfo/DWARF/DWARFDebugLoc.cpp | 3 +- lib/DebugInfo/DWARF/DWARFFormValue.cpp | 19 + lib/DebugInfo/PDB/CMakeLists.txt | 2 +- lib/DebugInfo/PDB/DIA/DIASession.cpp | 45 +- lib/DebugInfo/PDB/PDB.cpp | 8 +- lib/DebugInfo/PDB/PDBExtras.cpp | 39 +- lib/DebugInfo/PDB/PDBSymDumper.cpp | 93 +- lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolBlock.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolCompiland.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolCustom.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolData.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolExe.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolFunc.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolLabel.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolThunk.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp | 15 +- lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolUnknown.cpp | 5 +- lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp | 5 +- lib/ExecutionEngine/ExecutionEngine.cpp | 38 +- lib/ExecutionEngine/Interpreter/CMakeLists.txt | 2 +- lib/ExecutionEngine/Interpreter/Execution.cpp | 17 +- lib/ExecutionEngine/MCJIT/MCJIT.cpp | 7 +- lib/ExecutionEngine/Orc/IndirectionUtils.cpp | 1 + lib/ExecutionEngine/Orc/OrcMCJITReplacement.h | 4 +- lib/ExecutionEngine/Orc/OrcTargetSupport.cpp | 6 +- lib/ExecutionEngine/RuntimeDyld/Android.mk | 1 + lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt | 1 + lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp | 81 +- .../RuntimeDyld/RuntimeDyldCOFF.cpp | 85 + lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h | 46 + .../RuntimeDyld/RuntimeDyldChecker.cpp | 18 +- .../RuntimeDyld/RuntimeDyldCheckerImpl.h | 2 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp | 2 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h | 10 - lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h | 49 +- .../RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h | 214 +++ lib/Fuzzer/FuzzerDriver.cpp | 1 + lib/Fuzzer/FuzzerFlags.def | 1 + lib/Fuzzer/FuzzerInternal.h | 10 + lib/Fuzzer/FuzzerLoop.cpp | 14 +- lib/Fuzzer/test/CMakeLists.txt | 1 + lib/Fuzzer/test/CounterTest.cpp | 14 + lib/Fuzzer/test/fuzzer.test | 3 + lib/IR/AsmWriter.cpp | 975 +++++----- lib/IR/AsmWriter.h | 129 -- lib/IR/AutoUpgrade.cpp | 152 +- lib/IR/BasicBlock.cpp | 24 +- lib/IR/ConstantFold.cpp | 59 +- lib/IR/ConstantRange.cpp | 45 +- lib/IR/Constants.cpp | 25 +- lib/IR/Core.cpp | 2 +- lib/IR/DIBuilder.cpp | 707 ++----- lib/IR/DataLayout.cpp | 134 +- lib/IR/DebugInfo.cpp | 833 ++------ lib/IR/DiagnosticInfo.cpp | 2 +- lib/IR/GCOV.cpp | 7 +- lib/IR/Globals.cpp | 4 - lib/IR/InlineAsm.cpp | 2 +- lib/IR/Instruction.cpp | 8 +- lib/IR/Instructions.cpp | 253 +-- lib/IR/LLVMContextImpl.h | 6 +- lib/IR/LegacyPassManager.cpp | 2 +- lib/IR/Mangler.cpp | 6 +- lib/IR/Module.cpp | 30 +- lib/IR/TypeFinder.cpp | 2 +- lib/IR/Value.cpp | 46 +- lib/IR/Verifier.cpp | 2012 ++++++++++---------- lib/LLVMBuild.txt | 2 +- lib/LTO/LTOCodeGenerator.cpp | 38 +- lib/LTO/LTOModule.cpp | 5 +- lib/Linker/LinkModules.cpp | 66 +- lib/MC/ELFObjectWriter.cpp | 39 +- lib/MC/MCAsmInfo.cpp | 2 + lib/MC/MCAsmInfoDarwin.cpp | 1 - lib/MC/MCAsmStreamer.cpp | 2 +- lib/MC/MCAssembler.cpp | 65 +- lib/MC/MCContext.cpp | 155 +- lib/MC/MCDwarf.cpp | 44 +- lib/MC/MCELFStreamer.cpp | 10 +- lib/MC/MCExpr.cpp | 119 +- lib/MC/MCLinkerOptimizationHint.cpp | 2 +- lib/MC/MCMachOStreamer.cpp | 50 +- lib/MC/MCObjectFileInfo.cpp | 521 +++-- lib/MC/MCObjectStreamer.cpp | 14 +- lib/MC/MCObjectWriter.cpp | 10 +- lib/MC/MCParser/AsmLexer.cpp | 2 +- lib/MC/MCParser/AsmParser.cpp | 44 +- lib/MC/MCParser/DarwinAsmParser.cpp | 2 +- lib/MC/MCSection.cpp | 9 + lib/MC/MCSectionMachO.cpp | 6 +- lib/MC/MCStreamer.cpp | 28 + lib/MC/MCWinEH.cpp | 1 + lib/MC/MachObjectWriter.cpp | 1 + lib/MC/SubtargetFeature.cpp | 20 +- lib/MC/WinCOFFObjectWriter.cpp | 8 + lib/MC/WinCOFFStreamer.cpp | 4 +- lib/Makefile | 2 +- lib/Object/Archive.cpp | 61 +- lib/Object/COFFObjectFile.cpp | 13 +- lib/Object/ELFYAML.cpp | 1 + lib/Object/IRObjectFile.cpp | 14 +- lib/Option/Arg.cpp | 25 +- lib/Option/ArgList.cpp | 34 +- lib/Option/OptTable.cpp | 14 +- lib/Option/Option.cpp | 3 - lib/Passes/Android.mk | 30 + lib/Passes/CMakeLists.txt | 8 + lib/Passes/LLVMBuild.txt | 22 + lib/Passes/Makefile | 14 + lib/Passes/PassBuilder.cpp | 412 ++++ lib/Passes/PassRegistry.def | 77 + lib/ProfileData/CoverageMapping.cpp | 6 +- lib/ProfileData/CoverageMappingReader.cpp | 161 +- lib/ProfileData/InstrProfReader.cpp | 3 +- lib/Support/APFloat.cpp | 6 +- lib/Support/APInt.cpp | 70 +- lib/Support/Allocator.cpp | 5 - lib/Support/Android.mk | 2 - lib/Support/CMakeLists.txt | 2 - lib/Support/CommandLine.cpp | 29 +- lib/Support/Compression.cpp | 1 + lib/Support/CrashRecoveryContext.cpp | 2 - lib/Support/DAGDeltaAlgorithm.cpp | 26 +- lib/Support/DataStream.cpp | 2 - lib/Support/Debug.cpp | 1 + lib/Support/FileOutputBuffer.cpp | 14 +- lib/Support/FoldingSet.cpp | 7 +- lib/Support/FormattedStream.cpp | 1 + lib/Support/GraphWriter.cpp | 3 +- lib/Support/Host.cpp | 8 +- lib/Support/IsInf.cpp | 49 - lib/Support/IsNAN.cpp | 33 - lib/Support/LockFileManager.cpp | 27 +- lib/Support/MemoryBuffer.cpp | 1 - lib/Support/Path.cpp | 15 +- lib/Support/Process.cpp | 2 +- lib/Support/Program.cpp | 1 + lib/Support/RandomNumberGenerator.cpp | 6 +- lib/Support/Regex.cpp | 3 +- lib/Support/ScaledNumber.cpp | 1 + lib/Support/SourceMgr.cpp | 2 - lib/Support/SpecialCaseList.cpp | 2 - lib/Support/StreamingMemoryObject.cpp | 3 - lib/Support/StringExtras.cpp | 1 - lib/Support/SystemUtils.cpp | 2 - lib/Support/TargetRegistry.cpp | 1 - lib/Support/Timer.cpp | 2 - lib/Support/Triple.cpp | 46 +- lib/Support/Twine.cpp | 14 +- lib/Support/Unix/Program.inc | 3 +- lib/Support/Unix/Signals.inc | 44 +- lib/Support/Windows/Path.inc | 4 +- lib/Support/Windows/Process.inc | 13 + lib/Support/Windows/Signals.inc | 222 ++- lib/Support/YAMLParser.cpp | 1 + lib/Support/YAMLTraits.cpp | 22 +- lib/TableGen/TGParser.cpp | 1 + lib/Target/AArch64/AArch64.td | 7 + lib/Target/AArch64/AArch64A53Fix835769.cpp | 1 + lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp | 26 +- lib/Target/AArch64/AArch64AddressTypePromotion.cpp | 1 + lib/Target/AArch64/AArch64AsmPrinter.cpp | 77 +- .../AArch64/AArch64CleanupLocalDynamicTLSPass.cpp | 6 +- lib/Target/AArch64/AArch64CollectLOH.cpp | 14 +- lib/Target/AArch64/AArch64FastISel.cpp | 4 +- lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 47 +- lib/Target/AArch64/AArch64ISelLowering.cpp | 302 +-- lib/Target/AArch64/AArch64ISelLowering.h | 35 +- lib/Target/AArch64/AArch64InstrInfo.cpp | 10 +- lib/Target/AArch64/AArch64InstrInfo.h | 7 +- lib/Target/AArch64/AArch64InstrInfo.td | 297 +-- lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp | 127 +- lib/Target/AArch64/AArch64MCInstLower.cpp | 11 + lib/Target/AArch64/AArch64PBQPRegAlloc.cpp | 2 +- lib/Target/AArch64/AArch64PromoteConstant.cpp | 7 +- lib/Target/AArch64/AArch64RegisterInfo.cpp | 47 +- lib/Target/AArch64/AArch64RegisterInfo.h | 19 +- lib/Target/AArch64/AArch64Subtarget.cpp | 2 +- lib/Target/AArch64/AArch64Subtarget.h | 3 + lib/Target/AArch64/AArch64TargetMachine.cpp | 31 +- lib/Target/AArch64/AArch64TargetMachine.h | 7 - lib/Target/AArch64/AArch64TargetObjectFile.cpp | 21 + lib/Target/AArch64/AArch64TargetObjectFile.h | 7 + lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 12 +- lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp | 125 +- .../AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 21 +- .../AArch64/MCTargetDesc/AArch64ELFStreamer.cpp | 23 +- .../AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp | 7 +- lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp | 1 + .../AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp | 129 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.h | 20 +- lib/Target/AArch64/Utils/AArch64BaseInfo.cpp | 96 +- lib/Target/AArch64/Utils/AArch64BaseInfo.h | 51 +- lib/Target/ARM/A15SDOptimizer.cpp | 12 +- lib/Target/ARM/ARM.td | 30 +- lib/Target/ARM/ARMAsmPrinter.cpp | 72 +- lib/Target/ARM/ARMAsmPrinter.h | 9 +- lib/Target/ARM/ARMBaseInstrInfo.cpp | 34 +- lib/Target/ARM/ARMBaseRegisterInfo.cpp | 83 +- lib/Target/ARM/ARMBaseRegisterInfo.h | 27 +- lib/Target/ARM/ARMFastISel.cpp | 4 +- lib/Target/ARM/ARMFrameLowering.cpp | 5 +- lib/Target/ARM/ARMISelDAGToDAG.cpp | 9 +- lib/Target/ARM/ARMISelLowering.cpp | 457 ++--- lib/Target/ARM/ARMISelLowering.h | 33 +- lib/Target/ARM/ARMInstrFormats.td | 7 +- lib/Target/ARM/ARMInstrInfo.cpp | 7 +- lib/Target/ARM/ARMInstrInfo.td | 15 +- lib/Target/ARM/ARMInstrNEON.td | 215 ++- lib/Target/ARM/ARMInstrVFP.td | 107 +- lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 3 +- lib/Target/ARM/ARMMachineFunctionInfo.h | 6 +- lib/Target/ARM/ARMRegisterInfo.cpp | 4 +- lib/Target/ARM/ARMRegisterInfo.h | 2 +- lib/Target/ARM/ARMSubtarget.cpp | 2 + lib/Target/ARM/ARMSubtarget.h | 8 +- lib/Target/ARM/ARMTargetMachine.cpp | 66 +- lib/Target/ARM/ARMTargetMachine.h | 4 +- lib/Target/ARM/Android.mk | 4 +- lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 9 +- lib/Target/ARM/CMakeLists.txt | 3 +- lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp | 4 +- lib/Target/ARM/MCTargetDesc/ARMArchName.def | 3 + lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp | 29 +- lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp | 1 + lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp | 2 - lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp | 1 + lib/Target/ARM/MCTargetDesc/ARMMCExpr.h | 4 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp | 183 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h | 21 +- lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp | 72 +- lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp | 10 +- lib/Target/ARM/MLxExpansionPass.cpp | 3 + lib/Target/ARM/README-Thumb.txt | 2 +- lib/Target/ARM/Thumb1FrameLowering.cpp | 22 +- lib/Target/ARM/Thumb1FrameLowering.h | 2 +- lib/Target/ARM/Thumb1InstrInfo.cpp | 3 +- lib/Target/ARM/Thumb1InstrInfo.h | 6 +- lib/Target/ARM/Thumb1RegisterInfo.cpp | 577 ------ lib/Target/ARM/Thumb1RegisterInfo.h | 63 - lib/Target/ARM/Thumb2ITBlockPass.cpp | 2 + lib/Target/ARM/Thumb2InstrInfo.cpp | 3 +- lib/Target/ARM/Thumb2InstrInfo.h | 6 +- lib/Target/ARM/Thumb2RegisterInfo.cpp | 53 - lib/Target/ARM/Thumb2RegisterInfo.h | 38 - lib/Target/ARM/Thumb2SizeReduction.cpp | 4 + lib/Target/ARM/ThumbRegisterInfo.cpp | 623 ++++++ lib/Target/ARM/ThumbRegisterInfo.h | 65 + lib/Target/BPF/BPFISelDAGToDAG.cpp | 6 +- lib/Target/BPF/BPFISelLowering.h | 1 + lib/Target/BPF/BPFRegisterInfo.h | 3 +- lib/Target/BPF/BPFTargetMachine.cpp | 4 +- lib/Target/BPF/BPFTargetMachine.h | 7 +- lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp | 1 - lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp | 10 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h | 1 - lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp | 2 +- lib/Target/CppBackend/CPPBackend.cpp | 3 +- lib/Target/CppBackend/CPPTargetMachine.h | 15 +- lib/Target/Hexagon/CMakeLists.txt | 1 - lib/Target/Hexagon/Hexagon.h | 1 - lib/Target/Hexagon/Hexagon.td | 16 +- lib/Target/Hexagon/HexagonCopyToCombine.cpp | 17 +- lib/Target/Hexagon/HexagonExpandPredSpillCode.cpp | 161 +- lib/Target/Hexagon/HexagonFrameLowering.cpp | 2 +- lib/Target/Hexagon/HexagonHardwareLoops.cpp | 2 +- lib/Target/Hexagon/HexagonISelDAGToDAG.cpp | 1555 +++++++-------- lib/Target/Hexagon/HexagonISelLowering.cpp | 896 ++++++++- lib/Target/Hexagon/HexagonISelLowering.h | 81 +- lib/Target/Hexagon/HexagonInstrFormats.td | 63 +- lib/Target/Hexagon/HexagonInstrFormatsV4.td | 94 +- lib/Target/Hexagon/HexagonInstrInfo.cpp | 179 +- lib/Target/Hexagon/HexagonInstrInfo.h | 22 +- lib/Target/Hexagon/HexagonInstrInfo.td | 473 ++--- lib/Target/Hexagon/HexagonInstrInfoV4.td | 465 ++--- lib/Target/Hexagon/HexagonInstrInfoV5.td | 8 +- lib/Target/Hexagon/HexagonInstrInfoVector.td | 418 ++++ lib/Target/Hexagon/HexagonIntrinsics.td | 49 +- lib/Target/Hexagon/HexagonIntrinsicsV4.td | 10 +- lib/Target/Hexagon/HexagonNewValueJump.cpp | 6 +- lib/Target/Hexagon/HexagonOperands.td | 388 +--- lib/Target/Hexagon/HexagonPeephole.cpp | 7 - lib/Target/Hexagon/HexagonRegisterInfo.cpp | 15 +- lib/Target/Hexagon/HexagonRegisterInfo.h | 12 +- .../Hexagon/HexagonSplitConst32AndConst64.cpp | 85 +- lib/Target/Hexagon/HexagonSplitTFRCondSets.cpp | 172 -- lib/Target/Hexagon/HexagonSubtarget.cpp | 10 + lib/Target/Hexagon/HexagonSubtarget.h | 5 + lib/Target/Hexagon/HexagonTargetMachine.cpp | 23 +- lib/Target/Hexagon/HexagonTargetMachine.h | 4 +- lib/Target/Hexagon/HexagonVLIWPacketizer.cpp | 9 +- .../MCTargetDesc/HexagonELFObjectWriter.cpp | 1 + .../Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp | 10 +- .../Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h | 4 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp | 22 - .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.h | 1 - .../MSP430/MCTargetDesc/MSP430MCTargetDesc.h | 2 + lib/Target/MSP430/MSP430ISelDAGToDAG.cpp | 8 +- lib/Target/MSP430/MSP430ISelLowering.h | 6 + lib/Target/MSP430/MSP430RegisterInfo.h | 3 +- lib/Target/MSP430/MSP430Subtarget.cpp | 3 +- lib/Target/MSP430/MSP430TargetMachine.cpp | 5 +- lib/Target/MSP430/MSP430TargetMachine.h | 4 +- lib/Target/Mips/AsmParser/MipsAsmParser.cpp | 99 +- lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h | 7 +- .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 16 +- lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp | 12 +- lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h | 4 +- lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h | 2 +- lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h | 1 - lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp | 148 +- lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h | 2 - .../Mips/MCTargetDesc/MipsNaClELFStreamer.cpp | 8 +- .../Mips/MCTargetDesc/MipsTargetStreamer.cpp | 36 +- lib/Target/Mips/MicroMipsInstrInfo.td | 8 +- lib/Target/Mips/Mips.h | 5 + lib/Target/Mips/Mips.td | 12 +- lib/Target/Mips/Mips16HardFloat.cpp | 185 +- lib/Target/Mips/Mips16HardFloat.h | 43 - lib/Target/Mips/Mips16InstrInfo.cpp | 4 +- lib/Target/Mips/Mips16InstrInfo.h | 2 +- lib/Target/Mips/Mips16RegisterInfo.cpp | 9 +- lib/Target/Mips/Mips16RegisterInfo.h | 2 +- lib/Target/Mips/Mips64InstrInfo.td | 4 +- lib/Target/Mips/MipsAsmPrinter.cpp | 27 +- lib/Target/Mips/MipsCallingConv.td | 2 +- lib/Target/Mips/MipsDelaySlotFiller.cpp | 36 +- lib/Target/Mips/MipsFastISel.cpp | 92 +- lib/Target/Mips/MipsISelDAGToDAG.cpp | 17 +- lib/Target/Mips/MipsISelDAGToDAG.h | 2 +- lib/Target/Mips/MipsISelLowering.cpp | 46 +- lib/Target/Mips/MipsISelLowering.h | 11 + lib/Target/Mips/MipsInstrInfo.h | 2 +- lib/Target/Mips/MipsInstrInfo.td | 24 +- lib/Target/Mips/MipsMCInstLower.cpp | 1 + lib/Target/Mips/MipsMachineFunction.cpp | 17 +- lib/Target/Mips/MipsModuleISelDAGToDAG.cpp | 38 +- lib/Target/Mips/MipsModuleISelDAGToDAG.h | 58 - lib/Target/Mips/MipsOs16.cpp | 129 +- lib/Target/Mips/MipsOs16.h | 47 - lib/Target/Mips/MipsRegisterInfo.cpp | 15 +- lib/Target/Mips/MipsRegisterInfo.h | 14 +- lib/Target/Mips/MipsSEISelDAGToDAG.cpp | 84 +- lib/Target/Mips/MipsSEISelDAGToDAG.h | 10 + lib/Target/Mips/MipsSEInstrInfo.cpp | 2 +- lib/Target/Mips/MipsSERegisterInfo.cpp | 14 +- lib/Target/Mips/MipsSERegisterInfo.h | 2 +- lib/Target/Mips/MipsSchedule.td | 4 + lib/Target/Mips/MipsTargetMachine.cpp | 22 +- lib/Target/Mips/MipsTargetMachine.h | 4 +- lib/Target/Mips/MipsTargetObjectFile.cpp | 14 +- lib/Target/Mips/MipsTargetObjectFile.h | 4 +- lib/Target/Mips/MipsTargetStreamer.h | 7 +- lib/Target/NVPTX/CMakeLists.txt | 1 - lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp | 2 + .../NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp | 50 +- lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h | 2 + lib/Target/NVPTX/NVPTXAllocaHoisting.cpp | 40 +- lib/Target/NVPTX/NVPTXAllocaHoisting.h | 28 - lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 34 +- lib/Target/NVPTX/NVPTXAsmPrinter.h | 4 +- .../NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp | 7 +- lib/Target/NVPTX/NVPTXGenericToNVVM.cpp | 1 + lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 11 +- lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 2 +- lib/Target/NVPTX/NVPTXISelLowering.cpp | 15 +- lib/Target/NVPTX/NVPTXISelLowering.h | 6 + lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp | 35 +- lib/Target/NVPTX/NVPTXLowerAggrCopies.h | 29 +- lib/Target/NVPTX/NVPTXLowerStructArgs.cpp | 4 +- lib/Target/NVPTX/NVPTXMCExpr.h | 4 +- lib/Target/NVPTX/NVPTXRegisterInfo.cpp | 2 +- lib/Target/NVPTX/NVPTXRegisterInfo.h | 3 +- lib/Target/NVPTX/NVPTXSection.h | 5 +- lib/Target/NVPTX/NVPTXTargetMachine.cpp | 12 +- lib/Target/NVPTX/NVPTXTargetMachine.h | 7 +- lib/Target/NVPTX/NVPTXTargetObjectFile.h | 3 - lib/Target/NVPTX/NVPTXUtilities.cpp | 9 +- lib/Target/NVPTX/NVPTXutil.cpp | 90 - lib/Target/NVPTX/NVPTXutil.h | 25 - lib/Target/NVPTX/NVVMReflect.cpp | 38 +- lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp | 8 +- .../PowerPC/Disassembler/PPCDisassembler.cpp | 6 + lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp | 14 + lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h | 2 + lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp | 4 + .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp | 16 +- lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h | 5 +- .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp | 122 +- lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h | 3 +- lib/Target/PowerPC/PPC.td | 19 +- lib/Target/PowerPC/PPCAsmPrinter.cpp | 23 +- lib/Target/PowerPC/PPCCTRLoops.cpp | 5 +- lib/Target/PowerPC/PPCFastISel.cpp | 16 +- lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 48 +- lib/Target/PowerPC/PPCISelLowering.cpp | 297 ++- lib/Target/PowerPC/PPCISelLowering.h | 26 +- lib/Target/PowerPC/PPCInstr64Bit.td | 28 +- lib/Target/PowerPC/PPCInstrAltivec.td | 94 +- lib/Target/PowerPC/PPCInstrFormats.td | 87 + lib/Target/PowerPC/PPCInstrHTM.td | 172 ++ lib/Target/PowerPC/PPCInstrInfo.cpp | 58 +- lib/Target/PowerPC/PPCInstrInfo.h | 2 +- lib/Target/PowerPC/PPCInstrInfo.td | 82 +- lib/Target/PowerPC/PPCInstrQPX.td | 2 +- lib/Target/PowerPC/PPCInstrVSX.td | 2 +- lib/Target/PowerPC/PPCLoopDataPrefetch.cpp | 6 +- lib/Target/PowerPC/PPCLoopPreIncPrep.cpp | 21 +- lib/Target/PowerPC/PPCMCInstLower.cpp | 3 + lib/Target/PowerPC/PPCRegisterInfo.cpp | 170 +- lib/Target/PowerPC/PPCRegisterInfo.h | 52 +- lib/Target/PowerPC/PPCRegisterInfo.td | 2 + lib/Target/PowerPC/PPCSchedule.td | 398 ---- lib/Target/PowerPC/PPCSubtarget.cpp | 3 + lib/Target/PowerPC/PPCSubtarget.h | 6 + lib/Target/PowerPC/PPCTargetMachine.cpp | 17 +- lib/Target/PowerPC/PPCTargetMachine.h | 6 - lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 4 + lib/Target/PowerPC/PPCTargetTransformInfo.h | 1 + lib/Target/PowerPC/README.txt | 7 + lib/Target/PowerPC/README_ALTIVEC.txt | 104 + lib/Target/R600/AMDGPU.td | 5 + lib/Target/R600/AMDGPUAsmPrinter.cpp | 10 +- lib/Target/R600/AMDGPUISelDAGToDAG.cpp | 127 +- lib/Target/R600/AMDGPUISelLowering.cpp | 3 - lib/Target/R600/AMDGPUInstrInfo.cpp | 30 +- lib/Target/R600/AMDGPUInstrInfo.h | 13 +- lib/Target/R600/AMDGPUInstructions.td | 22 +- lib/Target/R600/AMDGPUIntrinsics.td | 1 + lib/Target/R600/AMDGPUPromoteAlloca.cpp | 13 +- lib/Target/R600/AMDGPURegisterInfo.cpp | 5 +- lib/Target/R600/AMDGPURegisterInfo.h | 3 +- lib/Target/R600/AMDGPUSubtarget.cpp | 2 +- lib/Target/R600/AMDGPUSubtarget.h | 9 + lib/Target/R600/AMDGPUTargetMachine.cpp | 30 +- lib/Target/R600/AMDGPUTargetMachine.h | 10 +- lib/Target/R600/AMDGPUTargetTransformInfo.cpp | 5 +- lib/Target/R600/AMDILCFGStructurizer.cpp | 26 +- lib/Target/R600/AsmParser/AMDGPUAsmParser.cpp | 7 +- lib/Target/R600/EvergreenInstructions.td | 5 +- lib/Target/R600/InstPrinter/AMDGPUInstPrinter.cpp | 16 + lib/Target/R600/InstPrinter/AMDGPUInstPrinter.h | 2 + .../R600/MCTargetDesc/AMDGPUMCTargetDesc.cpp | 58 +- lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.h | 4 +- lib/Target/R600/MCTargetDesc/R600MCCodeEmitter.cpp | 4 +- lib/Target/R600/MCTargetDesc/SIMCCodeEmitter.cpp | 1 - lib/Target/R600/Processors.td | 8 +- lib/Target/R600/R600ClauseMergePass.cpp | 2 +- lib/Target/R600/R600ISelLowering.cpp | 10 +- lib/Target/R600/R600InstrInfo.cpp | 14 +- lib/Target/R600/R600OptimizeVectorRegisters.cpp | 4 +- lib/Target/R600/R600RegisterInfo.cpp | 10 +- lib/Target/R600/R600RegisterInfo.h | 2 +- lib/Target/R600/SIFixSGPRLiveRanges.cpp | 1 + lib/Target/R600/SIFoldOperands.cpp | 3 +- lib/Target/R600/SIISelLowering.cpp | 53 +- lib/Target/R600/SIInsertWaits.cpp | 6 +- lib/Target/R600/SIInstrFormats.td | 11 +- lib/Target/R600/SIInstrInfo.cpp | 112 +- lib/Target/R600/SIInstrInfo.h | 6 +- lib/Target/R600/SIInstrInfo.td | 569 +++--- lib/Target/R600/SIInstructions.td | 486 +++-- lib/Target/R600/SILoadStoreOptimizer.cpp | 5 +- lib/Target/R600/SIRegisterInfo.cpp | 84 +- lib/Target/R600/SIRegisterInfo.h | 9 +- lib/Target/R600/SIRegisterInfo.td | 7 - lib/Target/R600/SIShrinkInstructions.cpp | 8 +- lib/Target/README.txt | 41 +- .../Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp | 1 - lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h | 4 +- .../Sparc/MCTargetDesc/SparcMCTargetDesc.cpp | 96 +- lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h | 1 - lib/Target/Sparc/SparcISelDAGToDAG.cpp | 9 +- lib/Target/Sparc/SparcISelLowering.cpp | 14 +- lib/Target/Sparc/SparcInstrInfo.cpp | 5 +- lib/Target/Sparc/SparcInstrInfo.h | 2 + lib/Target/Sparc/SparcRegisterInfo.cpp | 16 +- lib/Target/Sparc/SparcRegisterInfo.h | 15 +- lib/Target/Sparc/SparcTargetMachine.cpp | 11 +- lib/Target/Sparc/SparcTargetMachine.h | 6 +- .../SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp | 1 - .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp | 12 - .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.h | 1 - lib/Target/SystemZ/SystemZISelDAGToDAG.cpp | 39 +- lib/Target/SystemZ/SystemZISelLowering.cpp | 5 +- lib/Target/SystemZ/SystemZISelLowering.h | 20 + lib/Target/SystemZ/SystemZInstrInfo.cpp | 15 +- lib/Target/SystemZ/SystemZInstrInfo.h | 8 +- lib/Target/SystemZ/SystemZRegisterInfo.cpp | 3 +- lib/Target/SystemZ/SystemZRegisterInfo.h | 6 +- lib/Target/SystemZ/SystemZTargetMachine.cpp | 10 +- lib/Target/SystemZ/SystemZTargetMachine.h | 6 +- lib/Target/Target.cpp | 4 - lib/Target/TargetLoweringObjectFile.cpp | 6 + lib/Target/TargetMachine.cpp | 17 +- lib/Target/TargetMachineC.cpp | 3 +- lib/Target/TargetSubtargetInfo.cpp | 20 +- lib/Target/X86/Android.mk | 2 + lib/Target/X86/AsmParser/X86AsmParser.cpp | 42 + lib/Target/X86/AsmParser/X86Operand.h | 8 +- lib/Target/X86/Disassembler/X86Disassembler.cpp | 42 +- .../X86/Disassembler/X86DisassemblerDecoder.cpp | 9 +- .../Disassembler/X86DisassemblerDecoderCommon.h | 12 - lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 8 +- lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp | 417 ++-- .../X86/MCTargetDesc/X86ELFRelocationInfo.cpp | 2 +- lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp | 2 - lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp | 238 +-- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h | 19 +- .../X86/MCTargetDesc/X86MachORelocationInfo.cpp | 4 +- lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp | 8 +- lib/Target/X86/README-SSE.txt | 78 - lib/Target/X86/X86AsmPrinter.cpp | 3 - lib/Target/X86/X86FastISel.cpp | 152 +- lib/Target/X86/X86FloatingPoint.cpp | 6 +- lib/Target/X86/X86FrameLowering.cpp | 2 +- lib/Target/X86/X86ISelDAGToDAG.cpp | 21 +- lib/Target/X86/X86ISelLowering.cpp | 1258 ++++++++---- lib/Target/X86/X86ISelLowering.h | 147 +- lib/Target/X86/X86InstrAVX512.td | 773 ++++---- lib/Target/X86/X86InstrFragmentsSIMD.td | 108 +- lib/Target/X86/X86InstrInfo.cpp | 55 +- lib/Target/X86/X86InstrInfo.h | 21 +- lib/Target/X86/X86InstrInfo.td | 18 +- lib/Target/X86/X86InstrSSE.td | 163 +- lib/Target/X86/X86IntrinsicsInfo.h | 2 - lib/Target/X86/X86MCInstLower.cpp | 4 +- lib/Target/X86/X86RegisterInfo.cpp | 81 +- lib/Target/X86/X86RegisterInfo.h | 17 +- lib/Target/X86/X86SchedHaswell.td | 4 +- lib/Target/X86/X86SelectionDAGInfo.cpp | 8 +- lib/Target/X86/X86TargetMachine.cpp | 10 +- lib/Target/X86/X86TargetMachine.h | 4 - lib/Target/X86/X86TargetObjectFile.cpp | 42 +- lib/Target/X86/X86TargetObjectFile.h | 23 +- .../XCore/MCTargetDesc/XCoreMCTargetDesc.cpp | 17 +- lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h | 2 + lib/Target/XCore/XCoreISelDAGToDAG.cpp | 8 +- lib/Target/XCore/XCoreISelLowering.h | 6 + lib/Target/XCore/XCoreRegisterInfo.cpp | 4 +- lib/Target/XCore/XCoreRegisterInfo.h | 3 +- lib/Target/XCore/XCoreTargetMachine.cpp | 5 +- lib/Target/XCore/XCoreTargetMachine.h | 7 +- lib/Transforms/IPO/ArgumentPromotion.cpp | 80 +- lib/Transforms/IPO/ConstantMerge.cpp | 25 +- lib/Transforms/IPO/GlobalDCE.cpp | 26 +- lib/Transforms/IPO/GlobalOpt.cpp | 106 +- lib/Transforms/IPO/Inliner.cpp | 40 +- lib/Transforms/IPO/LowerBitSets.cpp | 276 ++- lib/Transforms/IPO/MergeFunctions.cpp | 54 +- lib/Transforms/IPO/PassManagerBuilder.cpp | 48 +- lib/Transforms/InstCombine/InstCombineAddSub.cpp | 88 +- lib/Transforms/InstCombine/InstCombineAndOrXor.cpp | 18 +- lib/Transforms/InstCombine/InstCombineCalls.cpp | 142 +- lib/Transforms/InstCombine/InstCombineCasts.cpp | 148 +- lib/Transforms/InstCombine/InstCombineCompares.cpp | 63 +- lib/Transforms/InstCombine/InstCombineInternal.h | 24 +- .../InstCombine/InstCombineLoadStoreAlloca.cpp | 203 +- .../InstCombine/InstCombineMulDivRem.cpp | 30 +- lib/Transforms/InstCombine/InstCombinePHI.cpp | 8 +- lib/Transforms/InstCombine/InstCombineSelect.cpp | 98 +- lib/Transforms/InstCombine/InstCombineShifts.cpp | 25 +- .../InstCombine/InstCombineSimplifyDemanded.cpp | 148 +- .../InstCombine/InstCombineVectorOps.cpp | 7 +- .../InstCombine/InstructionCombining.cpp | 258 +-- .../Instrumentation/AddressSanitizer.cpp | 818 ++++---- lib/Transforms/Instrumentation/BoundsChecking.cpp | 26 +- .../Instrumentation/DataFlowSanitizer.cpp | 24 +- lib/Transforms/Instrumentation/GCOVProfiling.cpp | 39 +- lib/Transforms/Instrumentation/MemorySanitizer.cpp | 57 +- .../Instrumentation/SanitizerCoverage.cpp | 157 +- lib/Transforms/Instrumentation/ThreadSanitizer.cpp | 62 +- lib/Transforms/ObjCARC/ARCInstKind.cpp | 96 +- lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h | 48 +- lib/Transforms/ObjCARC/Android.mk | 1 + lib/Transforms/ObjCARC/BlotMapVector.h | 108 ++ lib/Transforms/ObjCARC/CMakeLists.txt | 1 + lib/Transforms/ObjCARC/DependencyAnalysis.cpp | 16 +- lib/Transforms/ObjCARC/ObjCARC.h | 55 +- lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.cpp | 11 +- lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.h | 4 +- lib/Transforms/ObjCARC/ObjCARCContract.cpp | 11 +- lib/Transforms/ObjCARC/ObjCARCOpts.cpp | 1308 +++---------- lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp | 28 +- lib/Transforms/ObjCARC/ProvenanceAnalysis.h | 5 +- .../ObjCARC/ProvenanceAnalysisEvaluator.cpp | 4 +- lib/Transforms/ObjCARC/PtrState.cpp | 404 ++++ lib/Transforms/ObjCARC/PtrState.h | 210 ++ lib/Transforms/Scalar/AlignmentFromAssumptions.cpp | 13 +- lib/Transforms/Scalar/Android.mk | 1 + lib/Transforms/Scalar/BDCE.cpp | 31 +- lib/Transforms/Scalar/CMakeLists.txt | 1 + lib/Transforms/Scalar/ConstantHoisting.cpp | 1 + lib/Transforms/Scalar/ConstantProp.cpp | 4 +- .../Scalar/CorrelatedValuePropagation.cpp | 6 +- lib/Transforms/Scalar/DeadStoreElimination.cpp | 60 +- lib/Transforms/Scalar/EarlyCSE.cpp | 18 +- lib/Transforms/Scalar/GVN.cpp | 134 +- lib/Transforms/Scalar/IndVarSimplify.cpp | 85 +- .../Scalar/InductiveRangeCheckElimination.cpp | 401 ++-- lib/Transforms/Scalar/JumpThreading.cpp | 15 +- lib/Transforms/Scalar/LICM.cpp | 79 +- lib/Transforms/Scalar/LoadCombine.cpp | 37 +- lib/Transforms/Scalar/LoopIdiomRecognize.cpp | 37 +- lib/Transforms/Scalar/LoopInstSimplify.cpp | 3 +- lib/Transforms/Scalar/LoopInterchange.cpp | 1154 +++++++++++ lib/Transforms/Scalar/LoopRerollPass.cpp | 30 +- lib/Transforms/Scalar/LoopRotation.cpp | 8 +- lib/Transforms/Scalar/LoopStrengthReduce.cpp | 11 +- lib/Transforms/Scalar/LoopUnrollPass.cpp | 19 +- lib/Transforms/Scalar/LoopUnswitch.cpp | 4 +- lib/Transforms/Scalar/MemCpyOptimizer.cpp | 86 +- lib/Transforms/Scalar/MergedLoadStoreMotion.cpp | 3 +- lib/Transforms/Scalar/Reassociate.cpp | 34 +- lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 314 +-- lib/Transforms/Scalar/SCCP.cpp | 14 +- lib/Transforms/Scalar/SROA.cpp | 89 +- lib/Transforms/Scalar/SampleProfile.cpp | 3 + lib/Transforms/Scalar/Scalar.cpp | 2 +- lib/Transforms/Scalar/ScalarReplAggregates.cpp | 190 +- lib/Transforms/Scalar/Scalarizer.cpp | 26 +- .../Scalar/SeparateConstOffsetFromGEP.cpp | 71 +- lib/Transforms/Scalar/SimplifyCFGPass.cpp | 18 +- lib/Transforms/Scalar/Sink.cpp | 6 +- .../Scalar/StraightLineStrengthReduce.cpp | 395 +++- lib/Transforms/Scalar/StructurizeCFG.cpp | 3 +- lib/Transforms/Scalar/TailRecursionElimination.cpp | 70 +- lib/Transforms/Utils/BuildLibCalls.cpp | 127 +- lib/Transforms/Utils/CloneFunction.cpp | 77 +- lib/Transforms/Utils/CodeExtractor.cpp | 30 +- lib/Transforms/Utils/CtorUtils.cpp | 3 +- lib/Transforms/Utils/InlineFunction.cpp | 132 +- lib/Transforms/Utils/Local.cpp | 22 +- lib/Transforms/Utils/LoopSimplify.cpp | 23 +- lib/Transforms/Utils/LoopUnroll.cpp | 9 +- lib/Transforms/Utils/LoopUnrollRuntime.cpp | 4 +- lib/Transforms/Utils/LowerSwitch.cpp | 11 +- lib/Transforms/Utils/PromoteMemoryToRegister.cpp | 5 +- lib/Transforms/Utils/SSAUpdater.cpp | 4 +- lib/Transforms/Utils/SimplifyCFG.cpp | 353 ++-- lib/Transforms/Utils/SimplifyIndVar.cpp | 118 +- lib/Transforms/Utils/SimplifyInstructions.cpp | 3 +- lib/Transforms/Utils/SimplifyLibCalls.cpp | 264 +-- lib/Transforms/Utils/SymbolRewriter.cpp | 13 +- lib/Transforms/Utils/ValueMapper.cpp | 6 +- lib/Transforms/Vectorize/BBVectorize.cpp | 35 +- lib/Transforms/Vectorize/LoopVectorize.cpp | 412 ++-- lib/Transforms/Vectorize/SLPVectorizer.cpp | 114 +- 823 files changed, 29271 insertions(+), 24503 deletions(-) delete mode 100644 lib/Analysis/JumpInstrTableInfo.cpp delete mode 100644 lib/Bitcode/Reader/BitcodeReader.h delete mode 100644 lib/CodeGen/ForwardControlFlowIntegrity.cpp delete mode 100644 lib/CodeGen/JumpInstrTables.cpp delete mode 100644 lib/CodeGen/PrologEpilogInserter.h create mode 100644 lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp create mode 100644 lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h create mode 100644 lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h create mode 100644 lib/Fuzzer/test/CounterTest.cpp delete mode 100644 lib/IR/AsmWriter.h create mode 100644 lib/Passes/Android.mk create mode 100644 lib/Passes/CMakeLists.txt create mode 100644 lib/Passes/LLVMBuild.txt create mode 100644 lib/Passes/Makefile create mode 100644 lib/Passes/PassBuilder.cpp create mode 100644 lib/Passes/PassRegistry.def delete mode 100644 lib/Support/IsInf.cpp delete mode 100644 lib/Support/IsNAN.cpp delete mode 100644 lib/Target/ARM/Thumb1RegisterInfo.cpp delete mode 100644 lib/Target/ARM/Thumb1RegisterInfo.h delete mode 100644 lib/Target/ARM/Thumb2RegisterInfo.cpp delete mode 100644 lib/Target/ARM/Thumb2RegisterInfo.h create mode 100644 lib/Target/ARM/ThumbRegisterInfo.cpp create mode 100644 lib/Target/ARM/ThumbRegisterInfo.h delete mode 100644 lib/Target/Hexagon/HexagonSplitTFRCondSets.cpp delete mode 100644 lib/Target/Mips/Mips16HardFloat.h delete mode 100644 lib/Target/Mips/MipsModuleISelDAGToDAG.h delete mode 100644 lib/Target/Mips/MipsOs16.h delete mode 100644 lib/Target/NVPTX/NVPTXutil.cpp delete mode 100644 lib/Target/NVPTX/NVPTXutil.h create mode 100644 lib/Target/PowerPC/PPCInstrHTM.td create mode 100644 lib/Transforms/ObjCARC/BlotMapVector.h create mode 100644 lib/Transforms/ObjCARC/PtrState.cpp create mode 100644 lib/Transforms/ObjCARC/PtrState.h create mode 100644 lib/Transforms/Scalar/LoopInterchange.cpp (limited to 'lib') diff --git a/lib/Analysis/AliasAnalysis.cpp b/lib/Analysis/AliasAnalysis.cpp index 4e95aa0..0b0fd50 100644 --- a/lib/Analysis/AliasAnalysis.cpp +++ b/lib/Analysis/AliasAnalysis.cpp @@ -407,9 +407,10 @@ AliasAnalysis::ModRefResult AliasAnalysis::callCapturesBefore(const Instruction *I, const AliasAnalysis::Location &MemLoc, DominatorTree *DT) { - if (!DT || !DL) return AliasAnalysis::ModRef; + if (!DT) + return AliasAnalysis::ModRef; - const Value *Object = GetUnderlyingObject(MemLoc.Ptr, DL); + const Value *Object = GetUnderlyingObject(MemLoc.Ptr, *DL); if (!isIdentifiedObject(Object) || isa(Object) || isa(Object)) return AliasAnalysis::ModRef; @@ -462,9 +463,8 @@ AliasAnalysis::~AliasAnalysis() {} /// InitializeAliasAnalysis - Subclasses must call this method to initialize the /// AliasAnalysis interface before any other methods are called. /// -void AliasAnalysis::InitializeAliasAnalysis(Pass *P) { - DataLayoutPass *DLP = P->getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; +void AliasAnalysis::InitializeAliasAnalysis(Pass *P, const DataLayout *NewDL) { + DL = NewDL; auto *TLIP = P->getAnalysisIfAvailable(); TLI = TLIP ? &TLIP->getTLI() : nullptr; AA = &P->getAnalysis(); diff --git a/lib/Analysis/AliasAnalysisCounter.cpp b/lib/Analysis/AliasAnalysisCounter.cpp index b860914..5865259 100644 --- a/lib/Analysis/AliasAnalysisCounter.cpp +++ b/lib/Analysis/AliasAnalysisCounter.cpp @@ -14,6 +14,7 @@ #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -76,7 +77,7 @@ namespace { bool runOnModule(Module &M) override { this->M = &M; - InitializeAliasAnalysis(this); + InitializeAliasAnalysis(this, &M.getDataLayout()); return false; } diff --git a/lib/Analysis/AliasDebugger.cpp b/lib/Analysis/AliasDebugger.cpp index 5d61cf9..f98b578 100644 --- a/lib/Analysis/AliasDebugger.cpp +++ b/lib/Analysis/AliasDebugger.cpp @@ -44,7 +44,7 @@ namespace { } bool runOnModule(Module &M) override { - InitializeAliasAnalysis(this); // set up super class + InitializeAliasAnalysis(this, &M.getDataLayout()); // set up super class for(Module::global_iterator I = M.global_begin(), E = M.global_end(); I != E; ++I) { diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp index 1bfb06d..4549c1e 100644 --- a/lib/Analysis/Analysis.cpp +++ b/lib/Analysis/Analysis.cpp @@ -49,7 +49,6 @@ void llvm::initializeAnalysis(PassRegistry &Registry) { initializeIVUsersPass(Registry); initializeInstCountPass(Registry); initializeIntervalPartitionPass(Registry); - initializeJumpInstrTableInfoPass(Registry); initializeLazyValueInfoPass(Registry); initializeLibCallAliasAnalysisPass(Registry); initializeLintPass(Registry); diff --git a/lib/Analysis/Android.mk b/lib/Analysis/Android.mk index e17b870..277956c 100644 --- a/lib/Analysis/Android.mk +++ b/lib/Analysis/Android.mk @@ -29,7 +29,6 @@ analysis_SRC_FILES := \ InstructionSimplify.cpp \ Interval.cpp \ IntervalPartition.cpp \ - JumpInstrTableInfo.cpp \ LazyCallGraph.cpp \ LazyValueInfo.cpp \ LibCallAliasAnalysis.cpp \ diff --git a/lib/Analysis/BasicAliasAnalysis.cpp b/lib/Analysis/BasicAliasAnalysis.cpp index 46ca6ee..be2282f 100644 --- a/lib/Analysis/BasicAliasAnalysis.cpp +++ b/lib/Analysis/BasicAliasAnalysis.cpp @@ -103,7 +103,7 @@ static uint64_t getObjectSize(const Value *V, const DataLayout &DL, const TargetLibraryInfo &TLI, bool RoundToAlign = false) { uint64_t Size; - if (getObjectSize(V, Size, &DL, &TLI, RoundToAlign)) + if (getObjectSize(V, Size, DL, &TLI, RoundToAlign)) return Size; return AliasAnalysis::UnknownSize; } @@ -221,7 +221,7 @@ static Value *GetLinearExpression(Value *V, APInt &Scale, APInt &Offset, case Instruction::Or: // X|C == X+C if all the bits in C are unset in X. Otherwise we can't // analyze it. - if (!MaskedValueIsZero(BOp->getOperand(0), RHSC->getValue(), &DL, 0, AC, + if (!MaskedValueIsZero(BOp->getOperand(0), RHSC->getValue(), DL, 0, AC, BOp, DT)) break; // FALL THROUGH. @@ -292,7 +292,7 @@ static Value *GetLinearExpression(Value *V, APInt &Scale, APInt &Offset, static const Value * DecomposeGEPExpression(const Value *V, int64_t &BaseOffs, SmallVectorImpl &VarIndices, - bool &MaxLookupReached, const DataLayout *DL, + bool &MaxLookupReached, const DataLayout &DL, AssumptionCache *AC, DominatorTree *DT) { // Limit recursion depth to limit compile time in crazy cases. unsigned MaxLookup = MaxLookupSearchDepth; @@ -341,16 +341,6 @@ DecomposeGEPExpression(const Value *V, int64_t &BaseOffs, if (!GEPOp->getOperand(0)->getType()->getPointerElementType()->isSized()) return V; - // If we are lacking DataLayout information, we can't compute the offets of - // elements computed by GEPs. However, we can handle bitcast equivalent - // GEPs. - if (!DL) { - if (!GEPOp->hasAllZeroIndices()) - return V; - V = GEPOp->getOperand(0); - continue; - } - unsigned AS = GEPOp->getPointerAddressSpace(); // Walk the indices of the GEP, accumulating them into BaseOff/VarIndices. gep_type_iterator GTI = gep_type_begin(GEPOp); @@ -363,30 +353,30 @@ DecomposeGEPExpression(const Value *V, int64_t &BaseOffs, unsigned FieldNo = cast(Index)->getZExtValue(); if (FieldNo == 0) continue; - BaseOffs += DL->getStructLayout(STy)->getElementOffset(FieldNo); + BaseOffs += DL.getStructLayout(STy)->getElementOffset(FieldNo); continue; } // For an array/pointer, add the element offset, explicitly scaled. if (ConstantInt *CIdx = dyn_cast(Index)) { if (CIdx->isZero()) continue; - BaseOffs += DL->getTypeAllocSize(*GTI)*CIdx->getSExtValue(); + BaseOffs += DL.getTypeAllocSize(*GTI) * CIdx->getSExtValue(); continue; } - uint64_t Scale = DL->getTypeAllocSize(*GTI); + uint64_t Scale = DL.getTypeAllocSize(*GTI); ExtensionKind Extension = EK_NotExtended; // If the integer type is smaller than the pointer size, it is implicitly // sign extended to pointer size. unsigned Width = Index->getType()->getIntegerBitWidth(); - if (DL->getPointerSizeInBits(AS) > Width) + if (DL.getPointerSizeInBits(AS) > Width) Extension = EK_SignExt; // Use GetLinearExpression to decompose the index into a C1*V+C2 form. APInt IndexScale(Width, 0), IndexOffset(Width, 0); - Index = GetLinearExpression(Index, IndexScale, IndexOffset, Extension, - *DL, 0, AC, DT); + Index = GetLinearExpression(Index, IndexScale, IndexOffset, Extension, DL, + 0, AC, DT); // The GEP index scale ("Scale") scales C1*V+C2, yielding (C1*V+C2)*Scale. // This gives us an aggregate computation of (C1*Scale)*V + C2*Scale. @@ -408,7 +398,7 @@ DecomposeGEPExpression(const Value *V, int64_t &BaseOffs, // Make sure that we have a scale that makes sense for this target's // pointer size. - if (unsigned ShiftBits = 64 - DL->getPointerSizeInBits(AS)) { + if (unsigned ShiftBits = 64 - DL.getPointerSizeInBits(AS)) { Scale <<= ShiftBits; Scale = (int64_t)Scale >> ShiftBits; } @@ -461,9 +451,7 @@ namespace { initializeBasicAliasAnalysisPass(*PassRegistry::getPassRegistry()); } - void initializePass() override { - InitializeAliasAnalysis(this); - } + bool doInitialization(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); @@ -612,7 +600,7 @@ BasicAliasAnalysis::pointsToConstantMemory(const Location &Loc, bool OrLocal) { SmallVector Worklist; Worklist.push_back(Loc.Ptr); do { - const Value *V = GetUnderlyingObject(Worklist.pop_back_val(), DL); + const Value *V = GetUnderlyingObject(Worklist.pop_back_val(), *DL); if (!Visited.insert(V).second) { Visited.clear(); return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal); @@ -815,6 +803,11 @@ static bool isAssumeIntrinsic(ImmutableCallSite CS) { return false; } +bool BasicAliasAnalysis::doInitialization(Module &M) { + InitializeAliasAnalysis(this, &M.getDataLayout()); + return true; +} + /// getModRefInfo - Check to see if the specified callsite can clobber the /// specified memory object. Since we only look at local properties of this /// function, we really can't say much about this query. We do, however, use @@ -825,7 +818,7 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS, assert(notDifferentParent(CS.getInstruction(), Loc.Ptr) && "AliasAnalysis query involving multiple functions!"); - const Value *Object = GetUnderlyingObject(Loc.Ptr, DL); + const Value *Object = GetUnderlyingObject(Loc.Ptr, *DL); // If this is a tail call and Loc.Ptr points to a stack location, we know that // the tail call cannot access or modify the local stack. @@ -1042,10 +1035,10 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size, SmallVector GEP2VariableIndices; const Value *GEP2BasePtr = DecomposeGEPExpression(GEP2, GEP2BaseOffset, GEP2VariableIndices, - GEP2MaxLookupReached, DL, AC2, DT); + GEP2MaxLookupReached, *DL, AC2, DT); const Value *GEP1BasePtr = DecomposeGEPExpression(GEP1, GEP1BaseOffset, GEP1VariableIndices, - GEP1MaxLookupReached, DL, AC1, DT); + GEP1MaxLookupReached, *DL, AC1, DT); // DecomposeGEPExpression and GetUnderlyingObject should return the // same result except when DecomposeGEPExpression has no DataLayout. if (GEP1BasePtr != UnderlyingV1 || GEP2BasePtr != UnderlyingV2) { @@ -1074,14 +1067,14 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size, // about the relation of the resulting pointer. const Value *GEP1BasePtr = DecomposeGEPExpression(GEP1, GEP1BaseOffset, GEP1VariableIndices, - GEP1MaxLookupReached, DL, AC1, DT); + GEP1MaxLookupReached, *DL, AC1, DT); int64_t GEP2BaseOffset; bool GEP2MaxLookupReached; SmallVector GEP2VariableIndices; const Value *GEP2BasePtr = DecomposeGEPExpression(GEP2, GEP2BaseOffset, GEP2VariableIndices, - GEP2MaxLookupReached, DL, AC2, DT); + GEP2MaxLookupReached, *DL, AC2, DT); // DecomposeGEPExpression and GetUnderlyingObject should return the // same result except when DecomposeGEPExpression has no DataLayout. @@ -1131,7 +1124,7 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size, const Value *GEP1BasePtr = DecomposeGEPExpression(GEP1, GEP1BaseOffset, GEP1VariableIndices, - GEP1MaxLookupReached, DL, AC1, DT); + GEP1MaxLookupReached, *DL, AC1, DT); // DecomposeGEPExpression and GetUnderlyingObject should return the // same result except when DecomposeGEPExpression has no DataLayout. @@ -1200,7 +1193,7 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size, const Value *V = GEP1VariableIndices[i].V; bool SignKnownZero, SignKnownOne; - ComputeSignBit(const_cast(V), SignKnownZero, SignKnownOne, DL, + ComputeSignBit(const_cast(V), SignKnownZero, SignKnownOne, *DL, 0, AC1, nullptr, DT); // Zero-extension widens the variable, and so forces the sign @@ -1409,8 +1402,8 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size, return NoAlias; // Scalars cannot alias each other // Figure out what objects these things are pointing to if we can. - const Value *O1 = GetUnderlyingObject(V1, DL, MaxLookupSearchDepth); - const Value *O2 = GetUnderlyingObject(V2, DL, MaxLookupSearchDepth); + const Value *O1 = GetUnderlyingObject(V1, *DL, MaxLookupSearchDepth); + const Value *O2 = GetUnderlyingObject(V2, *DL, MaxLookupSearchDepth); // Null values in the default address space don't point to any object, so they // don't alias any other pointer. @@ -1533,6 +1526,9 @@ bool BasicAliasAnalysis::isValueEqualInPotentialCycles(const Value *V, if (!Inst) return true; + if (VisitedPhiBBs.empty()) + return true; + if (VisitedPhiBBs.size() > MaxNumPhiBBsValueReachabilityCheck) return false; diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp index 8cd6ea4..14800f4 100644 --- a/lib/Analysis/BranchProbabilityInfo.cpp +++ b/lib/Analysis/BranchProbabilityInfo.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; diff --git a/lib/Analysis/CFLAliasAnalysis.cpp b/lib/Analysis/CFLAliasAnalysis.cpp index 82fbfe0..53d748d 100644 --- a/lib/Analysis/CFLAliasAnalysis.cpp +++ b/lib/Analysis/CFLAliasAnalysis.cpp @@ -45,9 +45,11 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include #include #include +#include #include using namespace llvm; @@ -77,7 +79,7 @@ static Optional getTargetValue(Instruction *); static bool hasUsefulEdges(Instruction *); const StratifiedIndex StratifiedLink::SetSentinel = - std::numeric_limits::max(); + std::numeric_limits::max(); namespace { // StratifiedInfo Attribute things. @@ -85,11 +87,13 @@ typedef unsigned StratifiedAttr; LLVM_CONSTEXPR unsigned MaxStratifiedAttrIndex = NumStratifiedAttrs; LLVM_CONSTEXPR unsigned AttrAllIndex = 0; LLVM_CONSTEXPR unsigned AttrGlobalIndex = 1; -LLVM_CONSTEXPR unsigned AttrFirstArgIndex = 2; +LLVM_CONSTEXPR unsigned AttrUnknownIndex = 2; +LLVM_CONSTEXPR unsigned AttrFirstArgIndex = 3; LLVM_CONSTEXPR unsigned AttrLastArgIndex = MaxStratifiedAttrIndex; LLVM_CONSTEXPR unsigned AttrMaxNumArgs = AttrLastArgIndex - AttrFirstArgIndex; LLVM_CONSTEXPR StratifiedAttr AttrNone = 0; +LLVM_CONSTEXPR StratifiedAttr AttrUnknown = 1 << AttrUnknownIndex; LLVM_CONSTEXPR StratifiedAttr AttrAll = ~AttrNone; // \brief StratifiedSets call for knowledge of "direction", so this is how we @@ -144,9 +148,8 @@ struct FunctionInfo { // Lots of functions have < 4 returns. Adjust as necessary. SmallVector ReturnedValues; - FunctionInfo(StratifiedSets &&S, - SmallVector &&RV) - : Sets(std::move(S)), ReturnedValues(std::move(RV)) {} + FunctionInfo(StratifiedSets &&S, SmallVector &&RV) + : Sets(std::move(S)), ReturnedValues(std::move(RV)) {} }; struct CFLAliasAnalysis; @@ -229,6 +232,10 @@ public: // Comparisons between global variables and other constants should be // handled by BasicAA. + // TODO: ConstantExpr handling -- CFLAA may report NoAlias when comparing + // a GlobalValue and ConstantExpr, but every query needs to have at least + // one Value tied to a Function, and neither GlobalValues nor ConstantExprs + // are. if (isa(LocA.Ptr) && isa(LocB.Ptr)) { return AliasAnalysis::alias(LocA, LocB); } @@ -240,7 +247,7 @@ public: return QueryResult; } - void initializePass() override { InitializeAliasAnalysis(this); } + bool doInitialization(Module &M) override; }; void FunctionHandle::removeSelfFromCache() { @@ -263,9 +270,19 @@ public: llvm_unreachable("Unsupported instruction encountered"); } + void visitPtrToIntInst(PtrToIntInst &Inst) { + auto *Ptr = Inst.getOperand(0); + Output.push_back(Edge(Ptr, Ptr, EdgeType::Assign, AttrUnknown)); + } + + void visitIntToPtrInst(IntToPtrInst &Inst) { + auto *Ptr = &Inst; + Output.push_back(Edge(Ptr, Ptr, EdgeType::Assign, AttrUnknown)); + } + void visitCastInst(CastInst &Inst) { - Output.push_back(Edge(&Inst, Inst.getOperand(0), EdgeType::Assign, - AttrNone)); + Output.push_back( + Edge(&Inst, Inst.getOperand(0), EdgeType::Assign, AttrNone)); } void visitBinaryOperator(BinaryOperator &Inst) { @@ -377,7 +394,7 @@ public: // I put this here to give us an upper bound on time taken by IPA. Is it // really (realistically) needed? Keep in mind that we do have an n^2 algo. - if (std::distance(Args.begin(), Args.end()) > (int) MaxSupportedArgs) + if (std::distance(Args.begin(), Args.end()) > (int)MaxSupportedArgs) return false; // Exit early if we'll fail anyway @@ -429,7 +446,7 @@ public: } if (AddEdge) Output.push_back(Edge(FuncValue, ArgVal, EdgeType::Assign, - StratifiedAttrs().flip())); + StratifiedAttrs().flip())); } if (Parameters.size() != Arguments.size()) @@ -571,8 +588,7 @@ private: EdgeTypeT Weight; Node Other; - Edge(const EdgeTypeT &W, const Node &N) - : Weight(W), Other(N) {} + Edge(const EdgeTypeT &W, const Node &N) : Weight(W), Other(N) {} bool operator==(const Edge &E) const { return Weight == E.Weight && Other == E.Other; @@ -735,6 +751,25 @@ static Level directionOfEdgeType(EdgeType); static void buildGraphFrom(CFLAliasAnalysis &, Function *, SmallVectorImpl &, NodeMapT &, GraphT &); +// Gets the edges of a ConstantExpr as if it was an Instruction. This +// function also acts on any nested ConstantExprs, adding the edges +// of those to the given SmallVector as well. +static void constexprToEdges(CFLAliasAnalysis &, ConstantExpr &, + SmallVectorImpl &); + +// Given an Instruction, this will add it to the graph, along with any +// Instructions that are potentially only available from said Instruction +// For example, given the following line: +// %0 = load i16* getelementptr ([1 x i16]* @a, 0, 0), align 2 +// addInstructionToGraph would add both the `load` and `getelementptr` +// instructions to the graph appropriately. +static void addInstructionToGraph(CFLAliasAnalysis &, Instruction &, + SmallVectorImpl &, NodeMapT &, + GraphT &); + +// Notes whether it would be pointless to add the given Value to our sets. +static bool canSkipAddingToSets(Value *Val); + // Builds the graph + StratifiedSets for a function. static FunctionInfo buildSetsFrom(CFLAliasAnalysis &, Function *); @@ -806,6 +841,8 @@ static EdgeType flipWeight(EdgeType Initial) { static void argsToEdges(CFLAliasAnalysis &Analysis, Instruction *Inst, SmallVectorImpl &Output) { + assert(hasUsefulEdges(Inst) && + "Expected instructions to have 'useful' edges"); GetEdgesVisitor v(Analysis, Output); v.visit(Inst); } @@ -822,13 +859,41 @@ static Level directionOfEdgeType(EdgeType Weight) { llvm_unreachable("Incomplete switch coverage"); } -// Aside: We may remove graph construction entirely, because it doesn't really -// buy us much that we don't already have. I'd like to add interprocedural -// analysis prior to this however, in case that somehow requires the graph -// produced by this for efficient execution -static void buildGraphFrom(CFLAliasAnalysis &Analysis, Function *Fn, - SmallVectorImpl &ReturnedValues, - NodeMapT &Map, GraphT &Graph) { +static void constexprToEdges(CFLAliasAnalysis &Analysis, + ConstantExpr &CExprToCollapse, + SmallVectorImpl &Results) { + SmallVector Worklist; + Worklist.push_back(&CExprToCollapse); + + SmallVector ConstexprEdges; + while (!Worklist.empty()) { + auto *CExpr = Worklist.pop_back_val(); + std::unique_ptr Inst(CExpr->getAsInstruction()); + + if (!hasUsefulEdges(Inst.get())) + continue; + + ConstexprEdges.clear(); + argsToEdges(Analysis, Inst.get(), ConstexprEdges); + for (auto &Edge : ConstexprEdges) { + if (Edge.From == Inst.get()) + Edge.From = CExpr; + else if (auto *Nested = dyn_cast(Edge.From)) + Worklist.push_back(Nested); + + if (Edge.To == Inst.get()) + Edge.To = CExpr; + else if (auto *Nested = dyn_cast(Edge.To)) + Worklist.push_back(Nested); + } + + Results.append(ConstexprEdges.begin(), ConstexprEdges.end()); + } +} + +static void addInstructionToGraph(CFLAliasAnalysis &Analysis, Instruction &Inst, + SmallVectorImpl &ReturnedValues, + NodeMapT &Map, GraphT &Graph) { const auto findOrInsertNode = [&Map, &Graph](Value *Val) { auto Pair = Map.insert(std::make_pair(Val, GraphT::Node())); auto &Iter = Pair.first; @@ -839,42 +904,86 @@ static void buildGraphFrom(CFLAliasAnalysis &Analysis, Function *Fn, return Iter->second; }; + // We don't want the edges of most "return" instructions, but we *do* want + // to know what can be returned. + if (isa(&Inst)) + ReturnedValues.push_back(&Inst); + + if (!hasUsefulEdges(&Inst)) + return; + SmallVector Edges; - for (auto &Bb : Fn->getBasicBlockList()) { - for (auto &Inst : Bb.getInstList()) { - // We don't want the edges of most "return" instructions, but we *do* want - // to know what can be returned. - if (auto *Ret = dyn_cast(&Inst)) - ReturnedValues.push_back(Ret); - - if (!hasUsefulEdges(&Inst)) - continue; + argsToEdges(Analysis, &Inst, Edges); + + // In the case of an unused alloca (or similar), edges may be empty. Note + // that it exists so we can potentially answer NoAlias. + if (Edges.empty()) { + auto MaybeVal = getTargetValue(&Inst); + assert(MaybeVal.hasValue()); + auto *Target = *MaybeVal; + findOrInsertNode(Target); + return; + } - Edges.clear(); - argsToEdges(Analysis, &Inst, Edges); + const auto addEdgeToGraph = [&Graph, &findOrInsertNode](const Edge &E) { + auto To = findOrInsertNode(E.To); + auto From = findOrInsertNode(E.From); + auto FlippedWeight = flipWeight(E.Weight); + auto Attrs = E.AdditionalAttrs; + Graph.addEdge(From, To, std::make_pair(E.Weight, Attrs), + std::make_pair(FlippedWeight, Attrs)); + }; - // In the case of an unused alloca (or similar), edges may be empty. Note - // that it exists so we can potentially answer NoAlias. - if (Edges.empty()) { - auto MaybeVal = getTargetValue(&Inst); - assert(MaybeVal.hasValue()); - auto *Target = *MaybeVal; - findOrInsertNode(Target); - continue; - } + SmallVector ConstantExprs; + for (const Edge &E : Edges) { + addEdgeToGraph(E); + if (auto *Constexpr = dyn_cast(E.To)) + ConstantExprs.push_back(Constexpr); + if (auto *Constexpr = dyn_cast(E.From)) + ConstantExprs.push_back(Constexpr); + } - for (const Edge &E : Edges) { - auto To = findOrInsertNode(E.To); - auto From = findOrInsertNode(E.From); - auto FlippedWeight = flipWeight(E.Weight); - auto Attrs = E.AdditionalAttrs; - Graph.addEdge(From, To, std::make_pair(E.Weight, Attrs), - std::make_pair(FlippedWeight, Attrs)); - } - } + for (ConstantExpr *CE : ConstantExprs) { + Edges.clear(); + constexprToEdges(Analysis, *CE, Edges); + std::for_each(Edges.begin(), Edges.end(), addEdgeToGraph); } } +// Aside: We may remove graph construction entirely, because it doesn't really +// buy us much that we don't already have. I'd like to add interprocedural +// analysis prior to this however, in case that somehow requires the graph +// produced by this for efficient execution +static void buildGraphFrom(CFLAliasAnalysis &Analysis, Function *Fn, + SmallVectorImpl &ReturnedValues, + NodeMapT &Map, GraphT &Graph) { + for (auto &Bb : Fn->getBasicBlockList()) + for (auto &Inst : Bb.getInstList()) + addInstructionToGraph(Analysis, Inst, ReturnedValues, Map, Graph); +} + +static bool canSkipAddingToSets(Value *Val) { + // Constants can share instances, which may falsely unify multiple + // sets, e.g. in + // store i32* null, i32** %ptr1 + // store i32* null, i32** %ptr2 + // clearly ptr1 and ptr2 should not be unified into the same set, so + // we should filter out the (potentially shared) instance to + // i32* null. + if (isa(Val)) { + bool Container = isa(Val) || isa(Val) || + isa(Val); + // TODO: Because all of these things are constant, we can determine whether + // the data is *actually* mutable at graph building time. This will probably + // come for free/cheap with offset awareness. + bool CanStoreMutableData = + isa(Val) || isa(Val) || Container; + return !CanStoreMutableData; + } + + return false; +} + static FunctionInfo buildSetsFrom(CFLAliasAnalysis &Analysis, Function *Fn) { NodeMapT Map; GraphT Graph; @@ -906,7 +1015,7 @@ static FunctionInfo buildSetsFrom(CFLAliasAnalysis &Analysis, Function *Fn) { while (!Worklist.empty()) { auto Node = Worklist.pop_back_val(); auto *CurValue = findValueOrDie(Node); - if (isa(CurValue) && !isa(CurValue)) + if (canSkipAddingToSets(CurValue)) continue; for (const auto &EdgeTuple : Graph.edgesFor(Node)) { @@ -915,7 +1024,7 @@ static FunctionInfo buildSetsFrom(CFLAliasAnalysis &Analysis, Function *Fn) { auto &OtherNode = std::get<1>(EdgeTuple); auto *OtherValue = findValueOrDie(OtherNode); - if (isa(OtherValue) && !isa(OtherValue)) + if (canSkipAddingToSets(OtherValue)) continue; bool Added; @@ -931,16 +1040,16 @@ static FunctionInfo buildSetsFrom(CFLAliasAnalysis &Analysis, Function *Fn) { break; } - if (Added) { - auto Aliasing = Weight.second; - if (auto MaybeCurIndex = valueToAttrIndex(CurValue)) - Aliasing.set(*MaybeCurIndex); - if (auto MaybeOtherIndex = valueToAttrIndex(OtherValue)) - Aliasing.set(*MaybeOtherIndex); - Builder.noteAttributes(CurValue, Aliasing); - Builder.noteAttributes(OtherValue, Aliasing); + auto Aliasing = Weight.second; + if (auto MaybeCurIndex = valueToAttrIndex(CurValue)) + Aliasing.set(*MaybeCurIndex); + if (auto MaybeOtherIndex = valueToAttrIndex(OtherValue)) + Aliasing.set(*MaybeOtherIndex); + Builder.noteAttributes(CurValue, Aliasing); + Builder.noteAttributes(OtherValue, Aliasing); + + if (Added) Worklist.push_back(OtherNode); - } } } } @@ -950,7 +1059,12 @@ static FunctionInfo buildSetsFrom(CFLAliasAnalysis &Analysis, Function *Fn) { // things that were present during construction being present in the graph. // So, we add all present arguments here. for (auto &Arg : Fn->args()) { - Builder.add(&Arg); + if (!Builder.add(&Arg)) + continue; + + auto Attrs = valueToAttrIndex(&Arg); + if (Attrs.hasValue()) + Builder.noteAttributes(&Arg, *Attrs); } return FunctionInfo(Builder.build(), std::move(ReturnedValues)); @@ -1034,3 +1148,8 @@ CFLAliasAnalysis::query(const AliasAnalysis::Location &LocA, return AliasAnalysis::NoAlias; } + +bool CFLAliasAnalysis::doInitialization(Module &M) { + InitializeAliasAnalysis(this, &M.getDataLayout()); + return true; +} diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt index d840037..ae40321 100644 --- a/lib/Analysis/CMakeLists.txt +++ b/lib/Analysis/CMakeLists.txt @@ -27,7 +27,6 @@ add_llvm_library(LLVMAnalysis InstructionSimplify.cpp Interval.cpp IntervalPartition.cpp - JumpInstrTableInfo.cpp LazyCallGraph.cpp LazyValueInfo.cpp LibCallAliasAnalysis.cpp diff --git a/lib/Analysis/CodeMetrics.cpp b/lib/Analysis/CodeMetrics.cpp index fa5683c..46a2c43 100644 --- a/lib/Analysis/CodeMetrics.cpp +++ b/lib/Analysis/CodeMetrics.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "code-metrics" diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp index fcafb41..995465d 100644 --- a/lib/Analysis/ConstantFolding.cpp +++ b/lib/Analysis/ConstantFolding.cpp @@ -50,8 +50,7 @@ using namespace llvm; /// Constant fold bitcast, symbolically evaluating it with DataLayout. /// This always returns a non-null constant, but it may be a /// ConstantExpr if unfoldable. -static Constant *FoldBitCast(Constant *C, Type *DestTy, - const DataLayout &TD) { +static Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) { // Catch the obvious splat cases. if (C->isNullValue() && !DestTy->isX86_MMXTy()) return Constant::getNullValue(DestTy); @@ -84,11 +83,11 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy, // Now that we know that the input value is a vector of integers, just shift // and insert them into our result. - unsigned BitShift = TD.getTypeAllocSizeInBits(SrcEltTy); + unsigned BitShift = DL.getTypeAllocSizeInBits(SrcEltTy); APInt Result(IT->getBitWidth(), 0); for (unsigned i = 0; i != NumSrcElts; ++i) { Result <<= BitShift; - if (TD.isLittleEndian()) + if (DL.isLittleEndian()) Result |= CDV->getElementAsInteger(NumSrcElts-i-1); else Result |= CDV->getElementAsInteger(i); @@ -106,7 +105,7 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy, // vector so the code below can handle it uniformly. if (isa(C) || isa(C)) { Constant *Ops = C; // don't take the address of C! - return FoldBitCast(ConstantVector::get(Ops), DestTy, TD); + return FoldBitCast(ConstantVector::get(Ops), DestTy, DL); } // If this is a bitcast from constant vector -> vector, fold it. @@ -138,7 +137,7 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy, Type *DestIVTy = VectorType::get(IntegerType::get(C->getContext(), FPWidth), NumDstElt); // Recursively handle this integer conversion, if possible. - C = FoldBitCast(C, DestIVTy, TD); + C = FoldBitCast(C, DestIVTy, DL); // Finally, IR can handle this now that #elts line up. return ConstantExpr::getBitCast(C, DestTy); @@ -162,7 +161,7 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy, // of the same size, and that their #elements is not the same. Do the // conversion here, which depends on whether the input or output has // more elements. - bool isLittleEndian = TD.isLittleEndian(); + bool isLittleEndian = DL.isLittleEndian(); SmallVector Result; if (NumDstElt < NumSrcElt) { @@ -198,7 +197,7 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy, // Handle: bitcast (<2 x i64> to <4 x i32>) unsigned Ratio = NumDstElt/NumSrcElt; - unsigned DstBitSize = TD.getTypeSizeInBits(DstEltTy); + unsigned DstBitSize = DL.getTypeSizeInBits(DstEltTy); // Loop over each source value, expanding into multiple results. for (unsigned i = 0; i != NumSrcElt; ++i) { @@ -235,10 +234,10 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy, /// If this constant is a constant offset from a global, return the global and /// the constant. Because of constantexprs, this function is recursive. static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, - APInt &Offset, const DataLayout &TD) { + APInt &Offset, const DataLayout &DL) { // Trivial case, constant is the global. if ((GV = dyn_cast(C))) { - unsigned BitWidth = TD.getPointerTypeSizeInBits(GV->getType()); + unsigned BitWidth = DL.getPointerTypeSizeInBits(GV->getType()); Offset = APInt(BitWidth, 0); return true; } @@ -251,22 +250,22 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, if (CE->getOpcode() == Instruction::PtrToInt || CE->getOpcode() == Instruction::BitCast || CE->getOpcode() == Instruction::AddrSpaceCast) - return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, TD); + return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL); // i32* getelementptr ([5 x i32]* @a, i32 0, i32 5) GEPOperator *GEP = dyn_cast(CE); if (!GEP) return false; - unsigned BitWidth = TD.getPointerTypeSizeInBits(GEP->getType()); + unsigned BitWidth = DL.getPointerTypeSizeInBits(GEP->getType()); APInt TmpOffset(BitWidth, 0); // If the base isn't a global+constant, we aren't either. - if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, TD)) + if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, DL)) return false; // Otherwise, add any offset that our operands provide. - if (!GEP->accumulateConstantOffset(TD, TmpOffset)) + if (!GEP->accumulateConstantOffset(DL, TmpOffset)) return false; Offset = TmpOffset; @@ -276,11 +275,11 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, /// Recursive helper to read bits out of global. C is the constant being copied /// out of. ByteOffset is an offset into C. CurPtr is the pointer to copy /// results into and BytesLeft is the number of bytes left in -/// the CurPtr buffer. TD is the target data. +/// the CurPtr buffer. DL is the DataLayout. static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr, unsigned BytesLeft, - const DataLayout &TD) { - assert(ByteOffset <= TD.getTypeAllocSize(C->getType()) && + const DataLayout &DL) { + assert(ByteOffset <= DL.getTypeAllocSize(C->getType()) && "Out of range access"); // If this element is zero or undefined, we can just return since *CurPtr is @@ -298,7 +297,7 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, for (unsigned i = 0; i != BytesLeft && ByteOffset != IntBytes; ++i) { int n = ByteOffset; - if (!TD.isLittleEndian()) + if (!DL.isLittleEndian()) n = IntBytes - n - 1; CurPtr[i] = (unsigned char)(Val >> (n * 8)); ++ByteOffset; @@ -308,22 +307,22 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, if (ConstantFP *CFP = dyn_cast(C)) { if (CFP->getType()->isDoubleTy()) { - C = FoldBitCast(C, Type::getInt64Ty(C->getContext()), TD); - return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, TD); + C = FoldBitCast(C, Type::getInt64Ty(C->getContext()), DL); + return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL); } if (CFP->getType()->isFloatTy()){ - C = FoldBitCast(C, Type::getInt32Ty(C->getContext()), TD); - return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, TD); + C = FoldBitCast(C, Type::getInt32Ty(C->getContext()), DL); + return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL); } if (CFP->getType()->isHalfTy()){ - C = FoldBitCast(C, Type::getInt16Ty(C->getContext()), TD); - return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, TD); + C = FoldBitCast(C, Type::getInt16Ty(C->getContext()), DL); + return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL); } return false; } if (ConstantStruct *CS = dyn_cast(C)) { - const StructLayout *SL = TD.getStructLayout(CS->getType()); + const StructLayout *SL = DL.getStructLayout(CS->getType()); unsigned Index = SL->getElementContainingOffset(ByteOffset); uint64_t CurEltOffset = SL->getElementOffset(Index); ByteOffset -= CurEltOffset; @@ -331,11 +330,11 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, while (1) { // If the element access is to the element itself and not to tail padding, // read the bytes from the element. - uint64_t EltSize = TD.getTypeAllocSize(CS->getOperand(Index)->getType()); + uint64_t EltSize = DL.getTypeAllocSize(CS->getOperand(Index)->getType()); if (ByteOffset < EltSize && !ReadDataFromGlobal(CS->getOperand(Index), ByteOffset, CurPtr, - BytesLeft, TD)) + BytesLeft, DL)) return false; ++Index; @@ -362,7 +361,7 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, if (isa(C) || isa(C) || isa(C)) { Type *EltTy = C->getType()->getSequentialElementType(); - uint64_t EltSize = TD.getTypeAllocSize(EltTy); + uint64_t EltSize = DL.getTypeAllocSize(EltTy); uint64_t Index = ByteOffset / EltSize; uint64_t Offset = ByteOffset - Index * EltSize; uint64_t NumElts; @@ -373,7 +372,7 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, for (; Index != NumElts; ++Index) { if (!ReadDataFromGlobal(C->getAggregateElement(Index), Offset, CurPtr, - BytesLeft, TD)) + BytesLeft, DL)) return false; uint64_t BytesWritten = EltSize - Offset; @@ -390,9 +389,9 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, if (ConstantExpr *CE = dyn_cast(C)) { if (CE->getOpcode() == Instruction::IntToPtr && - CE->getOperand(0)->getType() == TD.getIntPtrType(CE->getType())) { + CE->getOperand(0)->getType() == DL.getIntPtrType(CE->getType())) { return ReadDataFromGlobal(CE->getOperand(0), ByteOffset, CurPtr, - BytesLeft, TD); + BytesLeft, DL); } } @@ -401,7 +400,7 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, } static Constant *FoldReinterpretLoadFromConstPtr(Constant *C, - const DataLayout &TD) { + const DataLayout &DL) { PointerType *PTy = cast(C->getType()); Type *LoadTy = PTy->getElementType(); IntegerType *IntType = dyn_cast(LoadTy); @@ -423,14 +422,13 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C, MapTy = Type::getInt64PtrTy(C->getContext(), AS); else if (LoadTy->isVectorTy()) { MapTy = PointerType::getIntNPtrTy(C->getContext(), - TD.getTypeAllocSizeInBits(LoadTy), - AS); + DL.getTypeAllocSizeInBits(LoadTy), AS); } else return nullptr; - C = FoldBitCast(C, MapTy, TD); - if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, TD)) - return FoldBitCast(Res, LoadTy, TD); + C = FoldBitCast(C, MapTy, DL); + if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, DL)) + return FoldBitCast(Res, LoadTy, DL); return nullptr; } @@ -440,7 +438,7 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C, GlobalValue *GVal; APInt Offset; - if (!IsConstantOffsetFromGlobal(C, GVal, Offset, TD)) + if (!IsConstantOffsetFromGlobal(C, GVal, Offset, DL)) return nullptr; GlobalVariable *GV = dyn_cast(GVal); @@ -455,16 +453,16 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C, // If we're not accessing anything in this constant, the result is undefined. if (Offset.getZExtValue() >= - TD.getTypeAllocSize(GV->getInitializer()->getType())) + DL.getTypeAllocSize(GV->getInitializer()->getType())) return UndefValue::get(IntType); unsigned char RawBytes[32] = {0}; if (!ReadDataFromGlobal(GV->getInitializer(), Offset.getZExtValue(), RawBytes, - BytesLoaded, TD)) + BytesLoaded, DL)) return nullptr; APInt ResultVal = APInt(IntType->getBitWidth(), 0); - if (TD.isLittleEndian()) { + if (DL.isLittleEndian()) { ResultVal = RawBytes[BytesLoaded - 1]; for (unsigned i = 1; i != BytesLoaded; ++i) { ResultVal <<= 8; @@ -482,9 +480,7 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C, } static Constant *ConstantFoldLoadThroughBitcast(ConstantExpr *CE, - const DataLayout *DL) { - if (!DL) - return nullptr; + const DataLayout &DL) { auto *DestPtrTy = dyn_cast(CE->getType()); if (!DestPtrTy) return nullptr; @@ -499,7 +495,7 @@ static Constant *ConstantFoldLoadThroughBitcast(ConstantExpr *CE, // If the type sizes are the same and a cast is legal, just directly // cast the constant. - if (DL->getTypeSizeInBits(DestTy) == DL->getTypeSizeInBits(SrcTy)) { + if (DL.getTypeSizeInBits(DestTy) == DL.getTypeSizeInBits(SrcTy)) { Instruction::CastOps Cast = Instruction::BitCast; // If we are going from a pointer to int or vice versa, we spell the cast // differently. @@ -530,7 +526,7 @@ static Constant *ConstantFoldLoadThroughBitcast(ConstantExpr *CE, /// Return the value that a load from C would produce if it is constant and /// determinable. If this is not determinable, return null. Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, - const DataLayout *TD) { + const DataLayout &DL) { // First, try the easy cases: if (GlobalVariable *GV = dyn_cast(C)) if (GV->isConstant() && GV->hasDefinitiveInitializer()) @@ -552,13 +548,13 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, } if (CE->getOpcode() == Instruction::BitCast) - if (Constant *LoadedC = ConstantFoldLoadThroughBitcast(CE, TD)) + if (Constant *LoadedC = ConstantFoldLoadThroughBitcast(CE, DL)) return LoadedC; // Instead of loading constant c string, use corresponding integer value // directly if string length is small enough. StringRef Str; - if (TD && getConstantStringInfo(CE, Str) && !Str.empty()) { + if (getConstantStringInfo(CE, Str) && !Str.empty()) { unsigned StrLen = Str.size(); Type *Ty = cast(CE->getType())->getElementType(); unsigned NumBits = Ty->getPrimitiveSizeInBits(); @@ -568,7 +564,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, (isa(Ty) || Ty->isFloatingPointTy())) { APInt StrVal(NumBits, 0); APInt SingleChar(NumBits, 0); - if (TD->isLittleEndian()) { + if (DL.isLittleEndian()) { for (signed i = StrLen-1; i >= 0; i--) { SingleChar = (uint64_t) Str[i] & UCHAR_MAX; StrVal = (StrVal << 8) | SingleChar; @@ -593,7 +589,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, // If this load comes from anywhere in a constant global, and if the global // is all undef or zero, we know what it loads. if (GlobalVariable *GV = - dyn_cast(GetUnderlyingObject(CE, TD))) { + dyn_cast(GetUnderlyingObject(CE, DL))) { if (GV->isConstant() && GV->hasDefinitiveInitializer()) { Type *ResTy = cast(C->getType())->getElementType(); if (GV->getInitializer()->isNullValue()) @@ -604,16 +600,15 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, } // Try hard to fold loads from bitcasted strange and non-type-safe things. - if (TD) - return FoldReinterpretLoadFromConstPtr(CE, *TD); - return nullptr; + return FoldReinterpretLoadFromConstPtr(CE, DL); } -static Constant *ConstantFoldLoadInst(const LoadInst *LI, const DataLayout *TD){ +static Constant *ConstantFoldLoadInst(const LoadInst *LI, + const DataLayout &DL) { if (LI->isVolatile()) return nullptr; if (Constant *C = dyn_cast(LI->getOperand(0))) - return ConstantFoldLoadFromConstPtr(C, TD); + return ConstantFoldLoadFromConstPtr(C, DL); return nullptr; } @@ -623,16 +618,16 @@ static Constant *ConstantFoldLoadInst(const LoadInst *LI, const DataLayout *TD){ /// these together. If target data info is available, it is provided as DL, /// otherwise DL is null. static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, - Constant *Op1, const DataLayout *DL){ + Constant *Op1, + const DataLayout &DL) { // SROA // Fold (and 0xffffffff00000000, (shl x, 32)) -> shl. // Fold (lshr (or X, Y), 32) -> (lshr [X/Y], 32) if one doesn't contribute // bits. - - if (Opc == Instruction::And && DL) { - unsigned BitWidth = DL->getTypeSizeInBits(Op0->getType()->getScalarType()); + if (Opc == Instruction::And) { + unsigned BitWidth = DL.getTypeSizeInBits(Op0->getType()->getScalarType()); APInt KnownZero0(BitWidth, 0), KnownOne0(BitWidth, 0); APInt KnownZero1(BitWidth, 0), KnownOne1(BitWidth, 0); computeKnownBits(Op0, KnownZero0, KnownOne0, DL); @@ -655,14 +650,13 @@ static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, // If the constant expr is something like &A[123] - &A[4].f, fold this into a // constant. This happens frequently when iterating over a global array. - if (Opc == Instruction::Sub && DL) { + if (Opc == Instruction::Sub) { GlobalValue *GV1, *GV2; APInt Offs1, Offs2; - if (IsConstantOffsetFromGlobal(Op0, GV1, Offs1, *DL)) - if (IsConstantOffsetFromGlobal(Op1, GV2, Offs2, *DL) && - GV1 == GV2) { - unsigned OpSize = DL->getTypeSizeInBits(Op0->getType()); + if (IsConstantOffsetFromGlobal(Op0, GV1, Offs1, DL)) + if (IsConstantOffsetFromGlobal(Op1, GV2, Offs2, DL) && GV1 == GV2) { + unsigned OpSize = DL.getTypeSizeInBits(Op0->getType()); // (&GV+C1) - (&GV+C2) -> C1-C2, pointer arithmetic cannot overflow. // PtrToInt may change the bitwidth so we have convert to the right size @@ -677,13 +671,10 @@ static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, /// If array indices are not pointer-sized integers, explicitly cast them so /// that they aren't implicitly casted by the getelementptr. -static Constant *CastGEPIndices(ArrayRef Ops, - Type *ResultTy, const DataLayout *TD, +static Constant *CastGEPIndices(ArrayRef Ops, Type *ResultTy, + const DataLayout &DL, const TargetLibraryInfo *TLI) { - if (!TD) - return nullptr; - - Type *IntPtrTy = TD->getIntPtrType(ResultTy); + Type *IntPtrTy = DL.getIntPtrType(ResultTy); bool Any = false; SmallVector NewIdxs; @@ -708,7 +699,7 @@ static Constant *CastGEPIndices(ArrayRef Ops, Constant *C = ConstantExpr::getGetElementPtr(Ops[0], NewIdxs); if (ConstantExpr *CE = dyn_cast(C)) { - if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) + if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI)) C = Folded; } @@ -733,14 +724,14 @@ static Constant* StripPtrCastKeepAS(Constant* Ptr) { /// If we can symbolically evaluate the GEP constant expression, do so. static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, - Type *ResultTy, const DataLayout *TD, + Type *ResultTy, const DataLayout &DL, const TargetLibraryInfo *TLI) { Constant *Ptr = Ops[0]; - if (!TD || !Ptr->getType()->getPointerElementType()->isSized() || + if (!Ptr->getType()->getPointerElementType()->isSized() || !Ptr->getType()->isPointerTy()) return nullptr; - Type *IntPtrTy = TD->getIntPtrType(Ptr->getType()); + Type *IntPtrTy = DL.getIntPtrType(Ptr->getType()); Type *ResultElementTy = ResultTy->getPointerElementType(); // If this is a constant expr gep that is effectively computing an @@ -760,19 +751,19 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, Res = ConstantExpr::getSub(Res, CE->getOperand(1)); Res = ConstantExpr::getIntToPtr(Res, ResultTy); if (ConstantExpr *ResCE = dyn_cast(Res)) - Res = ConstantFoldConstantExpression(ResCE, TD, TLI); + Res = ConstantFoldConstantExpression(ResCE, DL, TLI); return Res; } } return nullptr; } - unsigned BitWidth = TD->getTypeSizeInBits(IntPtrTy); + unsigned BitWidth = DL.getTypeSizeInBits(IntPtrTy); APInt Offset = - APInt(BitWidth, TD->getIndexedOffset(Ptr->getType(), - makeArrayRef((Value *const*) - Ops.data() + 1, - Ops.size() - 1))); + APInt(BitWidth, + DL.getIndexedOffset( + Ptr->getType(), + makeArrayRef((Value * const *)Ops.data() + 1, Ops.size() - 1))); Ptr = StripPtrCastKeepAS(Ptr); // If this is a GEP of a GEP, fold it all into a single GEP. @@ -790,8 +781,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, break; Ptr = cast(GEP->getOperand(0)); - Offset += APInt(BitWidth, - TD->getIndexedOffset(Ptr->getType(), NestedOps)); + Offset += APInt(BitWidth, DL.getIndexedOffset(Ptr->getType(), NestedOps)); Ptr = StripPtrCastKeepAS(Ptr); } @@ -831,7 +821,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, } // Determine which element of the array the offset points into. - APInt ElemSize(BitWidth, TD->getTypeAllocSize(ATy->getElementType())); + APInt ElemSize(BitWidth, DL.getTypeAllocSize(ATy->getElementType())); if (ElemSize == 0) // The element size is 0. This may be [0 x Ty]*, so just use a zero // index for this level and proceed to the next level to see if it can @@ -850,7 +840,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, // can't re-form this GEP in a regular form, so bail out. The pointer // operand likely went through casts that are necessary to make the GEP // sensible. - const StructLayout &SL = *TD->getStructLayout(STy); + const StructLayout &SL = *DL.getStructLayout(STy); if (Offset.uge(SL.getSizeInBytes())) break; @@ -882,7 +872,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, // If we ended up indexing a member with a type that doesn't match // the type of what the original indices indexed, add a cast. if (Ty != ResultElementTy) - C = FoldBitCast(C, ResultTy, *TD); + C = FoldBitCast(C, ResultTy, DL); return C; } @@ -898,8 +888,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, /// Note that this fails if not all of the operands are constant. Otherwise, /// this function can only fail when attempting to fold instructions like loads /// and stores, which have no constant expression form. -Constant *llvm::ConstantFoldInstruction(Instruction *I, - const DataLayout *TD, +Constant *llvm::ConstantFoldInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI) { // Handle PHI nodes quickly here... if (PHINode *PN = dyn_cast(I)) { @@ -919,7 +908,7 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I, return nullptr; // Fold the PHI's operands. if (ConstantExpr *NewC = dyn_cast(C)) - C = ConstantFoldConstantExpression(NewC, TD, TLI); + C = ConstantFoldConstantExpression(NewC, DL, TLI); // If the incoming value is a different constant to // the one we saw previously, then give up. if (CommonValue && C != CommonValue) @@ -942,17 +931,17 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I, // Fold the Instruction's operands. if (ConstantExpr *NewCE = dyn_cast(Op)) - Op = ConstantFoldConstantExpression(NewCE, TD, TLI); + Op = ConstantFoldConstantExpression(NewCE, DL, TLI); Ops.push_back(Op); } if (const CmpInst *CI = dyn_cast(I)) return ConstantFoldCompareInstOperands(CI->getPredicate(), Ops[0], Ops[1], - TD, TLI); + DL, TLI); if (const LoadInst *LI = dyn_cast(I)) - return ConstantFoldLoadInst(LI, TD); + return ConstantFoldLoadInst(LI, DL); if (InsertValueInst *IVI = dyn_cast(I)) { return ConstantExpr::getInsertValue( @@ -967,11 +956,11 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I, EVI->getIndices()); } - return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Ops, TD, TLI); + return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Ops, DL, TLI); } static Constant * -ConstantFoldConstantExpressionImpl(const ConstantExpr *CE, const DataLayout *TD, +ConstantFoldConstantExpressionImpl(const ConstantExpr *CE, const DataLayout &DL, const TargetLibraryInfo *TLI, SmallPtrSetImpl &FoldedOps) { SmallVector Ops; @@ -982,25 +971,25 @@ ConstantFoldConstantExpressionImpl(const ConstantExpr *CE, const DataLayout *TD, // a ConstantExpr, we don't have to process it again. if (ConstantExpr *NewCE = dyn_cast(NewC)) { if (FoldedOps.insert(NewCE).second) - NewC = ConstantFoldConstantExpressionImpl(NewCE, TD, TLI, FoldedOps); + NewC = ConstantFoldConstantExpressionImpl(NewCE, DL, TLI, FoldedOps); } Ops.push_back(NewC); } if (CE->isCompare()) return ConstantFoldCompareInstOperands(CE->getPredicate(), Ops[0], Ops[1], - TD, TLI); - return ConstantFoldInstOperands(CE->getOpcode(), CE->getType(), Ops, TD, TLI); + DL, TLI); + return ConstantFoldInstOperands(CE->getOpcode(), CE->getType(), Ops, DL, TLI); } /// Attempt to fold the constant expression /// using the specified DataLayout. If successful, the constant result is /// result is returned, if not, null is returned. Constant *llvm::ConstantFoldConstantExpression(const ConstantExpr *CE, - const DataLayout *TD, + const DataLayout &DL, const TargetLibraryInfo *TLI) { SmallPtrSet FoldedOps; - return ConstantFoldConstantExpressionImpl(CE, TD, TLI, FoldedOps); + return ConstantFoldConstantExpressionImpl(CE, DL, TLI, FoldedOps); } /// Attempt to constant fold an instruction with the @@ -1015,12 +1004,12 @@ Constant *llvm::ConstantFoldConstantExpression(const ConstantExpr *CE, /// Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, ArrayRef Ops, - const DataLayout *TD, + const DataLayout &DL, const TargetLibraryInfo *TLI) { // Handle easy binops first. if (Instruction::isBinaryOp(Opcode)) { if (isa(Ops[0]) || isa(Ops[1])) { - if (Constant *C = SymbolicallyEvaluateBinop(Opcode, Ops[0], Ops[1], TD)) + if (Constant *C = SymbolicallyEvaluateBinop(Opcode, Ops[0], Ops[1], DL)) return C; } @@ -1040,10 +1029,10 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, // If the input is a inttoptr, eliminate the pair. This requires knowing // the width of a pointer, so it can't be done in ConstantExpr::getCast. if (ConstantExpr *CE = dyn_cast(Ops[0])) { - if (TD && CE->getOpcode() == Instruction::IntToPtr) { + if (CE->getOpcode() == Instruction::IntToPtr) { Constant *Input = CE->getOperand(0); unsigned InWidth = Input->getType()->getScalarSizeInBits(); - unsigned PtrWidth = TD->getPointerTypeSizeInBits(CE->getType()); + unsigned PtrWidth = DL.getPointerTypeSizeInBits(CE->getType()); if (PtrWidth < InWidth) { Constant *Mask = ConstantInt::get(CE->getContext(), @@ -1061,15 +1050,15 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, // This requires knowing the width of a pointer, so it can't be done in // ConstantExpr::getCast. if (ConstantExpr *CE = dyn_cast(Ops[0])) { - if (TD && CE->getOpcode() == Instruction::PtrToInt) { + if (CE->getOpcode() == Instruction::PtrToInt) { Constant *SrcPtr = CE->getOperand(0); - unsigned SrcPtrSize = TD->getPointerTypeSizeInBits(SrcPtr->getType()); + unsigned SrcPtrSize = DL.getPointerTypeSizeInBits(SrcPtr->getType()); unsigned MidIntSize = CE->getType()->getScalarSizeInBits(); if (MidIntSize >= SrcPtrSize) { unsigned SrcAS = SrcPtr->getType()->getPointerAddressSpace(); if (SrcAS == DestTy->getPointerAddressSpace()) - return FoldBitCast(CE->getOperand(0), DestTy, *TD); + return FoldBitCast(CE->getOperand(0), DestTy, DL); } } } @@ -1087,9 +1076,7 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, case Instruction::AddrSpaceCast: return ConstantExpr::getCast(Opcode, Ops[0], DestTy); case Instruction::BitCast: - if (TD) - return FoldBitCast(Ops[0], DestTy, *TD); - return ConstantExpr::getBitCast(Ops[0], DestTy); + return FoldBitCast(Ops[0], DestTy, DL); case Instruction::Select: return ConstantExpr::getSelect(Ops[0], Ops[1], Ops[2]); case Instruction::ExtractElement: @@ -1099,9 +1086,9 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, case Instruction::ShuffleVector: return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]); case Instruction::GetElementPtr: - if (Constant *C = CastGEPIndices(Ops, DestTy, TD, TLI)) + if (Constant *C = CastGEPIndices(Ops, DestTy, DL, TLI)) return C; - if (Constant *C = SymbolicallyEvaluateGEP(Ops, DestTy, TD, TLI)) + if (Constant *C = SymbolicallyEvaluateGEP(Ops, DestTy, DL, TLI)) return C; return ConstantExpr::getGetElementPtr(Ops[0], Ops.slice(1)); @@ -1113,43 +1100,44 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, /// returns a constant expression of the specified operands. Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate, Constant *Ops0, Constant *Ops1, - const DataLayout *TD, + const DataLayout &DL, const TargetLibraryInfo *TLI) { // fold: icmp (inttoptr x), null -> icmp x, 0 // fold: icmp (ptrtoint x), 0 -> icmp x, null // fold: icmp (inttoptr x), (inttoptr y) -> icmp trunc/zext x, trunc/zext y // fold: icmp (ptrtoint x), (ptrtoint y) -> icmp x, y // - // ConstantExpr::getCompare cannot do this, because it doesn't have TD + // FIXME: The following comment is out of data and the DataLayout is here now. + // ConstantExpr::getCompare cannot do this, because it doesn't have DL // around to know if bit truncation is happening. if (ConstantExpr *CE0 = dyn_cast(Ops0)) { - if (TD && Ops1->isNullValue()) { + if (Ops1->isNullValue()) { if (CE0->getOpcode() == Instruction::IntToPtr) { - Type *IntPtrTy = TD->getIntPtrType(CE0->getType()); + Type *IntPtrTy = DL.getIntPtrType(CE0->getType()); // Convert the integer value to the right size to ensure we get the // proper extension or truncation. Constant *C = ConstantExpr::getIntegerCast(CE0->getOperand(0), IntPtrTy, false); Constant *Null = Constant::getNullValue(C->getType()); - return ConstantFoldCompareInstOperands(Predicate, C, Null, TD, TLI); + return ConstantFoldCompareInstOperands(Predicate, C, Null, DL, TLI); } // Only do this transformation if the int is intptrty in size, otherwise // there is a truncation or extension that we aren't modeling. if (CE0->getOpcode() == Instruction::PtrToInt) { - Type *IntPtrTy = TD->getIntPtrType(CE0->getOperand(0)->getType()); + Type *IntPtrTy = DL.getIntPtrType(CE0->getOperand(0)->getType()); if (CE0->getType() == IntPtrTy) { Constant *C = CE0->getOperand(0); Constant *Null = Constant::getNullValue(C->getType()); - return ConstantFoldCompareInstOperands(Predicate, C, Null, TD, TLI); + return ConstantFoldCompareInstOperands(Predicate, C, Null, DL, TLI); } } } if (ConstantExpr *CE1 = dyn_cast(Ops1)) { - if (TD && CE0->getOpcode() == CE1->getOpcode()) { + if (CE0->getOpcode() == CE1->getOpcode()) { if (CE0->getOpcode() == Instruction::IntToPtr) { - Type *IntPtrTy = TD->getIntPtrType(CE0->getType()); + Type *IntPtrTy = DL.getIntPtrType(CE0->getType()); // Convert the integer value to the right size to ensure we get the // proper extension or truncation. @@ -1157,20 +1145,17 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate, IntPtrTy, false); Constant *C1 = ConstantExpr::getIntegerCast(CE1->getOperand(0), IntPtrTy, false); - return ConstantFoldCompareInstOperands(Predicate, C0, C1, TD, TLI); + return ConstantFoldCompareInstOperands(Predicate, C0, C1, DL, TLI); } // Only do this transformation if the int is intptrty in size, otherwise // there is a truncation or extension that we aren't modeling. if (CE0->getOpcode() == Instruction::PtrToInt) { - Type *IntPtrTy = TD->getIntPtrType(CE0->getOperand(0)->getType()); + Type *IntPtrTy = DL.getIntPtrType(CE0->getOperand(0)->getType()); if (CE0->getType() == IntPtrTy && CE0->getOperand(0)->getType() == CE1->getOperand(0)->getType()) { - return ConstantFoldCompareInstOperands(Predicate, - CE0->getOperand(0), - CE1->getOperand(0), - TD, - TLI); + return ConstantFoldCompareInstOperands( + Predicate, CE0->getOperand(0), CE1->getOperand(0), DL, TLI); } } } @@ -1180,16 +1165,14 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate, // icmp ne (or x, y), 0 -> (icmp ne x, 0) | (icmp ne y, 0) if ((Predicate == ICmpInst::ICMP_EQ || Predicate == ICmpInst::ICMP_NE) && CE0->getOpcode() == Instruction::Or && Ops1->isNullValue()) { - Constant *LHS = - ConstantFoldCompareInstOperands(Predicate, CE0->getOperand(0), Ops1, - TD, TLI); - Constant *RHS = - ConstantFoldCompareInstOperands(Predicate, CE0->getOperand(1), Ops1, - TD, TLI); + Constant *LHS = ConstantFoldCompareInstOperands( + Predicate, CE0->getOperand(0), Ops1, DL, TLI); + Constant *RHS = ConstantFoldCompareInstOperands( + Predicate, CE0->getOperand(1), Ops1, DL, TLI); unsigned OpC = Predicate == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; Constant *Ops[] = { LHS, RHS }; - return ConstantFoldInstOperands(OpC, LHS->getType(), Ops, TD, TLI); + return ConstantFoldInstOperands(OpC, LHS->getType(), Ops, DL, TLI); } } @@ -1451,26 +1434,16 @@ static Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, default: break; case Intrinsic::fabs: return ConstantFoldFP(fabs, V, Ty); -#if HAVE_LOG2 case Intrinsic::log2: return ConstantFoldFP(log2, V, Ty); -#endif -#if HAVE_LOG case Intrinsic::log: return ConstantFoldFP(log, V, Ty); -#endif -#if HAVE_LOG10 case Intrinsic::log10: return ConstantFoldFP(log10, V, Ty); -#endif -#if HAVE_EXP case Intrinsic::exp: return ConstantFoldFP(exp, V, Ty); -#endif -#if HAVE_EXP2 case Intrinsic::exp2: return ConstantFoldFP(exp2, V, Ty); -#endif case Intrinsic::floor: return ConstantFoldFP(floor, V, Ty); case Intrinsic::ceil: diff --git a/lib/Analysis/DependenceAnalysis.cpp b/lib/Analysis/DependenceAnalysis.cpp index fda664b..3374b48 100644 --- a/lib/Analysis/DependenceAnalysis.cpp +++ b/lib/Analysis/DependenceAnalysis.cpp @@ -52,6 +52,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/DependenceAnalysis.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/LoopInfo.h" @@ -59,6 +60,7 @@ #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/InstIterator.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -225,13 +227,11 @@ bool Dependence::isScalar(unsigned level) const { //===----------------------------------------------------------------------===// // FullDependence methods -FullDependence::FullDependence(Instruction *Source, - Instruction *Destination, +FullDependence::FullDependence(Instruction *Source, Instruction *Destination, bool PossiblyLoopIndependent, - unsigned CommonLevels) : - Dependence(Source, Destination), - Levels(CommonLevels), - LoopIndependent(PossiblyLoopIndependent) { + unsigned CommonLevels) + : Dependence(Source, Destination), Levels(CommonLevels), + LoopIndependent(PossiblyLoopIndependent) { Consistent = true; DV = CommonLevels ? new DVEntry[CommonLevels] : nullptr; } @@ -625,14 +625,12 @@ void Dependence::dump(raw_ostream &OS) const { OS << "!\n"; } - - -static -AliasAnalysis::AliasResult underlyingObjectsAlias(AliasAnalysis *AA, - const Value *A, - const Value *B) { - const Value *AObj = GetUnderlyingObject(A); - const Value *BObj = GetUnderlyingObject(B); +static AliasAnalysis::AliasResult underlyingObjectsAlias(AliasAnalysis *AA, + const DataLayout &DL, + const Value *A, + const Value *B) { + const Value *AObj = GetUnderlyingObject(A, DL); + const Value *BObj = GetUnderlyingObject(B, DL); return AA->alias(AObj, AA->getTypeStoreSize(AObj->getType()), BObj, AA->getTypeStoreSize(BObj->getType())); } @@ -3314,7 +3312,8 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst, Value *SrcPtr = getPointerOperand(Src); Value *DstPtr = getPointerOperand(Dst); - switch (underlyingObjectsAlias(AA, DstPtr, SrcPtr)) { + switch (underlyingObjectsAlias(AA, F->getParent()->getDataLayout(), DstPtr, + SrcPtr)) { case AliasAnalysis::MayAlias: case AliasAnalysis::PartialAlias: // cannot analyse objects if we don't understand their aliasing. @@ -3347,9 +3346,9 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst, DEBUG(dbgs() << " SrcPtrSCEV = " << *SrcPtrSCEV << "\n"); DEBUG(dbgs() << " DstPtrSCEV = " << *DstPtrSCEV << "\n"); - UsefulGEP = - isLoopInvariant(SrcPtrSCEV, LI->getLoopFor(Src->getParent())) && - isLoopInvariant(DstPtrSCEV, LI->getLoopFor(Dst->getParent())); + UsefulGEP = isLoopInvariant(SrcPtrSCEV, LI->getLoopFor(Src->getParent())) && + isLoopInvariant(DstPtrSCEV, LI->getLoopFor(Dst->getParent())) && + (SrcGEP->getNumOperands() == DstGEP->getNumOperands()); } unsigned Pairs = UsefulGEP ? SrcGEP->idx_end() - SrcGEP->idx_begin() : 1; SmallVector Pair(Pairs); @@ -3472,8 +3471,7 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst, LI->getLoopFor(Dst->getParent()), Pair[SI].Loops); Result.Consistent = false; - } - else if (Pair[SI].Classification == Subscript::ZIV) { + } else if (Pair[SI].Classification == Subscript::ZIV) { // always separable Separable.set(SI); } @@ -3525,8 +3523,8 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst, DEBUG(dbgs() << ", SIV\n"); unsigned Level; const SCEV *SplitIter = nullptr; - if (testSIV(Pair[SI].Src, Pair[SI].Dst, Level, - Result, NewConstraint, SplitIter)) + if (testSIV(Pair[SI].Src, Pair[SI].Dst, Level, Result, NewConstraint, + SplitIter)) return nullptr; break; } @@ -3574,8 +3572,8 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst, unsigned Level; const SCEV *SplitIter = nullptr; DEBUG(dbgs() << "SIV\n"); - if (testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level, - Result, NewConstraint, SplitIter)) + if (testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level, Result, NewConstraint, + SplitIter)) return nullptr; ConstrainedLevels.set(Level); if (intersectConstraints(&Constraints[Level], &NewConstraint)) { @@ -3651,8 +3649,10 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst, // update Result.DV from constraint vector DEBUG(dbgs() << " updating\n"); - for (int SJ = ConstrainedLevels.find_first(); - SJ >= 0; SJ = ConstrainedLevels.find_next(SJ)) { + for (int SJ = ConstrainedLevels.find_first(); SJ >= 0; + SJ = ConstrainedLevels.find_next(SJ)) { + if (SJ > (int)CommonLevels) + break; updateDirection(Result.DV[SJ - 1], Constraints[SJ]); if (Result.DV[SJ - 1].Direction == Dependence::DVEntry::NONE) return nullptr; @@ -3759,8 +3759,8 @@ const SCEV *DependenceAnalysis::getSplitIteration(const Dependence &Dep, assert(isLoadOrStore(Dst)); Value *SrcPtr = getPointerOperand(Src); Value *DstPtr = getPointerOperand(Dst); - assert(underlyingObjectsAlias(AA, DstPtr, SrcPtr) == - AliasAnalysis::MustAlias); + assert(underlyingObjectsAlias(AA, F->getParent()->getDataLayout(), DstPtr, + SrcPtr) == AliasAnalysis::MustAlias); // establish loop nesting levels establishNestingLevels(Src, Dst); @@ -3775,9 +3775,9 @@ const SCEV *DependenceAnalysis::getSplitIteration(const Dependence &Dep, SrcGEP->getPointerOperandType() == DstGEP->getPointerOperandType()) { const SCEV *SrcPtrSCEV = SE->getSCEV(SrcGEP->getPointerOperand()); const SCEV *DstPtrSCEV = SE->getSCEV(DstGEP->getPointerOperand()); - UsefulGEP = - isLoopInvariant(SrcPtrSCEV, LI->getLoopFor(Src->getParent())) && - isLoopInvariant(DstPtrSCEV, LI->getLoopFor(Dst->getParent())); + UsefulGEP = isLoopInvariant(SrcPtrSCEV, LI->getLoopFor(Src->getParent())) && + isLoopInvariant(DstPtrSCEV, LI->getLoopFor(Dst->getParent())) && + (SrcGEP->getNumOperands() == DstGEP->getNumOperands()); } unsigned Pairs = UsefulGEP ? SrcGEP->idx_end() - SrcGEP->idx_begin() : 1; SmallVector Pair(Pairs); diff --git a/lib/Analysis/IPA/CallGraphSCCPass.cpp b/lib/Analysis/IPA/CallGraphSCCPass.cpp index ded1de7..9d607cc 100644 --- a/lib/Analysis/IPA/CallGraphSCCPass.cpp +++ b/lib/Analysis/IPA/CallGraphSCCPass.cpp @@ -49,7 +49,7 @@ public: explicit CGPassManager() : ModulePass(ID), PMDataManager() { } - /// run - Execute all of the passes scheduled for execution. Keep track of + /// Execute all of the passes scheduled for execution. Keep track of /// whether any of the passes modifies the module, and if so, return true. bool runOnModule(Module &M) override; @@ -142,9 +142,8 @@ bool CGPassManager::RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC, FPPassManager *FPP = (FPPassManager*)P; // Run pass P on all functions in the current SCC. - for (CallGraphSCC::iterator I = CurSCC.begin(), E = CurSCC.end(); - I != E; ++I) { - if (Function *F = (*I)->getFunction()) { + for (CallGraphNode *CGN : CurSCC) { + if (Function *F = CGN->getFunction()) { dumpPassInfo(P, EXECUTION_MSG, ON_FUNCTION_MSG, F->getName()); { TimeRegion PassTimer(getPassTimer(FPP)); @@ -165,7 +164,7 @@ bool CGPassManager::RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC, } -/// RefreshCallGraph - Scan the functions in the specified CFG and resync the +/// Scan the functions in the specified CFG and resync the /// callgraph with the call sites found in it. This is used after /// FunctionPasses have potentially munged the callgraph, and can be used after /// CallGraphSCC passes to verify that they correctly updated the callgraph. @@ -181,9 +180,8 @@ bool CGPassManager::RefreshCallGraph(CallGraphSCC &CurSCC, DEBUG(dbgs() << "CGSCCPASSMGR: Refreshing SCC with " << CurSCC.size() << " nodes:\n"; - for (CallGraphSCC::iterator I = CurSCC.begin(), E = CurSCC.end(); - I != E; ++I) - (*I)->dump(); + for (CallGraphNode *CGN : CurSCC) + CGN->dump(); ); bool MadeChange = false; @@ -357,9 +355,8 @@ bool CGPassManager::RefreshCallGraph(CallGraphSCC &CurSCC, DEBUG(if (MadeChange) { dbgs() << "CGSCCPASSMGR: Refreshed SCC is now:\n"; - for (CallGraphSCC::iterator I = CurSCC.begin(), E = CurSCC.end(); - I != E; ++I) - (*I)->dump(); + for (CallGraphNode *CGN : CurSCC) + CGN->dump(); if (DevirtualizedCall) dbgs() << "CGSCCPASSMGR: Refresh devirtualized a call!\n"; @@ -372,15 +369,15 @@ bool CGPassManager::RefreshCallGraph(CallGraphSCC &CurSCC, return DevirtualizedCall; } -/// RunAllPassesOnSCC - Execute the body of the entire pass manager on the -/// specified SCC. This keeps track of whether a function pass devirtualizes +/// Execute the body of the entire pass manager on the specified SCC. +/// This keeps track of whether a function pass devirtualizes /// any calls and returns it in DevirtualizedCall. bool CGPassManager::RunAllPassesOnSCC(CallGraphSCC &CurSCC, CallGraph &CG, bool &DevirtualizedCall) { bool Changed = false; - // CallGraphUpToDate - Keep track of whether the callgraph is known to be - // up-to-date or not. The CGSSC pass manager runs two types of passes: + // Keep track of whether the callgraph is known to be up-to-date or not. + // The CGSSC pass manager runs two types of passes: // CallGraphSCC Passes and other random function passes. Because other // random function passes are not CallGraph aware, they may clobber the // call graph by introducing new calls or deleting other ones. This flag @@ -433,7 +430,7 @@ bool CGPassManager::RunAllPassesOnSCC(CallGraphSCC &CurSCC, CallGraph &CG, return Changed; } -/// run - Execute all of the passes scheduled for execution. Keep track of +/// Execute all of the passes scheduled for execution. Keep track of /// whether any of the passes modifies the module, and if so, return true. bool CGPassManager::runOnModule(Module &M) { CallGraph &CG = getAnalysis().getCallGraph(); @@ -519,7 +516,7 @@ bool CGPassManager::doFinalization(CallGraph &CG) { // CallGraphSCC Implementation //===----------------------------------------------------------------------===// -/// ReplaceNode - This informs the SCC and the pass manager that the specified +/// This informs the SCC and the pass manager that the specified /// Old node has been deleted, and New is to be used in its place. void CallGraphSCC::ReplaceNode(CallGraphNode *Old, CallGraphNode *New) { assert(Old != New && "Should not replace node with self"); @@ -578,8 +575,8 @@ void CallGraphSCCPass::assignPassManager(PMStack &PMS, CGP->add(this); } -/// getAnalysisUsage - For this class, we declare that we require and preserve -/// the call graph. If the derived class implements this method, it should +/// For this class, we declare that we require and preserve the call graph. +/// If the derived class implements this method, it should /// always explicitly call the implementation here. void CallGraphSCCPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); @@ -609,9 +606,9 @@ namespace { bool runOnSCC(CallGraphSCC &SCC) override { Out << Banner; - for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) { - if ((*I)->getFunction()) - (*I)->getFunction()->print(Out); + for (CallGraphNode *CGN : SCC) { + if (CGN->getFunction()) + CGN->getFunction()->print(Out); else Out << "\nPrinting Function\n"; } diff --git a/lib/Analysis/IPA/GlobalsModRef.cpp b/lib/Analysis/IPA/GlobalsModRef.cpp index 607c068..2208f32 100644 --- a/lib/Analysis/IPA/GlobalsModRef.cpp +++ b/lib/Analysis/IPA/GlobalsModRef.cpp @@ -96,7 +96,7 @@ namespace { } bool runOnModule(Module &M) override { - InitializeAliasAnalysis(this); + InitializeAliasAnalysis(this, &M.getDataLayout()); // Find non-addr taken globals. AnalyzeGlobals(M); @@ -322,7 +322,8 @@ bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) { continue; // Check the value being stored. - Value *Ptr = GetUnderlyingObject(SI->getOperand(0)); + Value *Ptr = GetUnderlyingObject(SI->getOperand(0), + GV->getParent()->getDataLayout()); if (!isAllocLikeFn(Ptr, TLI)) return false; // Too hard to analyze. @@ -481,8 +482,8 @@ AliasAnalysis::AliasResult GlobalsModRef::alias(const Location &LocA, const Location &LocB) { // Get the base object these pointers point to. - const Value *UV1 = GetUnderlyingObject(LocA.Ptr); - const Value *UV2 = GetUnderlyingObject(LocB.Ptr); + const Value *UV1 = GetUnderlyingObject(LocA.Ptr, *DL); + const Value *UV2 = GetUnderlyingObject(LocB.Ptr, *DL); // If either of the underlying values is a global, they may be non-addr-taken // globals, which we can answer queries about. @@ -540,8 +541,9 @@ GlobalsModRef::getModRefInfo(ImmutableCallSite CS, // If we are asking for mod/ref info of a direct call with a pointer to a // global we are tracking, return information if we have it. + const DataLayout &DL = CS.getCaller()->getParent()->getDataLayout(); if (const GlobalValue *GV = - dyn_cast(GetUnderlyingObject(Loc.Ptr))) + dyn_cast(GetUnderlyingObject(Loc.Ptr, DL))) if (GV->hasLocalLinkage()) if (const Function *F = CS.getCalledFunction()) if (NonAddressTakenGlobals.count(GV)) diff --git a/lib/Analysis/IPA/InlineCost.cpp b/lib/Analysis/IPA/InlineCost.cpp index cd494ba..eeb3b87 100644 --- a/lib/Analysis/IPA/InlineCost.cpp +++ b/lib/Analysis/IPA/InlineCost.cpp @@ -45,9 +45,6 @@ class CallAnalyzer : public InstVisitor { typedef InstVisitor Base; friend class InstVisitor; - // DataLayout if available, or null. - const DataLayout *const DL; - /// The TargetTransformInfo available for this compilation. const TargetTransformInfo &TTI; @@ -145,9 +142,9 @@ class CallAnalyzer : public InstVisitor { bool visitUnreachableInst(UnreachableInst &I); public: - CallAnalyzer(const DataLayout *DL, const TargetTransformInfo &TTI, - AssumptionCacheTracker *ACT, Function &Callee, int Threshold) - : DL(DL), TTI(TTI), ACT(ACT), F(Callee), Threshold(Threshold), Cost(0), + CallAnalyzer(const TargetTransformInfo &TTI, AssumptionCacheTracker *ACT, + Function &Callee, int Threshold) + : TTI(TTI), ACT(ACT), F(Callee), Threshold(Threshold), Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), @@ -244,10 +241,8 @@ bool CallAnalyzer::isGEPOffsetConstant(GetElementPtrInst &GEP) { /// Returns false if unable to compute the offset for any reason. Respects any /// simplified values known during the analysis of this callsite. bool CallAnalyzer::accumulateGEPOffset(GEPOperator &GEP, APInt &Offset) { - if (!DL) - return false; - - unsigned IntPtrWidth = DL->getPointerSizeInBits(); + const DataLayout &DL = F.getParent()->getDataLayout(); + unsigned IntPtrWidth = DL.getPointerSizeInBits(); assert(IntPtrWidth == Offset.getBitWidth()); for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP); @@ -263,12 +258,12 @@ bool CallAnalyzer::accumulateGEPOffset(GEPOperator &GEP, APInt &Offset) { // Handle a struct index, which adds its field offset to the pointer. if (StructType *STy = dyn_cast(*GTI)) { unsigned ElementIdx = OpC->getZExtValue(); - const StructLayout *SL = DL->getStructLayout(STy); + const StructLayout *SL = DL.getStructLayout(STy); Offset += APInt(IntPtrWidth, SL->getElementOffset(ElementIdx)); continue; } - APInt TypeSize(IntPtrWidth, DL->getTypeAllocSize(GTI.getIndexedType())); + APInt TypeSize(IntPtrWidth, DL.getTypeAllocSize(GTI.getIndexedType())); Offset += OpC->getValue().sextOrTrunc(IntPtrWidth) * TypeSize; } return true; @@ -289,9 +284,9 @@ bool CallAnalyzer::visitAlloca(AllocaInst &I) { // Accumulate the allocated size. if (I.isStaticAlloca()) { + const DataLayout &DL = F.getParent()->getDataLayout(); Type *Ty = I.getAllocatedType(); - AllocatedSize += (DL ? DL->getTypeAllocSize(Ty) : - Ty->getPrimitiveSizeInBits()); + AllocatedSize += DL.getTypeAllocSize(Ty); } // We will happily inline static alloca instructions. @@ -327,7 +322,7 @@ bool CallAnalyzer::visitGetElementPtr(GetElementPtrInst &I) { // Try to fold GEPs of constant-offset call site argument pointers. This // requires target data and inbounds GEPs. - if (DL && I.isInBounds()) { + if (I.isInBounds()) { // Check if we have a base + offset for the pointer. Value *Ptr = I.getPointerOperand(); std::pair BaseAndOffset = ConstantOffsetPtrs.lookup(Ptr); @@ -396,7 +391,6 @@ bool CallAnalyzer::visitBitCast(BitCastInst &I) { } bool CallAnalyzer::visitPtrToInt(PtrToIntInst &I) { - const DataLayout *DL = I.getDataLayout(); // Propagate constants through ptrtoint. Constant *COp = dyn_cast(I.getOperand(0)); if (!COp) @@ -410,7 +404,8 @@ bool CallAnalyzer::visitPtrToInt(PtrToIntInst &I) { // Track base/offset pairs when converted to a plain integer provided the // integer is large enough to represent the pointer. unsigned IntegerSize = I.getType()->getScalarSizeInBits(); - if (DL && IntegerSize >= DL->getPointerSizeInBits()) { + const DataLayout &DL = F.getParent()->getDataLayout(); + if (IntegerSize >= DL.getPointerSizeInBits()) { std::pair BaseAndOffset = ConstantOffsetPtrs.lookup(I.getOperand(0)); if (BaseAndOffset.first) @@ -433,7 +428,6 @@ bool CallAnalyzer::visitPtrToInt(PtrToIntInst &I) { } bool CallAnalyzer::visitIntToPtr(IntToPtrInst &I) { - const DataLayout *DL = I.getDataLayout(); // Propagate constants through ptrtoint. Constant *COp = dyn_cast(I.getOperand(0)); if (!COp) @@ -448,7 +442,8 @@ bool CallAnalyzer::visitIntToPtr(IntToPtrInst &I) { // modifications provided the integer is not too large. Value *Op = I.getOperand(0); unsigned IntegerSize = Op->getType()->getScalarSizeInBits(); - if (DL && IntegerSize <= DL->getPointerSizeInBits()) { + const DataLayout &DL = F.getParent()->getDataLayout(); + if (IntegerSize <= DL.getPointerSizeInBits()) { std::pair BaseAndOffset = ConstantOffsetPtrs.lookup(Op); if (BaseAndOffset.first) ConstantOffsetPtrs[&I] = BaseAndOffset; @@ -485,12 +480,14 @@ bool CallAnalyzer::visitUnaryInstruction(UnaryInstruction &I) { Constant *COp = dyn_cast(Operand); if (!COp) COp = SimplifiedValues.lookup(Operand); - if (COp) + if (COp) { + const DataLayout &DL = F.getParent()->getDataLayout(); if (Constant *C = ConstantFoldInstOperands(I.getOpcode(), I.getType(), COp, DL)) { SimplifiedValues[&I] = C; return true; } + } // Disable any SROA on the argument to arbitrary unary operators. disableSROA(Operand); @@ -595,6 +592,7 @@ bool CallAnalyzer::visitSub(BinaryOperator &I) { bool CallAnalyzer::visitBinaryOperator(BinaryOperator &I) { Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); + const DataLayout &DL = F.getParent()->getDataLayout(); if (!isa(LHS)) if (Constant *SimpleLHS = SimplifiedValues.lookup(LHS)) LHS = SimpleLHS; @@ -623,7 +621,7 @@ bool CallAnalyzer::visitBinaryOperator(BinaryOperator &I) { bool CallAnalyzer::visitLoad(LoadInst &I) { Value *SROAArg; DenseMap::iterator CostIt; - if (lookupSROAArgAndCost(I.getOperand(0), SROAArg, CostIt)) { + if (lookupSROAArgAndCost(I.getPointerOperand(), SROAArg, CostIt)) { if (I.isSimple()) { accumulateSROACost(CostIt, InlineConstants::InstrCost); return true; @@ -638,7 +636,7 @@ bool CallAnalyzer::visitLoad(LoadInst &I) { bool CallAnalyzer::visitStore(StoreInst &I) { Value *SROAArg; DenseMap::iterator CostIt; - if (lookupSROAArgAndCost(I.getOperand(0), SROAArg, CostIt)) { + if (lookupSROAArgAndCost(I.getPointerOperand(), SROAArg, CostIt)) { if (I.isSimple()) { accumulateSROACost(CostIt, InlineConstants::InstrCost); return true; @@ -788,7 +786,7 @@ bool CallAnalyzer::visitCallSite(CallSite CS) { // during devirtualization and so we want to give it a hefty bonus for // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. - CallAnalyzer CA(DL, TTI, ACT, *F, InlineConstants::IndirectCallThreshold); + CallAnalyzer CA(TTI, ACT, *F, InlineConstants::IndirectCallThreshold); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // bonus we want to apply, but don't go below zero. @@ -976,10 +974,11 @@ bool CallAnalyzer::analyzeBlock(BasicBlock *BB, /// returns 0 if V is not a pointer, and returns the constant '0' if there are /// no constant offsets applied. ConstantInt *CallAnalyzer::stripAndComputeInBoundsConstantOffsets(Value *&V) { - if (!DL || !V->getType()->isPointerTy()) + if (!V->getType()->isPointerTy()) return nullptr; - unsigned IntPtrWidth = DL->getPointerSizeInBits(); + const DataLayout &DL = F.getParent()->getDataLayout(); + unsigned IntPtrWidth = DL.getPointerSizeInBits(); APInt Offset = APInt::getNullValue(IntPtrWidth); // Even though we don't look through PHI nodes, we could be called on an @@ -1003,7 +1002,7 @@ ConstantInt *CallAnalyzer::stripAndComputeInBoundsConstantOffsets(Value *&V) { assert(V->getType()->isPointerTy() && "Unexpected operand type!"); } while (Visited.insert(V).second); - Type *IntPtrTy = DL->getIntPtrType(V->getContext()); + Type *IntPtrTy = DL.getIntPtrType(V->getContext()); return cast(ConstantInt::get(IntPtrTy, Offset)); } @@ -1034,16 +1033,17 @@ bool CallAnalyzer::analyzeCall(CallSite CS) { assert(NumVectorInstructions == 0); FiftyPercentVectorBonus = Threshold; TenPercentVectorBonus = Threshold / 2; + const DataLayout &DL = F.getParent()->getDataLayout(); // Give out bonuses per argument, as the instructions setting them up will // be gone after inlining. for (unsigned I = 0, E = CS.arg_size(); I != E; ++I) { - if (DL && CS.isByValArgument(I)) { + if (CS.isByValArgument(I)) { // We approximate the number of loads and stores needed by dividing the // size of the byval type by the target's pointer size. PointerType *PTy = cast(CS.getArgument(I)->getType()); - unsigned TypeSize = DL->getTypeSizeInBits(PTy->getElementType()); - unsigned PointerSize = DL->getPointerSizeInBits(); + unsigned TypeSize = DL.getTypeSizeInBits(PTy->getElementType()); + unsigned PointerSize = DL.getPointerSizeInBits(); // Ceiling division. unsigned NumStores = (TypeSize + PointerSize - 1) / PointerSize; @@ -1333,8 +1333,7 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee, DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(Callee->getDataLayout(), TTIWP->getTTI(*Callee), - ACT, *Callee, Threshold); + CallAnalyzer CA(TTIWP->getTTI(*Callee), ACT, *Callee, Threshold); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); diff --git a/lib/Analysis/IVUsers.cpp b/lib/Analysis/IVUsers.cpp index 140753c..b88b249 100644 --- a/lib/Analysis/IVUsers.cpp +++ b/lib/Analysis/IVUsers.cpp @@ -22,6 +22,7 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -113,6 +114,8 @@ static bool isSimplifiedLoopNest(BasicBlock *BB, const DominatorTree *DT, /// return true. Otherwise, return false. bool IVUsers::AddUsersImpl(Instruction *I, SmallPtrSetImpl &SimpleLoopNests) { + const DataLayout &DL = I->getModule()->getDataLayout(); + // Add this IV user to the Processed set before returning false to ensure that // all IV users are members of the set. See IVUsers::isIVUserOrOperand. if (!Processed.insert(I).second) @@ -124,14 +127,14 @@ bool IVUsers::AddUsersImpl(Instruction *I, // IVUsers is used by LSR which assumes that all SCEV expressions are safe to // pass to SCEVExpander. Expressions are not safe to expand if they represent // operations that are not safe to speculate, namely integer division. - if (!isa(I) && !isSafeToSpeculativelyExecute(I, DL)) + if (!isa(I) && !isSafeToSpeculativelyExecute(I)) return false; // LSR is not APInt clean, do not touch integers bigger than 64-bits. // Also avoid creating IVs of non-native types. For example, we don't want a // 64-bit IV in 32-bit code just because the loop has one 64-bit cast. uint64_t Width = SE->getTypeSizeInBits(I->getType()); - if (Width > 64 || (DL && !DL->isLegalInteger(Width))) + if (Width > 64 || !DL.isLegalInteger(Width)) return false; // Get the symbolic expression for this instruction. @@ -253,8 +256,6 @@ bool IVUsers::runOnLoop(Loop *l, LPPassManager &LPM) { LI = &getAnalysis().getLoopInfo(); DT = &getAnalysis().getDomTree(); SE = &getAnalysis(); - DataLayoutPass *DLP = getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; // Find all uses of induction variables in this loop, and categorize // them by stride. Start by finding all of the PHI nodes in the header for diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp index 0cb0982..99c477d 100644 --- a/lib/Analysis/InstructionSimplify.cpp +++ b/lib/Analysis/InstructionSimplify.cpp @@ -45,13 +45,13 @@ STATISTIC(NumReassoc, "Number of reassociations"); namespace { struct Query { - const DataLayout *DL; + const DataLayout &DL; const TargetLibraryInfo *TLI; const DominatorTree *DT; AssumptionCache *AC; const Instruction *CxtI; - Query(const DataLayout *DL, const TargetLibraryInfo *tli, + Query(const DataLayout &DL, const TargetLibraryInfo *tli, const DominatorTree *dt, AssumptionCache *ac = nullptr, const Instruction *cxti = nullptr) : DL(DL), TLI(tli), DT(dt), AC(ac), CxtI(cxti) {} @@ -584,7 +584,7 @@ static Value *SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, } Value *llvm::SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, - const DataLayout *DL, const TargetLibraryInfo *TLI, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyAddInst(Op0, Op1, isNSW, isNUW, Query(DL, TLI, DT, AC, CxtI), @@ -601,17 +601,11 @@ Value *llvm::SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, /// This is very similar to GetPointerBaseWithConstantOffset except it doesn't /// follow non-inbounds geps. This allows it to remain usable for icmp ult/etc. /// folding. -static Constant *stripAndComputeConstantOffsets(const DataLayout *DL, - Value *&V, +static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, bool AllowNonInbounds = false) { assert(V->getType()->getScalarType()->isPointerTy()); - // Without DataLayout, just be conservative for now. Theoretically, more could - // be done in this case. - if (!DL) - return ConstantInt::get(IntegerType::get(V->getContext(), 64), 0); - - Type *IntPtrTy = DL->getIntPtrType(V->getType())->getScalarType(); + Type *IntPtrTy = DL.getIntPtrType(V->getType())->getScalarType(); APInt Offset = APInt::getNullValue(IntPtrTy->getIntegerBitWidth()); // Even though we don't look through PHI nodes, we could be called on an @@ -621,7 +615,7 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout *DL, do { if (GEPOperator *GEP = dyn_cast(V)) { if ((!AllowNonInbounds && !GEP->isInBounds()) || - !GEP->accumulateConstantOffset(*DL, Offset)) + !GEP->accumulateConstantOffset(DL, Offset)) break; V = GEP->getPointerOperand(); } else if (Operator::getOpcode(V) == Instruction::BitCast) { @@ -646,8 +640,8 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout *DL, /// \brief Compute the constant difference between two pointer values. /// If the difference is not a constant, returns zero. -static Constant *computePointerDifference(const DataLayout *DL, - Value *LHS, Value *RHS) { +static Constant *computePointerDifference(const DataLayout &DL, Value *LHS, + Value *RHS) { Constant *LHSOffset = stripAndComputeConstantOffsets(DL, LHS); Constant *RHSOffset = stripAndComputeConstantOffsets(DL, RHS); @@ -783,7 +777,7 @@ static Value *SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, } Value *llvm::SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, - const DataLayout *DL, const TargetLibraryInfo *TLI, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifySubInst(Op0, Op1, isNSW, isNUW, Query(DL, TLI, DT, AC, CxtI), @@ -962,7 +956,7 @@ static Value *SimplifyMulInst(Value *Op0, Value *Op1, const Query &Q, } Value *llvm::SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -971,7 +965,7 @@ Value *llvm::SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, } Value *llvm::SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -980,7 +974,7 @@ Value *llvm::SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, } Value *llvm::SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -988,7 +982,7 @@ Value *llvm::SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, RecursionLimit); } -Value *llvm::SimplifyMulInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifyMulInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1092,7 +1086,7 @@ static Value *SimplifySDivInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifySDivInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifySDivInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1110,7 +1104,7 @@ static Value *SimplifyUDivInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifyUDivInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifyUDivInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1138,7 +1132,7 @@ static Value *SimplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, } Value *llvm::SimplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1217,7 +1211,7 @@ static Value *SimplifySRemInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifySRemInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifySRemInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1235,7 +1229,7 @@ static Value *SimplifyURemInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifyURemInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifyURemInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1263,7 +1257,7 @@ static Value *SimplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, } Value *llvm::SimplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1387,7 +1381,7 @@ static Value *SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, } Value *llvm::SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, - const DataLayout *DL, const TargetLibraryInfo *TLI, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyShlInst(Op0, Op1, isNSW, isNUW, Query(DL, TLI, DT, AC, CxtI), @@ -1411,7 +1405,7 @@ static Value *SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, } Value *llvm::SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1445,7 +1439,7 @@ static Value *SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, } Value *llvm::SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1596,9 +1590,11 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const Query &Q, // A & (-A) = A if A is a power of two or zero. if (match(Op0, m_Neg(m_Specific(Op1))) || match(Op1, m_Neg(m_Specific(Op0)))) { - if (isKnownToBeAPowerOfTwo(Op0, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT)) + if (isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, + Q.DT)) return Op0; - if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT)) + if (isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, + Q.DT)) return Op1; } @@ -1643,7 +1639,7 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifyAndInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifyAndInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1831,7 +1827,7 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifyOrInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifyOrInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1888,7 +1884,7 @@ static Value *SimplifyXorInst(Value *Op0, Value *Op1, const Query &Q, return nullptr; } -Value *llvm::SimplifyXorInst(Value *Op0, Value *Op1, const DataLayout *DL, +Value *llvm::SimplifyXorInst(Value *Op0, Value *Op1, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -1948,10 +1944,10 @@ static Value *ExtractEquivalentCondition(Value *V, CmpInst::Predicate Pred, // If the C and C++ standards are ever made sufficiently restrictive in this // area, it may be possible to update LLVM's semantics accordingly and reinstate // this optimization. -static Constant *computePointerICmp(const DataLayout *DL, +static Constant *computePointerICmp(const DataLayout &DL, const TargetLibraryInfo *TLI, - CmpInst::Predicate Pred, - Value *LHS, Value *RHS) { + CmpInst::Predicate Pred, Value *LHS, + Value *RHS) { // First, skip past any trivial no-ops. LHS = LHS->stripPointerCasts(); RHS = RHS->stripPointerCasts(); @@ -2369,8 +2365,8 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, // Turn icmp (ptrtoint x), (ptrtoint/constant) into a compare of the input // if the integer type is the same size as the pointer type. - if (MaxRecurse && Q.DL && isa(LI) && - Q.DL->getTypeSizeInBits(SrcTy) == DstTy->getPrimitiveSizeInBits()) { + if (MaxRecurse && isa(LI) && + Q.DL.getTypeSizeInBits(SrcTy) == DstTy->getPrimitiveSizeInBits()) { if (Constant *RHSC = dyn_cast(RHS)) { // Transfer the cast to the constant. if (Value *V = SimplifyICmpInst(Pred, SrcOp, @@ -3024,7 +3020,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, } Value *llvm::SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, Instruction *CxtI) { @@ -3054,8 +3050,13 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, if (Pred == FCmpInst::FCMP_TRUE) return ConstantInt::get(GetCompareTy(LHS), 1); - if (isa(RHS)) // fcmp pred X, undef -> undef - return UndefValue::get(GetCompareTy(LHS)); + // fcmp pred x, undef and fcmp pred undef, x + // fold to true if unordered, false if ordered + if (isa(LHS) || isa(RHS)) { + // Choosing NaN for the undef will always make unordered comparison succeed + // and ordered comparison fail. + return ConstantInt::get(GetCompareTy(LHS), CmpInst::isUnordered(Pred)); + } // fcmp x,x -> true/false. Not all compares are foldable. if (LHS == RHS) { @@ -3135,7 +3136,7 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, } Value *llvm::SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -3230,7 +3231,7 @@ static Value *SimplifySelectInst(Value *CondVal, Value *TrueVal, } Value *llvm::SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -3264,10 +3265,10 @@ static Value *SimplifyGEPInst(ArrayRef Ops, const Query &Q, unsigned) { return Ops[0]; Type *Ty = PtrTy->getElementType(); - if (Q.DL && Ty->isSized()) { + if (Ty->isSized()) { Value *P; uint64_t C; - uint64_t TyAllocSize = Q.DL->getTypeAllocSize(Ty); + uint64_t TyAllocSize = Q.DL.getTypeAllocSize(Ty); // getelementptr P, N -> P if P points to a type of zero size. if (TyAllocSize == 0) return Ops[0]; @@ -3275,7 +3276,7 @@ static Value *SimplifyGEPInst(ArrayRef Ops, const Query &Q, unsigned) { // The following transforms are only safe if the ptrtoint cast // doesn't truncate the pointers. if (Ops[1]->getType()->getScalarSizeInBits() == - Q.DL->getPointerSizeInBits(AS)) { + Q.DL.getPointerSizeInBits(AS)) { auto PtrToIntOrZero = [GEPTy](Value *P) -> Value * { if (match(P, m_Zero())) return Constant::getNullValue(GEPTy); @@ -3320,7 +3321,7 @@ static Value *SimplifyGEPInst(ArrayRef Ops, const Query &Q, unsigned) { return ConstantExpr::getGetElementPtr(cast(Ops[0]), Ops.slice(1)); } -Value *llvm::SimplifyGEPInst(ArrayRef Ops, const DataLayout *DL, +Value *llvm::SimplifyGEPInst(ArrayRef Ops, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -3357,7 +3358,7 @@ static Value *SimplifyInsertValueInst(Value *Agg, Value *Val, } Value *llvm::SimplifyInsertValueInst( - Value *Agg, Value *Val, ArrayRef Idxs, const DataLayout *DL, + Value *Agg, Value *Val, ArrayRef Idxs, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyInsertValueInst(Agg, Val, Idxs, Query(DL, TLI, DT, AC, CxtI), @@ -3405,7 +3406,7 @@ static Value *SimplifyTruncInst(Value *Op, Type *Ty, const Query &Q, unsigned) { return nullptr; } -Value *llvm::SimplifyTruncInst(Value *Op, Type *Ty, const DataLayout *DL, +Value *llvm::SimplifyTruncInst(Value *Op, Type *Ty, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -3502,7 +3503,7 @@ static Value *SimplifyFPBinOp(unsigned Opcode, Value *LHS, Value *RHS, } Value *llvm::SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, - const DataLayout *DL, const TargetLibraryInfo *TLI, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyBinOp(Opcode, LHS, RHS, Query(DL, TLI, DT, AC, CxtI), @@ -3510,7 +3511,7 @@ Value *llvm::SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, } Value *llvm::SimplifyFPBinOp(unsigned Opcode, Value *LHS, Value *RHS, - const FastMathFlags &FMF, const DataLayout *DL, + const FastMathFlags &FMF, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { @@ -3528,7 +3529,7 @@ static Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, } Value *llvm::SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, - const DataLayout *DL, const TargetLibraryInfo *TLI, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyCmpInst(Predicate, LHS, RHS, Query(DL, TLI, DT, AC, CxtI), @@ -3604,7 +3605,7 @@ static Value *SimplifyCall(Value *V, IterTy ArgBegin, IterTy ArgEnd, } Value *llvm::SimplifyCall(Value *V, User::op_iterator ArgBegin, - User::op_iterator ArgEnd, const DataLayout *DL, + User::op_iterator ArgEnd, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyCall(V, ArgBegin, ArgEnd, Query(DL, TLI, DT, AC, CxtI), @@ -3612,7 +3613,7 @@ Value *llvm::SimplifyCall(Value *V, User::op_iterator ArgBegin, } Value *llvm::SimplifyCall(Value *V, ArrayRef Args, - const DataLayout *DL, const TargetLibraryInfo *TLI, + const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { return ::SimplifyCall(V, Args.begin(), Args.end(), @@ -3621,7 +3622,7 @@ Value *llvm::SimplifyCall(Value *V, ArrayRef Args, /// SimplifyInstruction - See if we can compute a simplified version of this /// instruction. If not, this returns null. -Value *llvm::SimplifyInstruction(Instruction *I, const DataLayout *DL, +Value *llvm::SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC) { Value *Result; @@ -3769,12 +3770,12 @@ Value *llvm::SimplifyInstruction(Instruction *I, const DataLayout *DL, /// This routine returns 'true' only when *it* simplifies something. The passed /// in simplified value does not count toward this. static bool replaceAndRecursivelySimplifyImpl(Instruction *I, Value *SimpleV, - const DataLayout *DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC) { bool Simplified = false; SmallSetVector Worklist; + const DataLayout &DL = I->getModule()->getDataLayout(); // If we have an explicit value to collapse to, do that round of the // simplification loop by hand initially. @@ -3822,19 +3823,18 @@ static bool replaceAndRecursivelySimplifyImpl(Instruction *I, Value *SimpleV, return Simplified; } -bool llvm::recursivelySimplifyInstruction(Instruction *I, const DataLayout *DL, +bool llvm::recursivelySimplifyInstruction(Instruction *I, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC) { - return replaceAndRecursivelySimplifyImpl(I, nullptr, DL, TLI, DT, AC); + return replaceAndRecursivelySimplifyImpl(I, nullptr, TLI, DT, AC); } bool llvm::replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, - const DataLayout *DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC) { assert(I != SimpleV && "replaceAndRecursivelySimplify(X,X) is not valid!"); assert(SimpleV && "Must provide a simplified value."); - return replaceAndRecursivelySimplifyImpl(I, SimpleV, DL, TLI, DT, AC); + return replaceAndRecursivelySimplifyImpl(I, SimpleV, TLI, DT, AC); } diff --git a/lib/Analysis/JumpInstrTableInfo.cpp b/lib/Analysis/JumpInstrTableInfo.cpp deleted file mode 100644 index 7aae2a5..0000000 --- a/lib/Analysis/JumpInstrTableInfo.cpp +++ /dev/null @@ -1,55 +0,0 @@ -//===-- JumpInstrTableInfo.cpp: Info for Jump-Instruction Tables ----------===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief Information about jump-instruction tables that have been created by -/// JumpInstrTables pass. -/// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "jiti" - -#include "llvm/Analysis/JumpInstrTableInfo.h" -#include "llvm/Analysis/Passes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Type.h" -#include "llvm/Support/MathExtras.h" - -using namespace llvm; - -INITIALIZE_PASS(JumpInstrTableInfo, "jump-instr-table-info", - "Jump-Instruction Table Info", true, true) -char JumpInstrTableInfo::ID = 0; - -ImmutablePass *llvm::createJumpInstrTableInfoPass() { - return new JumpInstrTableInfo(); -} - -ModulePass *llvm::createJumpInstrTableInfoPass(unsigned Bound) { - // This cast is always safe, since Bound is always in a subset of uint64_t. - uint64_t B = static_cast(Bound); - return new JumpInstrTableInfo(B); -} - -JumpInstrTableInfo::JumpInstrTableInfo(uint64_t ByteAlign) - : ImmutablePass(ID), Tables(), ByteAlignment(ByteAlign) { - if (!llvm::isPowerOf2_64(ByteAlign)) { - // Note that we don't explicitly handle overflow here, since we handle the 0 - // case explicitly when a caller actually tries to create jumptable entries, - // and this is the return value on overflow. - ByteAlignment = llvm::NextPowerOf2(ByteAlign); - } - - initializeJumpInstrTableInfoPass(*PassRegistry::getPassRegistry()); -} - -JumpInstrTableInfo::~JumpInstrTableInfo() {} - -void JumpInstrTableInfo::insertEntry(FunctionType *TableFunTy, Function *Target, - Function *Jump) { - Tables[TableFunTy].push_back(JumpPair(Target, Jump)); -} diff --git a/lib/Analysis/LazyValueInfo.cpp b/lib/Analysis/LazyValueInfo.cpp index 87c31fd..e6f586a 100644 --- a/lib/Analysis/LazyValueInfo.cpp +++ b/lib/Analysis/LazyValueInfo.cpp @@ -191,7 +191,7 @@ public: /// Merge the specified lattice value into this one, updating this /// one and returning true if anything changed. - bool mergeIn(const LVILatticeVal &RHS) { + bool mergeIn(const LVILatticeVal &RHS, const DataLayout &DL) { if (RHS.isUndefined() || isOverdefined()) return false; if (RHS.isOverdefined()) return markOverdefined(); @@ -215,11 +215,9 @@ public: // Unless we can prove that the two Constants are different, we must // move to overdefined. - // FIXME: use DataLayout/TargetLibraryInfo for smarter constant folding. - if (ConstantInt *Res = dyn_cast( - ConstantFoldCompareInstOperands(CmpInst::ICMP_NE, - getConstant(), - RHS.getNotConstant()))) + if (ConstantInt *Res = + dyn_cast(ConstantFoldCompareInstOperands( + CmpInst::ICMP_NE, getConstant(), RHS.getNotConstant(), DL))) if (Res->isOne()) return markNotConstant(RHS.getNotConstant()); @@ -241,11 +239,9 @@ public: // Unless we can prove that the two Constants are different, we must // move to overdefined. - // FIXME: use DataLayout/TargetLibraryInfo for smarter constant folding. - if (ConstantInt *Res = dyn_cast( - ConstantFoldCompareInstOperands(CmpInst::ICMP_NE, - getNotConstant(), - RHS.getConstant()))) + if (ConstantInt *Res = + dyn_cast(ConstantFoldCompareInstOperands( + CmpInst::ICMP_NE, getNotConstant(), RHS.getConstant(), DL))) if (Res->isOne()) return false; @@ -346,21 +342,17 @@ namespace { /// Push BV onto BlockValueStack unless it's already in there. /// Returns true on success. bool pushBlockValue(const std::pair &BV) { - if (BlockValueSet.count(BV)) + if (!BlockValueSet.insert(BV).second) return false; // It's already in the stack. BlockValueStack.push(BV); - BlockValueSet.insert(BV); return true; } - /// A pointer to the cache of @llvm.assume calls. - AssumptionCache *AC; - /// An optional DL pointer. - const DataLayout *DL; - /// An optional DT pointer. - DominatorTree *DT; - + AssumptionCache *AC; ///< A pointer to the cache of @llvm.assume calls. + const DataLayout &DL; ///< A mandatory DataLayout + DominatorTree *DT; ///< An optional DT pointer. + friend struct LVIValueHandle; void insertResult(Value *Val, BasicBlock *BB, const LVILatticeVal &Result) { @@ -426,7 +418,7 @@ namespace { OverDefinedCache.clear(); } - LazyValueInfoCache(AssumptionCache *AC, const DataLayout *DL = nullptr, + LazyValueInfoCache(AssumptionCache *AC, const DataLayout &DL, DominatorTree *DT = nullptr) : AC(AC), DL(DL), DT(DT) {} }; @@ -579,11 +571,13 @@ bool LazyValueInfoCache::solveBlockValue(Value *Val, BasicBlock *BB) { static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) { if (LoadInst *L = dyn_cast(I)) { return L->getPointerAddressSpace() == 0 && - GetUnderlyingObject(L->getPointerOperand()) == Ptr; + GetUnderlyingObject(L->getPointerOperand(), + L->getModule()->getDataLayout()) == Ptr; } if (StoreInst *S = dyn_cast(I)) { return S->getPointerAddressSpace() == 0 && - GetUnderlyingObject(S->getPointerOperand()) == Ptr; + GetUnderlyingObject(S->getPointerOperand(), + S->getModule()->getDataLayout()) == Ptr; } if (MemIntrinsic *MI = dyn_cast(I)) { if (MI->isVolatile()) return false; @@ -593,11 +587,13 @@ static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) { if (!Len || Len->isZero()) return false; if (MI->getDestAddressSpace() == 0) - if (GetUnderlyingObject(MI->getRawDest()) == Ptr) + if (GetUnderlyingObject(MI->getRawDest(), + MI->getModule()->getDataLayout()) == Ptr) return true; if (MemTransferInst *MTI = dyn_cast(MI)) if (MTI->getSourceAddressSpace() == 0) - if (GetUnderlyingObject(MTI->getRawSource()) == Ptr) + if (GetUnderlyingObject(MTI->getRawSource(), + MTI->getModule()->getDataLayout()) == Ptr) return true; } return false; @@ -614,10 +610,11 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV, if (isKnownNonNull(Val)) { NotNull = true; } else { - Value *UnderlyingVal = GetUnderlyingObject(Val); + const DataLayout &DL = BB->getModule()->getDataLayout(); + Value *UnderlyingVal = GetUnderlyingObject(Val, DL); // If 'GetUnderlyingObject' didn't converge, skip it. It won't converge // inside InstructionDereferencesPointer either. - if (UnderlyingVal == GetUnderlyingObject(UnderlyingVal, nullptr, 1)) { + if (UnderlyingVal == GetUnderlyingObject(UnderlyingVal, DL, 1)) { for (Instruction &I : *BB) { if (InstructionDereferencesPointer(&I, UnderlyingVal)) { NotNull = true; @@ -651,7 +648,7 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV, if (EdgesMissing) continue; - Result.mergeIn(EdgeResult); + Result.mergeIn(EdgeResult, DL); // If we hit overdefined, exit early. The BlockVals entry is already set // to overdefined. @@ -696,7 +693,7 @@ bool LazyValueInfoCache::solveBlockValuePHINode(LVILatticeVal &BBLV, if (EdgesMissing) continue; - Result.mergeIn(EdgeResult); + Result.mergeIn(EdgeResult, DL); // If we hit overdefined, exit early. The BlockVals entry is already set // to overdefined. @@ -735,7 +732,7 @@ void LazyValueInfoCache::mergeAssumeBlockValueConstantRange(Value *Val, if (!AssumeVH) continue; auto *I = cast(AssumeVH); - if (!isValidAssumeForContext(I, BBI, DL, DT)) + if (!isValidAssumeForContext(I, BBI, DT)) continue; Value *C = I->getArgOperand(0); @@ -745,7 +742,7 @@ void LazyValueInfoCache::mergeAssumeBlockValueConstantRange(Value *Val, if (BBLV.isOverdefined()) BBLV = Result; else - BBLV.mergeIn(Result); + BBLV.mergeIn(Result, DL); } } } @@ -857,10 +854,10 @@ bool getValueFromFromCondition(Value *Val, ICmpInst *ICI, ConstantInt *CI = dyn_cast(ICI->getOperand(1)); if (CI && (ICI->getOperand(0) == Val || NegOffset)) { - // Calculate the range of values that would satisfy the comparison. + // Calculate the range of values that are allowed by the comparison ConstantRange CmpRange(CI->getValue()); ConstantRange TrueValues = - ConstantRange::makeICmpRegion(ICI->getPredicate(), CmpRange); + ConstantRange::makeAllowedICmpRegion(ICI->getPredicate(), CmpRange); if (NegOffset) // Apply the offset from above. TrueValues = TrueValues.subtract(NegOffset->getValue()); @@ -1104,27 +1101,27 @@ void LazyValueInfoCache::threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc, /// This lazily constructs the LazyValueInfoCache. static LazyValueInfoCache &getCache(void *&PImpl, AssumptionCache *AC, - const DataLayout *DL = nullptr, + const DataLayout *DL, DominatorTree *DT = nullptr) { - if (!PImpl) - PImpl = new LazyValueInfoCache(AC, DL, DT); + if (!PImpl) { + assert(DL && "getCache() called with a null DataLayout"); + PImpl = new LazyValueInfoCache(AC, *DL, DT); + } return *static_cast(PImpl); } bool LazyValueInfo::runOnFunction(Function &F) { AC = &getAnalysis().getAssumptionCache(F); + const DataLayout &DL = F.getParent()->getDataLayout(); DominatorTreeWrapperPass *DTWP = getAnalysisIfAvailable(); DT = DTWP ? &DTWP->getDomTree() : nullptr; - DataLayoutPass *DLP = getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; - TLI = &getAnalysis().getTLI(); if (PImpl) - getCache(PImpl, AC, DL, DT).clear(); + getCache(PImpl, AC, &DL, DT).clear(); // Fully lazy. return false; @@ -1139,15 +1136,16 @@ void LazyValueInfo::getAnalysisUsage(AnalysisUsage &AU) const { void LazyValueInfo::releaseMemory() { // If the cache was allocated, free it. if (PImpl) { - delete &getCache(PImpl, AC); + delete &getCache(PImpl, AC, nullptr); PImpl = nullptr; } } Constant *LazyValueInfo::getConstant(Value *V, BasicBlock *BB, Instruction *CxtI) { + const DataLayout &DL = BB->getModule()->getDataLayout(); LVILatticeVal Result = - getCache(PImpl, AC, DL, DT).getValueInBlock(V, BB, CxtI); + getCache(PImpl, AC, &DL, DT).getValueInBlock(V, BB, CxtI); if (Result.isConstant()) return Result.getConstant(); @@ -1164,8 +1162,9 @@ Constant *LazyValueInfo::getConstant(Value *V, BasicBlock *BB, Constant *LazyValueInfo::getConstantOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB, Instruction *CxtI) { + const DataLayout &DL = FromBB->getModule()->getDataLayout(); LVILatticeVal Result = - getCache(PImpl, AC, DL, DT).getValueOnEdge(V, FromBB, ToBB, CxtI); + getCache(PImpl, AC, &DL, DT).getValueOnEdge(V, FromBB, ToBB, CxtI); if (Result.isConstant()) return Result.getConstant(); @@ -1177,9 +1176,10 @@ Constant *LazyValueInfo::getConstantOnEdge(Value *V, BasicBlock *FromBB, return nullptr; } -static LazyValueInfo::Tristate -getPredicateResult(unsigned Pred, Constant *C, LVILatticeVal &Result, - const DataLayout *DL, TargetLibraryInfo *TLI) { +static LazyValueInfo::Tristate getPredicateResult(unsigned Pred, Constant *C, + LVILatticeVal &Result, + const DataLayout &DL, + TargetLibraryInfo *TLI) { // If we know the value is a constant, evaluate the conditional. Constant *Res = nullptr; @@ -1250,8 +1250,9 @@ LazyValueInfo::Tristate LazyValueInfo::getPredicateOnEdge(unsigned Pred, Value *V, Constant *C, BasicBlock *FromBB, BasicBlock *ToBB, Instruction *CxtI) { + const DataLayout &DL = FromBB->getModule()->getDataLayout(); LVILatticeVal Result = - getCache(PImpl, AC, DL, DT).getValueOnEdge(V, FromBB, ToBB, CxtI); + getCache(PImpl, AC, &DL, DT).getValueOnEdge(V, FromBB, ToBB, CxtI); return getPredicateResult(Pred, C, Result, DL, TLI); } @@ -1259,18 +1260,23 @@ LazyValueInfo::getPredicateOnEdge(unsigned Pred, Value *V, Constant *C, LazyValueInfo::Tristate LazyValueInfo::getPredicateAt(unsigned Pred, Value *V, Constant *C, Instruction *CxtI) { - LVILatticeVal Result = getCache(PImpl, AC, DL, DT).getValueAt(V, CxtI); + const DataLayout &DL = CxtI->getModule()->getDataLayout(); + LVILatticeVal Result = getCache(PImpl, AC, &DL, DT).getValueAt(V, CxtI); return getPredicateResult(Pred, C, Result, DL, TLI); } void LazyValueInfo::threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc, BasicBlock *NewSucc) { - if (PImpl) - getCache(PImpl, AC, DL, DT).threadEdge(PredBB, OldSucc, NewSucc); + if (PImpl) { + const DataLayout &DL = PredBB->getModule()->getDataLayout(); + getCache(PImpl, AC, &DL, DT).threadEdge(PredBB, OldSucc, NewSucc); + } } void LazyValueInfo::eraseBlock(BasicBlock *BB) { - if (PImpl) - getCache(PImpl, AC, DL, DT).eraseBlock(BB); + if (PImpl) { + const DataLayout &DL = BB->getModule()->getDataLayout(); + getCache(PImpl, AC, &DL, DT).eraseBlock(BB); + } } diff --git a/lib/Analysis/LibCallAliasAnalysis.cpp b/lib/Analysis/LibCallAliasAnalysis.cpp index 016f8c5..f6025e3 100644 --- a/lib/Analysis/LibCallAliasAnalysis.cpp +++ b/lib/Analysis/LibCallAliasAnalysis.cpp @@ -36,7 +36,11 @@ void LibCallAliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); // Does not transform code } - +bool LibCallAliasAnalysis::runOnFunction(Function &F) { + // set up super class + InitializeAliasAnalysis(this, &F.getParent()->getDataLayout()); + return false; +} /// AnalyzeLibCallDetails - Given a call to a function with the specified /// LibCallFunctionInfo, see if we can improve the mod/ref footprint of the call diff --git a/lib/Analysis/LibCallSemantics.cpp b/lib/Analysis/LibCallSemantics.cpp index cf752dd..328b186 100644 --- a/lib/Analysis/LibCallSemantics.cpp +++ b/lib/Analysis/LibCallSemantics.cpp @@ -80,18 +80,6 @@ EHPersonality llvm::classifyEHPersonality(const Value *Pers) { .Default(EHPersonality::Unknown); } -bool llvm::isAsynchronousEHPersonality(EHPersonality Pers) { - // The two SEH personality functions can catch asynch exceptions. We assume - // unknown personalities don't catch asynch exceptions. - switch (Pers) { - case EHPersonality::MSVC_X86SEH: - case EHPersonality::MSVC_Win64SEH: - return true; - default: return false; - } - llvm_unreachable("invalid enum"); -} - bool llvm::canSimplifyInvokeNoUnwind(const InvokeInst *II) { const LandingPadInst *LP = II->getLandingPadInst(); EHPersonality Personality = classifyEHPersonality(LP->getPersonalityFn()); diff --git a/lib/Analysis/Lint.cpp b/lib/Analysis/Lint.cpp index 874ed0a..65a90d7 100644 --- a/lib/Analysis/Lint.cpp +++ b/lib/Analysis/Lint.cpp @@ -59,10 +59,10 @@ using namespace llvm; namespace { namespace MemRef { - static unsigned Read = 1; - static unsigned Write = 2; - static unsigned Callee = 4; - static unsigned Branchee = 8; + static const unsigned Read = 1; + static const unsigned Write = 2; + static const unsigned Callee = 4; + static const unsigned Branchee = 8; } class Lint : public FunctionPass, public InstVisitor { @@ -98,8 +98,8 @@ namespace { void visitInsertElementInst(InsertElementInst &I); void visitUnreachableInst(UnreachableInst &I); - Value *findValue(Value *V, bool OffsetOk) const; - Value *findValueImpl(Value *V, bool OffsetOk, + Value *findValue(Value *V, const DataLayout &DL, bool OffsetOk) const; + Value *findValueImpl(Value *V, const DataLayout &DL, bool OffsetOk, SmallPtrSetImpl &Visited) const; public: @@ -107,7 +107,6 @@ namespace { AliasAnalysis *AA; AssumptionCache *AC; DominatorTree *DT; - const DataLayout *DL; TargetLibraryInfo *TLI; std::string Messages; @@ -129,27 +128,33 @@ namespace { } void print(raw_ostream &O, const Module *M) const override {} - void WriteValue(const Value *V) { - if (!V) return; - if (isa(V)) { - MessagesStr << *V << '\n'; - } else { - V->printAsOperand(MessagesStr, true, Mod); - MessagesStr << '\n'; + void WriteValues(ArrayRef Vs) { + for (const Value *V : Vs) { + if (!V) + continue; + if (isa(V)) { + MessagesStr << *V << '\n'; + } else { + V->printAsOperand(MessagesStr, true, Mod); + MessagesStr << '\n'; + } } } - // CheckFailed - A check failed, so print out the condition and the message - // that failed. This provides a nice place to put a breakpoint if you want - // to see why something is not correct. - void CheckFailed(const Twine &Message, - const Value *V1 = nullptr, const Value *V2 = nullptr, - const Value *V3 = nullptr, const Value *V4 = nullptr) { - MessagesStr << Message.str() << "\n"; - WriteValue(V1); - WriteValue(V2); - WriteValue(V3); - WriteValue(V4); + /// \brief A check failed, so printout out the condition and the message. + /// + /// This provides a nice place to put a breakpoint if you want to see why + /// something is not correct. + void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; } + + /// \brief A check failed (with values to print). + /// + /// This calls the Message-only version so that the above is easier to set + /// a breakpoint on. + template + void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) { + CheckFailed(Message); + WriteValues({V1, Vs...}); } }; } @@ -165,16 +170,8 @@ INITIALIZE_PASS_END(Lint, "lint", "Statically lint-checks LLVM IR", false, true) // Assert - We know that cond should be true, if not print an error message. -#define Assert(C, M) \ - do { if (!(C)) { CheckFailed(M); return; } } while (0) -#define Assert1(C, M, V1) \ - do { if (!(C)) { CheckFailed(M, V1); return; } } while (0) -#define Assert2(C, M, V1, V2) \ - do { if (!(C)) { CheckFailed(M, V1, V2); return; } } while (0) -#define Assert3(C, M, V1, V2, V3) \ - do { if (!(C)) { CheckFailed(M, V1, V2, V3); return; } } while (0) -#define Assert4(C, M, V1, V2, V3, V4) \ - do { if (!(C)) { CheckFailed(M, V1, V2, V3, V4); return; } } while (0) +#define Assert(C, ...) \ + do { if (!(C)) { CheckFailed(__VA_ARGS__); return; } } while (0) // Lint::run - This is the main Analysis entry point for a // function. @@ -184,8 +181,6 @@ bool Lint::runOnFunction(Function &F) { AA = &getAnalysis(); AC = &getAnalysis().getAssumptionCache(F); DT = &getAnalysis().getDomTree(); - DataLayoutPass *DLP = getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; TLI = &getAnalysis().getTLI(); visit(F); dbgs() << MessagesStr.str(); @@ -196,8 +191,8 @@ bool Lint::runOnFunction(Function &F) { void Lint::visitFunction(Function &F) { // This isn't undefined behavior, it's just a little unusual, and it's a // fairly common mistake to neglect to name a function. - Assert1(F.hasName() || F.hasLocalLinkage(), - "Unusual: Unnamed function with non-local linkage", &F); + Assert(F.hasName() || F.hasLocalLinkage(), + "Unusual: Unnamed function with non-local linkage", &F); // TODO: Check for irreducible control flow. } @@ -205,27 +200,30 @@ void Lint::visitFunction(Function &F) { void Lint::visitCallSite(CallSite CS) { Instruction &I = *CS.getInstruction(); Value *Callee = CS.getCalledValue(); + const DataLayout &DL = CS->getModule()->getDataLayout(); visitMemoryReference(I, Callee, AliasAnalysis::UnknownSize, 0, nullptr, MemRef::Callee); - if (Function *F = dyn_cast(findValue(Callee, /*OffsetOk=*/false))) { - Assert1(CS.getCallingConv() == F->getCallingConv(), - "Undefined behavior: Caller and callee calling convention differ", - &I); + if (Function *F = dyn_cast(findValue(Callee, DL, + /*OffsetOk=*/false))) { + Assert(CS.getCallingConv() == F->getCallingConv(), + "Undefined behavior: Caller and callee calling convention differ", + &I); FunctionType *FT = F->getFunctionType(); unsigned NumActualArgs = CS.arg_size(); - Assert1(FT->isVarArg() ? - FT->getNumParams() <= NumActualArgs : - FT->getNumParams() == NumActualArgs, - "Undefined behavior: Call argument count mismatches callee " - "argument count", &I); + Assert(FT->isVarArg() ? FT->getNumParams() <= NumActualArgs + : FT->getNumParams() == NumActualArgs, + "Undefined behavior: Call argument count mismatches callee " + "argument count", + &I); - Assert1(FT->getReturnType() == I.getType(), - "Undefined behavior: Call return type mismatches " - "callee return type", &I); + Assert(FT->getReturnType() == I.getType(), + "Undefined behavior: Call return type mismatches " + "callee return type", + &I); // Check argument types (in case the callee was casted) and attributes. // TODO: Verify that caller and callee attributes are compatible. @@ -235,9 +233,10 @@ void Lint::visitCallSite(CallSite CS) { Value *Actual = *AI; if (PI != PE) { Argument *Formal = PI++; - Assert1(Formal->getType() == Actual->getType(), - "Undefined behavior: Call argument type mismatches " - "callee parameter type", &I); + Assert(Formal->getType() == Actual->getType(), + "Undefined behavior: Call argument type mismatches " + "callee parameter type", + &I); // Check that noalias arguments don't alias other arguments. This is // not fully precise because we don't know the sizes of the dereferenced @@ -246,9 +245,9 @@ void Lint::visitCallSite(CallSite CS) { for (CallSite::arg_iterator BI = CS.arg_begin(); BI != AE; ++BI) if (AI != BI && (*BI)->getType()->isPointerTy()) { AliasAnalysis::AliasResult Result = AA->alias(*AI, *BI); - Assert1(Result != AliasAnalysis::MustAlias && - Result != AliasAnalysis::PartialAlias, - "Unusual: noalias argument aliases another argument", &I); + Assert(Result != AliasAnalysis::MustAlias && + Result != AliasAnalysis::PartialAlias, + "Unusual: noalias argument aliases another argument", &I); } // Check that an sret argument points to valid memory. @@ -256,8 +255,8 @@ void Lint::visitCallSite(CallSite CS) { Type *Ty = cast(Formal->getType())->getElementType(); visitMemoryReference(I, Actual, AA->getTypeStoreSize(Ty), - DL ? DL->getABITypeAlignment(Ty) : 0, - Ty, MemRef::Read | MemRef::Write); + DL.getABITypeAlignment(Ty), Ty, + MemRef::Read | MemRef::Write); } } } @@ -266,10 +265,11 @@ void Lint::visitCallSite(CallSite CS) { if (CS.isCall() && cast(CS.getInstruction())->isTailCall()) for (CallSite::arg_iterator AI = CS.arg_begin(), AE = CS.arg_end(); AI != AE; ++AI) { - Value *Obj = findValue(*AI, /*OffsetOk=*/true); - Assert1(!isa(Obj), - "Undefined behavior: Call with \"tail\" keyword references " - "alloca", &I); + Value *Obj = findValue(*AI, DL, /*OffsetOk=*/true); + Assert(!isa(Obj), + "Undefined behavior: Call with \"tail\" keyword references " + "alloca", + &I); } @@ -294,13 +294,13 @@ void Lint::visitCallSite(CallSite CS) { // overlap is not distinguished from the case where nothing is known. uint64_t Size = 0; if (const ConstantInt *Len = - dyn_cast(findValue(MCI->getLength(), - /*OffsetOk=*/false))) + dyn_cast(findValue(MCI->getLength(), DL, + /*OffsetOk=*/false))) if (Len->getValue().isIntN(32)) Size = Len->getValue().getZExtValue(); - Assert1(AA->alias(MCI->getSource(), Size, MCI->getDest(), Size) != - AliasAnalysis::MustAlias, - "Undefined behavior: memcpy source and destination overlap", &I); + Assert(AA->alias(MCI->getSource(), Size, MCI->getDest(), Size) != + AliasAnalysis::MustAlias, + "Undefined behavior: memcpy source and destination overlap", &I); break; } case Intrinsic::memmove: { @@ -324,9 +324,9 @@ void Lint::visitCallSite(CallSite CS) { } case Intrinsic::vastart: - Assert1(I.getParent()->getParent()->isVarArg(), - "Undefined behavior: va_start called in a non-varargs function", - &I); + Assert(I.getParent()->getParent()->isVarArg(), + "Undefined behavior: va_start called in a non-varargs function", + &I); visitMemoryReference(I, CS.getArgument(0), AliasAnalysis::UnknownSize, 0, nullptr, MemRef::Read | MemRef::Write); @@ -369,14 +369,13 @@ void Lint::visitInvokeInst(InvokeInst &I) { void Lint::visitReturnInst(ReturnInst &I) { Function *F = I.getParent()->getParent(); - Assert1(!F->doesNotReturn(), - "Unusual: Return statement in function with noreturn attribute", - &I); + Assert(!F->doesNotReturn(), + "Unusual: Return statement in function with noreturn attribute", &I); if (Value *V = I.getReturnValue()) { - Value *Obj = findValue(V, /*OffsetOk=*/true); - Assert1(!isa(Obj), - "Unusual: Returning alloca value", &I); + Value *Obj = + findValue(V, F->getParent()->getDataLayout(), /*OffsetOk=*/true); + Assert(!isa(Obj), "Unusual: Returning alloca value", &I); } } @@ -390,45 +389,47 @@ void Lint::visitMemoryReference(Instruction &I, if (Size == 0) return; - Value *UnderlyingObject = findValue(Ptr, /*OffsetOk=*/true); - Assert1(!isa(UnderlyingObject), - "Undefined behavior: Null pointer dereference", &I); - Assert1(!isa(UnderlyingObject), - "Undefined behavior: Undef pointer dereference", &I); - Assert1(!isa(UnderlyingObject) || - !cast(UnderlyingObject)->isAllOnesValue(), - "Unusual: All-ones pointer dereference", &I); - Assert1(!isa(UnderlyingObject) || - !cast(UnderlyingObject)->isOne(), - "Unusual: Address one pointer dereference", &I); + Value *UnderlyingObject = + findValue(Ptr, I.getModule()->getDataLayout(), /*OffsetOk=*/true); + Assert(!isa(UnderlyingObject), + "Undefined behavior: Null pointer dereference", &I); + Assert(!isa(UnderlyingObject), + "Undefined behavior: Undef pointer dereference", &I); + Assert(!isa(UnderlyingObject) || + !cast(UnderlyingObject)->isAllOnesValue(), + "Unusual: All-ones pointer dereference", &I); + Assert(!isa(UnderlyingObject) || + !cast(UnderlyingObject)->isOne(), + "Unusual: Address one pointer dereference", &I); if (Flags & MemRef::Write) { if (const GlobalVariable *GV = dyn_cast(UnderlyingObject)) - Assert1(!GV->isConstant(), - "Undefined behavior: Write to read-only memory", &I); - Assert1(!isa(UnderlyingObject) && - !isa(UnderlyingObject), - "Undefined behavior: Write to text section", &I); + Assert(!GV->isConstant(), "Undefined behavior: Write to read-only memory", + &I); + Assert(!isa(UnderlyingObject) && + !isa(UnderlyingObject), + "Undefined behavior: Write to text section", &I); } if (Flags & MemRef::Read) { - Assert1(!isa(UnderlyingObject), - "Unusual: Load from function body", &I); - Assert1(!isa(UnderlyingObject), - "Undefined behavior: Load from block address", &I); + Assert(!isa(UnderlyingObject), "Unusual: Load from function body", + &I); + Assert(!isa(UnderlyingObject), + "Undefined behavior: Load from block address", &I); } if (Flags & MemRef::Callee) { - Assert1(!isa(UnderlyingObject), - "Undefined behavior: Call to block address", &I); + Assert(!isa(UnderlyingObject), + "Undefined behavior: Call to block address", &I); } if (Flags & MemRef::Branchee) { - Assert1(!isa(UnderlyingObject) || - isa(UnderlyingObject), - "Undefined behavior: Branch to non-blockaddress", &I); + Assert(!isa(UnderlyingObject) || + isa(UnderlyingObject), + "Undefined behavior: Branch to non-blockaddress", &I); } // Check for buffer overflows and misalignment. // Only handles memory references that read/write something simple like an // alloca instruction or a global variable. + auto &DL = I.getModule()->getDataLayout(); int64_t Offset = 0; if (Value *Base = GetPointerBaseWithConstantOffset(Ptr, Offset, DL)) { // OK, so the access is to a constant offset from Ptr. Check that Ptr is @@ -439,37 +440,37 @@ void Lint::visitMemoryReference(Instruction &I, if (AllocaInst *AI = dyn_cast(Base)) { Type *ATy = AI->getAllocatedType(); - if (DL && !AI->isArrayAllocation() && ATy->isSized()) - BaseSize = DL->getTypeAllocSize(ATy); + if (!AI->isArrayAllocation() && ATy->isSized()) + BaseSize = DL.getTypeAllocSize(ATy); BaseAlign = AI->getAlignment(); - if (DL && BaseAlign == 0 && ATy->isSized()) - BaseAlign = DL->getABITypeAlignment(ATy); + if (BaseAlign == 0 && ATy->isSized()) + BaseAlign = DL.getABITypeAlignment(ATy); } else if (GlobalVariable *GV = dyn_cast(Base)) { // If the global may be defined differently in another compilation unit // then don't warn about funky memory accesses. if (GV->hasDefinitiveInitializer()) { Type *GTy = GV->getType()->getElementType(); - if (DL && GTy->isSized()) - BaseSize = DL->getTypeAllocSize(GTy); + if (GTy->isSized()) + BaseSize = DL.getTypeAllocSize(GTy); BaseAlign = GV->getAlignment(); - if (DL && BaseAlign == 0 && GTy->isSized()) - BaseAlign = DL->getABITypeAlignment(GTy); + if (BaseAlign == 0 && GTy->isSized()) + BaseAlign = DL.getABITypeAlignment(GTy); } } // Accesses from before the start or after the end of the object are not // defined. - Assert1(Size == AliasAnalysis::UnknownSize || - BaseSize == AliasAnalysis::UnknownSize || - (Offset >= 0 && Offset + Size <= BaseSize), - "Undefined behavior: Buffer overflow", &I); + Assert(Size == AliasAnalysis::UnknownSize || + BaseSize == AliasAnalysis::UnknownSize || + (Offset >= 0 && Offset + Size <= BaseSize), + "Undefined behavior: Buffer overflow", &I); // Accesses that say that the memory is more aligned than it is are not // defined. - if (DL && Align == 0 && Ty && Ty->isSized()) - Align = DL->getABITypeAlignment(Ty); - Assert1(!BaseAlign || Align <= MinAlign(BaseAlign, Offset), - "Undefined behavior: Memory reference address is misaligned", &I); + if (Align == 0 && Ty && Ty->isSized()) + Align = DL.getABITypeAlignment(Ty); + Assert(!BaseAlign || Align <= MinAlign(BaseAlign, Offset), + "Undefined behavior: Memory reference address is misaligned", &I); } } @@ -487,36 +488,35 @@ void Lint::visitStoreInst(StoreInst &I) { } void Lint::visitXor(BinaryOperator &I) { - Assert1(!isa(I.getOperand(0)) || - !isa(I.getOperand(1)), - "Undefined result: xor(undef, undef)", &I); + Assert(!isa(I.getOperand(0)) || !isa(I.getOperand(1)), + "Undefined result: xor(undef, undef)", &I); } void Lint::visitSub(BinaryOperator &I) { - Assert1(!isa(I.getOperand(0)) || - !isa(I.getOperand(1)), - "Undefined result: sub(undef, undef)", &I); + Assert(!isa(I.getOperand(0)) || !isa(I.getOperand(1)), + "Undefined result: sub(undef, undef)", &I); } void Lint::visitLShr(BinaryOperator &I) { - if (ConstantInt *CI = - dyn_cast(findValue(I.getOperand(1), /*OffsetOk=*/false))) - Assert1(CI->getValue().ult(cast(I.getType())->getBitWidth()), - "Undefined result: Shift count out of range", &I); + if (ConstantInt *CI = dyn_cast( + findValue(I.getOperand(1), I.getModule()->getDataLayout(), + /*OffsetOk=*/false))) + Assert(CI->getValue().ult(cast(I.getType())->getBitWidth()), + "Undefined result: Shift count out of range", &I); } void Lint::visitAShr(BinaryOperator &I) { - if (ConstantInt *CI = - dyn_cast(findValue(I.getOperand(1), /*OffsetOk=*/false))) - Assert1(CI->getValue().ult(cast(I.getType())->getBitWidth()), - "Undefined result: Shift count out of range", &I); + if (ConstantInt *CI = dyn_cast(findValue( + I.getOperand(1), I.getModule()->getDataLayout(), /*OffsetOk=*/false))) + Assert(CI->getValue().ult(cast(I.getType())->getBitWidth()), + "Undefined result: Shift count out of range", &I); } void Lint::visitShl(BinaryOperator &I) { - if (ConstantInt *CI = - dyn_cast(findValue(I.getOperand(1), /*OffsetOk=*/false))) - Assert1(CI->getValue().ult(cast(I.getType())->getBitWidth()), - "Undefined result: Shift count out of range", &I); + if (ConstantInt *CI = dyn_cast(findValue( + I.getOperand(1), I.getModule()->getDataLayout(), /*OffsetOk=*/false))) + Assert(CI->getValue().ult(cast(I.getType())->getBitWidth()), + "Undefined result: Shift count out of range", &I); } static bool @@ -598,9 +598,9 @@ void Lint::visitEHBeginCatch(IntrinsicInst *II) { // The begin catch must occur in a landing pad block or all paths // to it must have come from a landing pad. - Assert1(allPredsCameFromLandingPad(CatchBB, VisitedBlocks), - "llvm.eh.begincatch may be reachable without passing a landingpad", - II); + Assert(allPredsCameFromLandingPad(CatchBB, VisitedBlocks), + "llvm.eh.begincatch may be reachable without passing a landingpad", + II); // Reset the visited block list. VisitedBlocks.clear(); @@ -612,13 +612,13 @@ void Lint::visitEHBeginCatch(IntrinsicInst *II) { bool EndCatchFound = allSuccessorsReachEndCatch( CatchBB, std::next(static_cast(II)), &SecondBeginCatch, VisitedBlocks); - Assert2( + Assert( SecondBeginCatch == nullptr, "llvm.eh.begincatch may be called a second time before llvm.eh.endcatch", II, SecondBeginCatch); - Assert1(EndCatchFound, - "Some paths from llvm.eh.begincatch may not reach llvm.eh.endcatch", - II); + Assert(EndCatchFound, + "Some paths from llvm.eh.begincatch may not reach llvm.eh.endcatch", + II); } static bool allPredCameFromBeginCatch( @@ -691,17 +691,16 @@ void Lint::visitEHEndCatch(IntrinsicInst *II) { bool BeginCatchFound = allPredCameFromBeginCatch(EndCatchBB, BasicBlock::reverse_iterator(II), &SecondEndCatch, VisitedBlocks); - Assert2( + Assert( SecondEndCatch == nullptr, "llvm.eh.endcatch may be called a second time after llvm.eh.begincatch", II, SecondEndCatch); - Assert1( - BeginCatchFound, - "llvm.eh.endcatch may be reachable without passing llvm.eh.begincatch", - II); + Assert(BeginCatchFound, + "llvm.eh.endcatch may be reachable without passing llvm.eh.begincatch", + II); } -static bool isZero(Value *V, const DataLayout *DL, DominatorTree *DT, +static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC) { // Assume undef could be zero. if (isa(V)) @@ -742,30 +741,30 @@ static bool isZero(Value *V, const DataLayout *DL, DominatorTree *DT, } void Lint::visitSDiv(BinaryOperator &I) { - Assert1(!isZero(I.getOperand(1), DL, DT, AC), - "Undefined behavior: Division by zero", &I); + Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitUDiv(BinaryOperator &I) { - Assert1(!isZero(I.getOperand(1), DL, DT, AC), - "Undefined behavior: Division by zero", &I); + Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitSRem(BinaryOperator &I) { - Assert1(!isZero(I.getOperand(1), DL, DT, AC), - "Undefined behavior: Division by zero", &I); + Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitURem(BinaryOperator &I) { - Assert1(!isZero(I.getOperand(1), DL, DT, AC), - "Undefined behavior: Division by zero", &I); + Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitAllocaInst(AllocaInst &I) { if (isa(I.getArraySize())) // This isn't undefined behavior, it's just an obvious pessimization. - Assert1(&I.getParent()->getParent()->getEntryBlock() == I.getParent(), - "Pessimization: Static alloca outside of entry block", &I); + Assert(&I.getParent()->getParent()->getEntryBlock() == I.getParent(), + "Pessimization: Static alloca outside of entry block", &I); // TODO: Check for an unusual size (MSB set?) } @@ -779,32 +778,33 @@ void Lint::visitIndirectBrInst(IndirectBrInst &I) { visitMemoryReference(I, I.getAddress(), AliasAnalysis::UnknownSize, 0, nullptr, MemRef::Branchee); - Assert1(I.getNumDestinations() != 0, - "Undefined behavior: indirectbr with no destinations", &I); + Assert(I.getNumDestinations() != 0, + "Undefined behavior: indirectbr with no destinations", &I); } void Lint::visitExtractElementInst(ExtractElementInst &I) { - if (ConstantInt *CI = - dyn_cast(findValue(I.getIndexOperand(), - /*OffsetOk=*/false))) - Assert1(CI->getValue().ult(I.getVectorOperandType()->getNumElements()), - "Undefined result: extractelement index out of range", &I); + if (ConstantInt *CI = dyn_cast( + findValue(I.getIndexOperand(), I.getModule()->getDataLayout(), + /*OffsetOk=*/false))) + Assert(CI->getValue().ult(I.getVectorOperandType()->getNumElements()), + "Undefined result: extractelement index out of range", &I); } void Lint::visitInsertElementInst(InsertElementInst &I) { - if (ConstantInt *CI = - dyn_cast(findValue(I.getOperand(2), - /*OffsetOk=*/false))) - Assert1(CI->getValue().ult(I.getType()->getNumElements()), - "Undefined result: insertelement index out of range", &I); + if (ConstantInt *CI = dyn_cast( + findValue(I.getOperand(2), I.getModule()->getDataLayout(), + /*OffsetOk=*/false))) + Assert(CI->getValue().ult(I.getType()->getNumElements()), + "Undefined result: insertelement index out of range", &I); } void Lint::visitUnreachableInst(UnreachableInst &I) { // This isn't undefined behavior, it's merely suspicious. - Assert1(&I == I.getParent()->begin() || - std::prev(BasicBlock::iterator(&I))->mayHaveSideEffects(), - "Unusual: unreachable immediately preceded by instruction without " - "side effects", &I); + Assert(&I == I.getParent()->begin() || + std::prev(BasicBlock::iterator(&I))->mayHaveSideEffects(), + "Unusual: unreachable immediately preceded by instruction without " + "side effects", + &I); } /// findValue - Look through bitcasts and simple memory reference patterns @@ -814,13 +814,13 @@ void Lint::visitUnreachableInst(UnreachableInst &I) { /// Most analysis passes don't require this logic, because instcombine /// will simplify most of these kinds of things away. But it's a goal of /// this Lint pass to be useful even on non-optimized IR. -Value *Lint::findValue(Value *V, bool OffsetOk) const { +Value *Lint::findValue(Value *V, const DataLayout &DL, bool OffsetOk) const { SmallPtrSet Visited; - return findValueImpl(V, OffsetOk, Visited); + return findValueImpl(V, DL, OffsetOk, Visited); } /// findValueImpl - Implementation helper for findValue. -Value *Lint::findValueImpl(Value *V, bool OffsetOk, +Value *Lint::findValueImpl(Value *V, const DataLayout &DL, bool OffsetOk, SmallPtrSetImpl &Visited) const { // Detect self-referential values. if (!Visited.insert(V).second) @@ -841,7 +841,7 @@ Value *Lint::findValueImpl(Value *V, bool OffsetOk, break; if (Value *U = FindAvailableLoadedValue(L->getPointerOperand(), BB, BBI, 6, AA)) - return findValueImpl(U, OffsetOk, Visited); + return findValueImpl(U, DL, OffsetOk, Visited); if (BBI != BB->begin()) break; BB = BB->getUniquePredecessor(); if (!BB) break; @@ -850,40 +850,38 @@ Value *Lint::findValueImpl(Value *V, bool OffsetOk, } else if (PHINode *PN = dyn_cast(V)) { if (Value *W = PN->hasConstantValue()) if (W != V) - return findValueImpl(W, OffsetOk, Visited); + return findValueImpl(W, DL, OffsetOk, Visited); } else if (CastInst *CI = dyn_cast(V)) { if (CI->isNoopCast(DL)) - return findValueImpl(CI->getOperand(0), OffsetOk, Visited); + return findValueImpl(CI->getOperand(0), DL, OffsetOk, Visited); } else if (ExtractValueInst *Ex = dyn_cast(V)) { if (Value *W = FindInsertedValue(Ex->getAggregateOperand(), Ex->getIndices())) if (W != V) - return findValueImpl(W, OffsetOk, Visited); + return findValueImpl(W, DL, OffsetOk, Visited); } else if (ConstantExpr *CE = dyn_cast(V)) { // Same as above, but for ConstantExpr instead of Instruction. if (Instruction::isCast(CE->getOpcode())) { if (CastInst::isNoopCast(Instruction::CastOps(CE->getOpcode()), - CE->getOperand(0)->getType(), - CE->getType(), - DL ? DL->getIntPtrType(V->getType()) : - Type::getInt64Ty(V->getContext()))) - return findValueImpl(CE->getOperand(0), OffsetOk, Visited); + CE->getOperand(0)->getType(), CE->getType(), + DL.getIntPtrType(V->getType()))) + return findValueImpl(CE->getOperand(0), DL, OffsetOk, Visited); } else if (CE->getOpcode() == Instruction::ExtractValue) { ArrayRef Indices = CE->getIndices(); if (Value *W = FindInsertedValue(CE->getOperand(0), Indices)) if (W != V) - return findValueImpl(W, OffsetOk, Visited); + return findValueImpl(W, DL, OffsetOk, Visited); } } // As a last resort, try SimplifyInstruction or constant folding. if (Instruction *Inst = dyn_cast(V)) { if (Value *W = SimplifyInstruction(Inst, DL, TLI, DT, AC)) - return findValueImpl(W, OffsetOk, Visited); + return findValueImpl(W, DL, OffsetOk, Visited); } else if (ConstantExpr *CE = dyn_cast(V)) { if (Value *W = ConstantFoldConstantExpression(CE, DL, TLI)) if (W != V) - return findValueImpl(W, OffsetOk, Visited); + return findValueImpl(W, DL, OffsetOk, Visited); } return V; diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp index 5042eb9..aed3b04 100644 --- a/lib/Analysis/Loads.cpp +++ b/lib/Analysis/Loads.cpp @@ -19,6 +19,7 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" using namespace llvm; @@ -62,7 +63,8 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) { /// This uses the pointee type to determine how many bytes need to be safe to /// load from the pointer. bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, - unsigned Align, const DataLayout *DL) { + unsigned Align) { + const DataLayout &DL = ScanFrom->getModule()->getDataLayout(); int64_t ByteOffset = 0; Value *Base = V; Base = GetPointerBaseWithConstantOffset(V, ByteOffset, DL); @@ -87,19 +89,19 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, } PointerType *AddrTy = cast(V->getType()); - uint64_t LoadSize = DL ? DL->getTypeStoreSize(AddrTy->getElementType()) : 0; + uint64_t LoadSize = DL.getTypeStoreSize(AddrTy->getElementType()); // If we found a base allocated type from either an alloca or global variable, // try to see if we are definitively within the allocated region. We need to // know the size of the base type and the loaded type to do anything in this - // case, so only try this when we have the DataLayout available. - if (BaseType && BaseType->isSized() && DL) { + // case. + if (BaseType && BaseType->isSized()) { if (BaseAlign == 0) - BaseAlign = DL->getPrefTypeAlignment(BaseType); + BaseAlign = DL.getPrefTypeAlignment(BaseType); if (Align <= BaseAlign) { // Check if the load is within the bounds of the underlying object. - if (ByteOffset + LoadSize <= DL->getTypeAllocSize(BaseType) && + if (ByteOffset + LoadSize <= DL.getTypeAllocSize(BaseType) && (Align == 0 || (ByteOffset % Align) == 0)) return true; } @@ -133,16 +135,13 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, else continue; - // Handle trivial cases even w/o DataLayout or other work. + // Handle trivial cases. if (AccessedPtr == V) return true; - if (!DL) - continue; - auto *AccessedTy = cast(AccessedPtr->getType()); if (AreEquivalentAddressValues(AccessedPtr->stripPointerCasts(), V) && - LoadSize <= DL->getTypeStoreSize(AccessedTy->getElementType())) + LoadSize <= DL.getTypeStoreSize(AccessedTy->getElementType())) return true; } return false; @@ -176,13 +175,10 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, Type *AccessTy = cast(Ptr->getType())->getElementType(); - // Try to get the DataLayout for this module. This may be null, in which case - // the optimizations will be limited. - const DataLayout *DL = ScanBB->getDataLayout(); + const DataLayout &DL = ScanBB->getModule()->getDataLayout(); // Try to get the store size for the type. - uint64_t AccessSize = DL ? DL->getTypeStoreSize(AccessTy) - : AA ? AA->getTypeStoreSize(AccessTy) : 0; + uint64_t AccessSize = DL.getTypeStoreSize(AccessTy); Value *StrippedPtr = Ptr->stripPointerCasts(); diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp index 7bedd40..1818e93 100644 --- a/lib/Analysis/LoopAccessAnalysis.cpp +++ b/lib/Analysis/LoopAccessAnalysis.cpp @@ -15,11 +15,13 @@ #include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolutionExpander.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/VectorUtils.h" using namespace llvm; @@ -49,6 +51,13 @@ unsigned VectorizerParams::RuntimeMemoryCheckThreshold; /// Maximum SIMD width. const unsigned VectorizerParams::MaxVectorWidth = 64; +/// \brief We collect interesting dependences up to this threshold. +static cl::opt MaxInterestingDependence( + "max-interesting-dependences", cl::Hidden, + cl::desc("Maximum number of interesting dependences collected by " + "loop-access analysis (default = 100)"), + cl::init(100)); + bool VectorizerParams::isInterleaveForced() { return ::VectorizationInterleave.getNumOccurrences() > 0; } @@ -120,8 +129,8 @@ void LoopAccessInfo::RuntimePointerCheck::insert( AliasSetId.push_back(ASId); } -bool LoopAccessInfo::RuntimePointerCheck::needsChecking(unsigned I, - unsigned J) const { +bool LoopAccessInfo::RuntimePointerCheck::needsChecking( + unsigned I, unsigned J, const SmallVectorImpl *PtrPartition) const { // No need to check if two readonly pointers intersect. if (!IsWritePtr[I] && !IsWritePtr[J]) return false; @@ -134,11 +143,19 @@ bool LoopAccessInfo::RuntimePointerCheck::needsChecking(unsigned I, if (AliasSetId[I] != AliasSetId[J]) return false; + // If PtrPartition is set omit checks between pointers of the same partition. + // Partition number -1 means that the pointer is used in multiple partitions. + // In this case we can't omit the check. + if (PtrPartition && (*PtrPartition)[I] != -1 && + (*PtrPartition)[I] == (*PtrPartition)[J]) + return false; + return true; } -void LoopAccessInfo::RuntimePointerCheck::print(raw_ostream &OS, - unsigned Depth) const { +void LoopAccessInfo::RuntimePointerCheck::print( + raw_ostream &OS, unsigned Depth, + const SmallVectorImpl *PtrPartition) const { unsigned NumPointers = Pointers.size(); if (NumPointers == 0) return; @@ -147,10 +164,16 @@ void LoopAccessInfo::RuntimePointerCheck::print(raw_ostream &OS, unsigned N = 0; for (unsigned I = 0; I < NumPointers; ++I) for (unsigned J = I + 1; J < NumPointers; ++J) - if (needsChecking(I, J)) { + if (needsChecking(I, J, PtrPartition)) { OS.indent(Depth) << N++ << ":\n"; - OS.indent(Depth + 2) << *Pointers[I] << "\n"; - OS.indent(Depth + 2) << *Pointers[J] << "\n"; + OS.indent(Depth + 2) << *Pointers[I]; + if (PtrPartition) + OS << " (Partition: " << (*PtrPartition)[I] << ")"; + OS << "\n"; + OS.indent(Depth + 2) << *Pointers[J]; + if (PtrPartition) + OS << " (Partition: " << (*PtrPartition)[J] << ")"; + OS << "\n"; } } @@ -165,11 +188,9 @@ public: typedef PointerIntPair MemAccessInfo; typedef SmallPtrSet MemAccessInfoSet; - /// \brief Set of potential dependent memory accesses. - typedef EquivalenceClasses DepCandidates; - - AccessAnalysis(const DataLayout *Dl, AliasAnalysis *AA, DepCandidates &DA) : - DL(Dl), AST(*AA), DepCands(DA), IsRTCheckNeeded(false) {} + AccessAnalysis(const DataLayout &Dl, AliasAnalysis *AA, + MemoryDepChecker::DepCandidates &DA) + : DL(Dl), AST(*AA), DepCands(DA), IsRTCheckNeeded(false) {} /// \brief Register a load and whether it is only read from. void addLoad(AliasAnalysis::Location &Loc, bool IsReadOnly) { @@ -217,14 +238,14 @@ private: /// Set of all accesses. PtrAccessSet Accesses; + const DataLayout &DL; + /// Set of accesses that need a further dependence check. MemAccessInfoSet CheckDeps; /// Set of pointers that are read only. SmallPtrSet ReadOnlyPtr; - const DataLayout *DL; - /// An alias set tracker to partition the access set by underlying object and //intrinsic property (such as TBAA metadata). AliasSetTracker AST; @@ -232,7 +253,7 @@ private: /// Sets of potentially dependent accesses - members of one set share an /// underlying pointer. The set "CheckDeps" identfies which sets really need a /// dependence check. - DepCandidates &DepCands; + MemoryDepChecker::DepCandidates &DepCands; bool IsRTCheckNeeded; }; @@ -252,8 +273,8 @@ static bool hasComputableBounds(ScalarEvolution *SE, /// \brief Check the stride of the pointer and ensure that it does not wrap in /// the address space. -static int isStridedPtr(ScalarEvolution *SE, const DataLayout *DL, Value *Ptr, - const Loop *Lp, const ValueToValueMap &StridesMap); +static int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp, + const ValueToValueMap &StridesMap); bool AccessAnalysis::canCheckPtrAtRT( LoopAccessInfo::RuntimePointerCheck &RtCheck, unsigned &NumComparisons, @@ -289,10 +310,10 @@ bool AccessAnalysis::canCheckPtrAtRT( ++NumReadPtrChecks; if (hasComputableBounds(SE, StridesMap, Ptr) && - // When we run after a failing dependency check we have to make sure we - // don't have wrapping pointers. + // When we run after a failing dependency check we have to make sure + // we don't have wrapping pointers. (!ShouldCheckStride || - isStridedPtr(SE, DL, Ptr, TheLoop, StridesMap) == 1)) { + isStridedPtr(SE, Ptr, TheLoop, StridesMap) == 1)) { // The id of the dependence set. unsigned DepId; @@ -362,7 +383,7 @@ void AccessAnalysis::processMemAccesses() { DEBUG(dbgs() << "LAA: Processing memory accesses...\n"); DEBUG(dbgs() << " AST: "; AST.dump()); - DEBUG(dbgs() << "LAA: Accesses:\n"); + DEBUG(dbgs() << "LAA: Accesses(" << Accesses.size() << "):\n"); DEBUG({ for (auto A : Accesses) dbgs() << "\t" << *A.getPointer() << " (" << @@ -460,124 +481,6 @@ void AccessAnalysis::processMemAccesses() { } } -namespace { -/// \brief Checks memory dependences among accesses to the same underlying -/// object to determine whether there vectorization is legal or not (and at -/// which vectorization factor). -/// -/// This class works under the assumption that we already checked that memory -/// locations with different underlying pointers are "must-not alias". -/// We use the ScalarEvolution framework to symbolically evalutate access -/// functions pairs. Since we currently don't restructure the loop we can rely -/// on the program order of memory accesses to determine their safety. -/// At the moment we will only deem accesses as safe for: -/// * A negative constant distance assuming program order. -/// -/// Safe: tmp = a[i + 1]; OR a[i + 1] = x; -/// a[i] = tmp; y = a[i]; -/// -/// The latter case is safe because later checks guarantuee that there can't -/// be a cycle through a phi node (that is, we check that "x" and "y" is not -/// the same variable: a header phi can only be an induction or a reduction, a -/// reduction can't have a memory sink, an induction can't have a memory -/// source). This is important and must not be violated (or we have to -/// resort to checking for cycles through memory). -/// -/// * A positive constant distance assuming program order that is bigger -/// than the biggest memory access. -/// -/// tmp = a[i] OR b[i] = x -/// a[i+2] = tmp y = b[i+2]; -/// -/// Safe distance: 2 x sizeof(a[0]), and 2 x sizeof(b[0]), respectively. -/// -/// * Zero distances and all accesses have the same size. -/// -class MemoryDepChecker { -public: - typedef PointerIntPair MemAccessInfo; - typedef SmallPtrSet MemAccessInfoSet; - - MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L) - : SE(Se), DL(Dl), InnermostLoop(L), AccessIdx(0), - ShouldRetryWithRuntimeCheck(false) {} - - /// \brief Register the location (instructions are given increasing numbers) - /// of a write access. - void addAccess(StoreInst *SI) { - Value *Ptr = SI->getPointerOperand(); - Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx); - InstMap.push_back(SI); - ++AccessIdx; - } - - /// \brief Register the location (instructions are given increasing numbers) - /// of a write access. - void addAccess(LoadInst *LI) { - Value *Ptr = LI->getPointerOperand(); - Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx); - InstMap.push_back(LI); - ++AccessIdx; - } - - /// \brief Check whether the dependencies between the accesses are safe. - /// - /// Only checks sets with elements in \p CheckDeps. - bool areDepsSafe(AccessAnalysis::DepCandidates &AccessSets, - MemAccessInfoSet &CheckDeps, const ValueToValueMap &Strides); - - /// \brief The maximum number of bytes of a vector register we can vectorize - /// the accesses safely with. - unsigned getMaxSafeDepDistBytes() { return MaxSafeDepDistBytes; } - - /// \brief In same cases when the dependency check fails we can still - /// vectorize the loop with a dynamic array access check. - bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; } - -private: - ScalarEvolution *SE; - const DataLayout *DL; - const Loop *InnermostLoop; - - /// \brief Maps access locations (ptr, read/write) to program order. - DenseMap > Accesses; - - /// \brief Memory access instructions in program order. - SmallVector InstMap; - - /// \brief The program order index to be used for the next instruction. - unsigned AccessIdx; - - // We can access this many bytes in parallel safely. - unsigned MaxSafeDepDistBytes; - - /// \brief If we see a non-constant dependence distance we can still try to - /// vectorize this loop with runtime checks. - bool ShouldRetryWithRuntimeCheck; - - /// \brief Check whether there is a plausible dependence between the two - /// accesses. - /// - /// Access \p A must happen before \p B in program order. The two indices - /// identify the index into the program order map. - /// - /// This function checks whether there is a plausible dependence (or the - /// absence of such can't be proved) between the two accesses. If there is a - /// plausible dependence but the dependence distance is bigger than one - /// element access it records this distance in \p MaxSafeDepDistBytes (if this - /// distance is smaller than any other distance encountered so far). - /// Otherwise, this function returns true signaling a possible dependence. - bool isDependent(const MemAccessInfo &A, unsigned AIdx, - const MemAccessInfo &B, unsigned BIdx, - const ValueToValueMap &Strides); - - /// \brief Check whether the data dependence could prevent store-load - /// forwarding. - bool couldPreventStoreLoadForward(unsigned Distance, unsigned TypeByteSize); -}; - -} // end anonymous namespace - static bool isInBoundsGep(Value *Ptr) { if (GetElementPtrInst *GEP = dyn_cast(Ptr)) return GEP->isInBounds(); @@ -585,8 +488,8 @@ static bool isInBoundsGep(Value *Ptr) { } /// \brief Check whether the access through \p Ptr has a constant stride. -static int isStridedPtr(ScalarEvolution *SE, const DataLayout *DL, Value *Ptr, - const Loop *Lp, const ValueToValueMap &StridesMap) { +static int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp, + const ValueToValueMap &StridesMap) { const Type *Ty = Ptr->getType(); assert(Ty->isPointerTy() && "Unexpected non-ptr"); @@ -640,7 +543,8 @@ static int isStridedPtr(ScalarEvolution *SE, const DataLayout *DL, Value *Ptr, return 0; } - int64_t Size = DL->getTypeAllocSize(PtrTy->getElementType()); + auto &DL = Lp->getHeader()->getModule()->getDataLayout(); + int64_t Size = DL.getTypeAllocSize(PtrTy->getElementType()); const APInt &APStepVal = C->getValue()->getValue(); // Huge step value - give up. @@ -665,6 +569,54 @@ static int isStridedPtr(ScalarEvolution *SE, const DataLayout *DL, Value *Ptr, return Stride; } +bool MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) { + switch (Type) { + case NoDep: + case Forward: + case BackwardVectorizable: + return true; + + case Unknown: + case ForwardButPreventsForwarding: + case Backward: + case BackwardVectorizableButPreventsForwarding: + return false; + } + llvm_unreachable("unexpected DepType!"); +} + +bool MemoryDepChecker::Dependence::isInterestingDependence(DepType Type) { + switch (Type) { + case NoDep: + case Forward: + return false; + + case BackwardVectorizable: + case Unknown: + case ForwardButPreventsForwarding: + case Backward: + case BackwardVectorizableButPreventsForwarding: + return true; + } + llvm_unreachable("unexpected DepType!"); +} + +bool MemoryDepChecker::Dependence::isPossiblyBackward() const { + switch (Type) { + case NoDep: + case Forward: + case ForwardButPreventsForwarding: + return false; + + case Unknown: + case BackwardVectorizable: + case Backward: + case BackwardVectorizableButPreventsForwarding: + return true; + } + llvm_unreachable("unexpected DepType!"); +} + bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance, unsigned TypeByteSize) { // If loads occur at a distance that is not a multiple of a feasible vector @@ -704,9 +656,10 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance, return false; } -bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, - const MemAccessInfo &B, unsigned BIdx, - const ValueToValueMap &Strides) { +MemoryDepChecker::Dependence::DepType +MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, + const MemAccessInfo &B, unsigned BIdx, + const ValueToValueMap &Strides) { assert (AIdx < BIdx && "Must pass arguments in program order"); Value *APtr = A.getPointer(); @@ -716,18 +669,18 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, // Two reads are independent. if (!AIsWrite && !BIsWrite) - return false; + return Dependence::NoDep; // We cannot check pointers in different address spaces. if (APtr->getType()->getPointerAddressSpace() != BPtr->getType()->getPointerAddressSpace()) - return true; + return Dependence::Unknown; const SCEV *AScev = replaceSymbolicStrideSCEV(SE, Strides, APtr); const SCEV *BScev = replaceSymbolicStrideSCEV(SE, Strides, BPtr); - int StrideAPtr = isStridedPtr(SE, DL, APtr, InnermostLoop, Strides); - int StrideBPtr = isStridedPtr(SE, DL, BPtr, InnermostLoop, Strides); + int StrideAPtr = isStridedPtr(SE, APtr, InnermostLoop, Strides); + int StrideBPtr = isStridedPtr(SE, BPtr, InnermostLoop, Strides); const SCEV *Src = AScev; const SCEV *Sink = BScev; @@ -756,19 +709,20 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, // the address space. if (!StrideAPtr || !StrideBPtr || StrideAPtr != StrideBPtr){ DEBUG(dbgs() << "Non-consecutive pointer access\n"); - return true; + return Dependence::Unknown; } const SCEVConstant *C = dyn_cast(Dist); if (!C) { DEBUG(dbgs() << "LAA: Dependence because of non-constant distance\n"); ShouldRetryWithRuntimeCheck = true; - return true; + return Dependence::Unknown; } Type *ATy = APtr->getType()->getPointerElementType(); Type *BTy = BPtr->getType()->getPointerElementType(); - unsigned TypeByteSize = DL->getTypeAllocSize(ATy); + auto &DL = InnermostLoop->getHeader()->getModule()->getDataLayout(); + unsigned TypeByteSize = DL.getTypeAllocSize(ATy); // Negative distances are not plausible dependencies. const APInt &Val = C->getValue()->getValue(); @@ -777,19 +731,19 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, if (IsTrueDataDependence && (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) || ATy != BTy)) - return true; + return Dependence::ForwardButPreventsForwarding; DEBUG(dbgs() << "LAA: Dependence is negative: NoDep\n"); - return false; + return Dependence::Forward; } // Write to the same location with the same size. // Could be improved to assert type sizes are the same (i32 == float, etc). if (Val == 0) { if (ATy == BTy) - return false; + return Dependence::NoDep; DEBUG(dbgs() << "LAA: Zero dependence difference but different types\n"); - return true; + return Dependence::Unknown; } assert(Val.isStrictlyPositive() && "Expect a positive value"); @@ -797,7 +751,7 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, if (ATy != BTy) { DEBUG(dbgs() << "LAA: ReadWrite-Write positive dependency with different types\n"); - return true; + return Dependence::Unknown; } unsigned Distance = (unsigned) Val.getZExtValue(); @@ -816,7 +770,7 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, Distance < TypeByteSize * ForcedUnroll * ForcedFactor) { DEBUG(dbgs() << "LAA: Failure because of Positive distance " << Val.getSExtValue() << '\n'); - return true; + return Dependence::Backward; } // Positive distance bigger than max vectorization factor. @@ -826,15 +780,15 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, bool IsTrueDataDependence = (!AIsWrite && BIsWrite); if (IsTrueDataDependence && couldPreventStoreLoadForward(Distance, TypeByteSize)) - return true; + return Dependence::BackwardVectorizableButPreventsForwarding; DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue() << " with max VF = " << MaxSafeDepDistBytes / TypeByteSize << '\n'); - return false; + return Dependence::BackwardVectorizable; } -bool MemoryDepChecker::areDepsSafe(AccessAnalysis::DepCandidates &AccessSets, +bool MemoryDepChecker::areDepsSafe(DepCandidates &AccessSets, MemAccessInfoSet &CheckDeps, const ValueToValueMap &Strides) { @@ -860,9 +814,33 @@ bool MemoryDepChecker::areDepsSafe(AccessAnalysis::DepCandidates &AccessSets, I1E = Accesses[*AI].end(); I1 != I1E; ++I1) for (std::vector::iterator I2 = Accesses[*OI].begin(), I2E = Accesses[*OI].end(); I2 != I2E; ++I2) { - if (*I1 < *I2 && isDependent(*AI, *I1, *OI, *I2, Strides)) - return false; - if (*I2 < *I1 && isDependent(*OI, *I2, *AI, *I1, Strides)) + auto A = std::make_pair(&*AI, *I1); + auto B = std::make_pair(&*OI, *I2); + + assert(*I1 != *I2); + if (*I1 > *I2) + std::swap(A, B); + + Dependence::DepType Type = + isDependent(*A.first, A.second, *B.first, B.second, Strides); + SafeForVectorization &= Dependence::isSafeForVectorization(Type); + + // Gather dependences unless we accumulated MaxInterestingDependence + // dependences. In that case return as soon as we find the first + // unsafe dependence. This puts a limit on this quadratic + // algorithm. + if (RecordInterestingDependences) { + if (Dependence::isInterestingDependence(Type)) + InterestingDependences.push_back( + Dependence(A.second, B.second, Type)); + + if (InterestingDependences.size() >= MaxInterestingDependence) { + RecordInterestingDependences = false; + InterestingDependences.clear(); + DEBUG(dbgs() << "Too many dependences, stopped recording\n"); + } + } + if (!RecordInterestingDependences && !SafeForVectorization) return false; } ++OI; @@ -870,7 +848,34 @@ bool MemoryDepChecker::areDepsSafe(AccessAnalysis::DepCandidates &AccessSets, AI++; } } - return true; + + DEBUG(dbgs() << "Total Interesting Dependences: " + << InterestingDependences.size() << "\n"); + return SafeForVectorization; +} + +SmallVector +MemoryDepChecker::getInstructionsForAccess(Value *Ptr, bool isWrite) const { + MemAccessInfo Access(Ptr, isWrite); + auto &IndexVector = Accesses.find(Access)->second; + + SmallVector Insts; + std::transform(IndexVector.begin(), IndexVector.end(), + std::back_inserter(Insts), + [&](unsigned Idx) { return this->InstMap[Idx]; }); + return Insts; +} + +const char *MemoryDepChecker::Dependence::DepName[] = { + "NoDep", "Unknown", "Forward", "ForwardButPreventsForwarding", "Backward", + "BackwardVectorizable", "BackwardVectorizableButPreventsForwarding"}; + +void MemoryDepChecker::Dependence::print( + raw_ostream &OS, unsigned Depth, + const SmallVectorImpl &Instrs) const { + OS.indent(Depth) << DepName[Type] << ":\n"; + OS.indent(Depth + 2) << *Instrs[Source] << " -> \n"; + OS.indent(Depth + 2) << *Instrs[Destination] << "\n"; } bool LoopAccessInfo::canAnalyzeLoop() { @@ -939,7 +944,6 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { PtrRtCheck.Need = false; const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel(); - MemoryDepChecker DepChecker(SE, DL, TheLoop); // For each block. for (Loop::block_iterator bb = TheLoop->block_begin(), @@ -960,6 +964,12 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { if (Call && getIntrinsicIDForCall(Call, TLI)) continue; + // If the function has an explicit vectorized counterpart, we can safely + // assume that it can be vectorized. + if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() && + TLI->isFunctionVectorizable(Call->getCalledFunction()->getName())) + continue; + LoadInst *Ld = dyn_cast(it); if (!Ld || (!Ld->isSimple() && !IsAnnotatedParallel)) { emitAnalysis(LoopAccessReport(Ld) @@ -1008,8 +1018,9 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { return; } - AccessAnalysis::DepCandidates DependentAccesses; - AccessAnalysis Accesses(DL, AA, DependentAccesses); + MemoryDepChecker::DepCandidates DependentAccesses; + AccessAnalysis Accesses(TheLoop->getHeader()->getModule()->getDataLayout(), + AA, DependentAccesses); // Holds the analyzed pointers. We don't want to call GetUnderlyingObjects // multiple times on the same object. If the ptr is accessed twice, once @@ -1068,8 +1079,7 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { // read a few words, modify, and write a few words, and some of the // words may be written to the same address. bool IsReadOnlyPtr = false; - if (Seen.insert(Ptr).second || - !isStridedPtr(SE, DL, Ptr, TheLoop, Strides)) { + if (Seen.insert(Ptr).second || !isStridedPtr(SE, Ptr, TheLoop, Strides)) { ++NumReads; IsReadOnlyPtr = true; } @@ -1099,7 +1109,6 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { // Find pointers with computable bounds. We are going to use this information // to place a runtime bound check. - unsigned NumComparisons = 0; bool CanDoRT = false; if (NeedRTCheck) CanDoRT = Accesses.canCheckPtrAtRT(PtrRtCheck, NumComparisons, SE, TheLoop, @@ -1113,18 +1122,10 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { if (NumComparisons == 0 && NeedRTCheck) NeedRTCheck = false; - // Check that we did not collect too many pointers or found an unsizeable - // pointer. - if (!CanDoRT || NumComparisons > RuntimeMemoryCheckThreshold) { - PtrRtCheck.reset(); - CanDoRT = false; - } - - if (CanDoRT) { + // Check that we found the bounds for the pointer. + if (CanDoRT) DEBUG(dbgs() << "LAA: We can perform a memory runtime check if needed.\n"); - } - - if (NeedRTCheck && !CanDoRT) { + else if (NeedRTCheck) { emitAnalysis(LoopAccessReport() << "cannot identify array bounds"); DEBUG(dbgs() << "LAA: We can't vectorize because we can't find " << "the array bounds.\n"); @@ -1154,17 +1155,10 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { CanDoRT = Accesses.canCheckPtrAtRT(PtrRtCheck, NumComparisons, SE, TheLoop, Strides, true); - // Check that we did not collect too many pointers or found an unsizeable - // pointer. - if (!CanDoRT || NumComparisons > RuntimeMemoryCheckThreshold) { - if (!CanDoRT && NumComparisons > 0) - emitAnalysis(LoopAccessReport() - << "cannot check memory dependencies at runtime"); - else - emitAnalysis(LoopAccessReport() - << NumComparisons << " exceeds limit of " - << RuntimeMemoryCheckThreshold - << " dependent memory operations checked at runtime"); + // Check that we found the bounds for the pointer. + if (!CanDoRT && NumComparisons > 0) { + emitAnalysis(LoopAccessReport() + << "cannot check memory dependencies at runtime"); DEBUG(dbgs() << "LAA: Can't vectorize with memory checks\n"); PtrRtCheck.reset(); CanVecMem = false; @@ -1175,12 +1169,15 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { } } - if (!CanVecMem) + if (CanVecMem) + DEBUG(dbgs() << "LAA: No unsafe dependent memory operations in loop. We" + << (NeedRTCheck ? "" : " don't") + << " need a runtime memory check.\n"); + else { emitAnalysis(LoopAccessReport() << "unsafe dependent memory operations in loop"); - - DEBUG(dbgs() << "LAA: We" << (NeedRTCheck ? "" : " don't") << - " need a runtime memory check.\n"); + DEBUG(dbgs() << "LAA: unsafe dependent memory operations in loop\n"); + } } bool LoopAccessInfo::blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, @@ -1212,8 +1209,8 @@ static Instruction *getFirstInst(Instruction *FirstInst, Value *V, return nullptr; } -std::pair -LoopAccessInfo::addRuntimeCheck(Instruction *Loc) const { +std::pair LoopAccessInfo::addRuntimeCheck( + Instruction *Loc, const SmallVectorImpl *PtrPartition) const { Instruction *tnullptr = nullptr; if (!PtrRtCheck.Need) return std::pair(tnullptr, tnullptr); @@ -1223,7 +1220,7 @@ LoopAccessInfo::addRuntimeCheck(Instruction *Loc) const { SmallVector , 2> Ends; LLVMContext &Ctx = Loc->getContext(); - SCEVExpander Exp(*SE, "induction"); + SCEVExpander Exp(*SE, DL, "induction"); Instruction *FirstInst = nullptr; for (unsigned i = 0; i < NumPointers; ++i) { @@ -1254,7 +1251,7 @@ LoopAccessInfo::addRuntimeCheck(Instruction *Loc) const { Value *MemoryRuntimeCheck = nullptr; for (unsigned i = 0; i < NumPointers; ++i) { for (unsigned j = i+1; j < NumPointers; ++j) { - if (!PtrRtCheck.needsChecking(i, j)) + if (!PtrRtCheck.needsChecking(i, j, PtrPartition)) continue; unsigned AS0 = Starts[i]->getType()->getPointerAddressSpace(); @@ -1298,12 +1295,13 @@ LoopAccessInfo::addRuntimeCheck(Instruction *Loc) const { } LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI, AliasAnalysis *AA, DominatorTree *DT, const ValueToValueMap &Strides) - : TheLoop(L), SE(SE), DL(DL), TLI(TLI), AA(AA), DT(DT), NumLoads(0), - NumStores(0), MaxSafeDepDistBytes(-1U), CanVecMem(false) { + : DepChecker(SE, L), NumComparisons(0), TheLoop(L), SE(SE), DL(DL), + TLI(TLI), AA(AA), DT(DT), NumLoads(0), NumStores(0), + MaxSafeDepDistBytes(-1U), CanVecMem(false) { if (canAnalyzeLoop()) analyzeLoop(Strides); } @@ -1319,7 +1317,14 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const { if (Report) OS.indent(Depth) << "Report: " << Report->str() << "\n"; - // FIXME: Print unsafe dependences + if (auto *InterestingDependences = DepChecker.getInterestingDependences()) { + OS.indent(Depth) << "Interesting Dependences:\n"; + for (auto &Dep : *InterestingDependences) { + Dep.print(OS, Depth + 2, DepChecker.getMemoryInstructions()); + OS << "\n"; + } + } else + OS.indent(Depth) << "Too many interesting dependences, not recorded\n"; // List the pair of accesses need run-time checks to prove independence. PtrRtCheck.print(OS, Depth); @@ -1336,6 +1341,7 @@ LoopAccessAnalysis::getInfo(Loop *L, const ValueToValueMap &Strides) { #endif if (!LAI) { + const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); LAI = llvm::make_unique(L, SE, DL, TLI, AA, DT, Strides); #ifndef NDEBUG LAI->NumSymbolicStrides = Strides.size(); @@ -1360,7 +1366,6 @@ void LoopAccessAnalysis::print(raw_ostream &OS, const Module *M) const { bool LoopAccessAnalysis::runOnFunction(Function &F) { SE = &getAnalysis(); - DL = F.getParent()->getDataLayout(); auto *TLIP = getAnalysisIfAvailable(); TLI = TLIP ? &TLIP->getTLI() : nullptr; AA = &getAnalysis(); diff --git a/lib/Analysis/LoopInfo.cpp b/lib/Analysis/LoopInfo.cpp index 95f6eb0..6462b06 100644 --- a/lib/Analysis/LoopInfo.cpp +++ b/lib/Analysis/LoopInfo.cpp @@ -29,6 +29,7 @@ #include "llvm/IR/PassManager.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include using namespace llvm; diff --git a/lib/Analysis/LoopPass.cpp b/lib/Analysis/LoopPass.cpp index a99c949..e9fcf02 100644 --- a/lib/Analysis/LoopPass.cpp +++ b/lib/Analysis/LoopPass.cpp @@ -18,6 +18,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; #define DEBUG_TYPE "loop-pass-manager" diff --git a/lib/Analysis/MemDerefPrinter.cpp b/lib/Analysis/MemDerefPrinter.cpp index 531d75e..6119a3d 100644 --- a/lib/Analysis/MemDerefPrinter.cpp +++ b/lib/Analysis/MemDerefPrinter.cpp @@ -14,6 +14,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -27,7 +28,6 @@ namespace { initializeMemDerefPrinterPass(*PassRegistry::getPassRegistry()); } void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); AU.setPreservesAll(); } bool runOnFunction(Function &F) override; @@ -41,7 +41,6 @@ namespace { char MemDerefPrinter::ID = 0; INITIALIZE_PASS_BEGIN(MemDerefPrinter, "print-memderefs", "Memory Dereferenciblity of pointers in function", false, true) -INITIALIZE_PASS_DEPENDENCY(DataLayoutPass) INITIALIZE_PASS_END(MemDerefPrinter, "print-memderefs", "Memory Dereferenciblity of pointers in function", false, true) @@ -50,7 +49,7 @@ FunctionPass *llvm::createMemDerefPrinter() { } bool MemDerefPrinter::runOnFunction(Function &F) { - const DataLayout *DL = &getAnalysis().getDataLayout(); + const DataLayout &DL = F.getParent()->getDataLayout(); for (auto &I: inst_range(F)) { if (LoadInst *LI = dyn_cast(&I)) { Value *PO = LI->getPointerOperand(); diff --git a/lib/Analysis/MemoryBuiltins.cpp b/lib/Analysis/MemoryBuiltins.cpp index 6108af3..8ddac8f 100644 --- a/lib/Analysis/MemoryBuiltins.cpp +++ b/lib/Analysis/MemoryBuiltins.cpp @@ -206,7 +206,7 @@ const CallInst *llvm::extractMallocCall(const Value *I, return isMallocLikeFn(I, TLI) ? dyn_cast(I) : nullptr; } -static Value *computeArraySize(const CallInst *CI, const DataLayout *DL, +static Value *computeArraySize(const CallInst *CI, const DataLayout &DL, const TargetLibraryInfo *TLI, bool LookThroughSExt = false) { if (!CI) @@ -214,12 +214,12 @@ static Value *computeArraySize(const CallInst *CI, const DataLayout *DL, // The size of the malloc's result type must be known to determine array size. Type *T = getMallocAllocatedType(CI, TLI); - if (!T || !T->isSized() || !DL) + if (!T || !T->isSized()) return nullptr; - unsigned ElementSize = DL->getTypeAllocSize(T); + unsigned ElementSize = DL.getTypeAllocSize(T); if (StructType *ST = dyn_cast(T)) - ElementSize = DL->getStructLayout(ST)->getSizeInBytes(); + ElementSize = DL.getStructLayout(ST)->getSizeInBytes(); // If malloc call's arg can be determined to be a multiple of ElementSize, // return the multiple. Otherwise, return NULL. @@ -232,23 +232,6 @@ static Value *computeArraySize(const CallInst *CI, const DataLayout *DL, return nullptr; } -/// isArrayMalloc - Returns the corresponding CallInst if the instruction -/// is a call to malloc whose array size can be determined and the array size -/// is not constant 1. Otherwise, return NULL. -const CallInst *llvm::isArrayMalloc(const Value *I, - const DataLayout *DL, - const TargetLibraryInfo *TLI) { - const CallInst *CI = extractMallocCall(I, TLI); - Value *ArraySize = computeArraySize(CI, DL, TLI); - - if (ConstantInt *ConstSize = dyn_cast_or_null(ArraySize)) - if (ConstSize->isOne()) - return CI; - - // CI is a non-array malloc or we can't figure out that it is an array malloc. - return nullptr; -} - /// getMallocType - Returns the PointerType resulting from the malloc call. /// The PointerType depends on the number of bitcast uses of the malloc call: /// 0: PointerType is the calls' return type. @@ -297,7 +280,7 @@ Type *llvm::getMallocAllocatedType(const CallInst *CI, /// then return that multiple. For non-array mallocs, the multiple is /// constant 1. Otherwise, return NULL for mallocs whose array size cannot be /// determined. -Value *llvm::getMallocArraySize(CallInst *CI, const DataLayout *DL, +Value *llvm::getMallocArraySize(CallInst *CI, const DataLayout &DL, const TargetLibraryInfo *TLI, bool LookThroughSExt) { assert(isMallocLikeFn(CI, TLI) && "getMallocArraySize and not malloc call"); @@ -367,11 +350,8 @@ const CallInst *llvm::isFreeCall(const Value *I, const TargetLibraryInfo *TLI) { /// object size in Size if successful, and false otherwise. /// If RoundToAlign is true, then Size is rounded up to the aligment of allocas, /// byval arguments, and global variables. -bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout *DL, +bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL, const TargetLibraryInfo *TLI, bool RoundToAlign) { - if (!DL) - return false; - ObjectSizeOffsetVisitor Visitor(DL, TLI, Ptr->getContext(), RoundToAlign); SizeOffsetType Data = Visitor.compute(const_cast(Ptr)); if (!Visitor.bothKnown(Data)) @@ -399,17 +379,17 @@ APInt ObjectSizeOffsetVisitor::align(APInt Size, uint64_t Align) { return Size; } -ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const DataLayout *DL, +ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, bool RoundToAlign) -: DL(DL), TLI(TLI), RoundToAlign(RoundToAlign) { + : DL(DL), TLI(TLI), RoundToAlign(RoundToAlign) { // Pointer size must be rechecked for each object visited since it could have // a different address space. } SizeOffsetType ObjectSizeOffsetVisitor::compute(Value *V) { - IntTyBits = DL->getPointerTypeSizeInBits(V->getType()); + IntTyBits = DL.getPointerTypeSizeInBits(V->getType()); Zero = APInt::getNullValue(IntTyBits); V = V->stripPointerCasts(); @@ -449,7 +429,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) { if (!I.getAllocatedType()->isSized()) return unknown(); - APInt Size(IntTyBits, DL->getTypeAllocSize(I.getAllocatedType())); + APInt Size(IntTyBits, DL.getTypeAllocSize(I.getAllocatedType())); if (!I.isArrayAllocation()) return std::make_pair(align(Size, I.getAlignment()), Zero); @@ -468,7 +448,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitArgument(Argument &A) { return unknown(); } PointerType *PT = cast(A.getType()); - APInt Size(IntTyBits, DL->getTypeAllocSize(PT->getElementType())); + APInt Size(IntTyBits, DL.getTypeAllocSize(PT->getElementType())); return std::make_pair(align(Size, A.getParamAlignment()), Zero); } @@ -541,7 +521,7 @@ ObjectSizeOffsetVisitor::visitExtractValueInst(ExtractValueInst&) { SizeOffsetType ObjectSizeOffsetVisitor::visitGEPOperator(GEPOperator &GEP) { SizeOffsetType PtrData = compute(GEP.getPointerOperand()); APInt Offset(IntTyBits, 0); - if (!bothKnown(PtrData) || !GEP.accumulateConstantOffset(*DL, Offset)) + if (!bothKnown(PtrData) || !GEP.accumulateConstantOffset(DL, Offset)) return unknown(); return std::make_pair(PtrData.first, PtrData.second + Offset); @@ -557,7 +537,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitGlobalVariable(GlobalVariable &GV){ if (!GV.hasDefinitiveInitializer()) return unknown(); - APInt Size(IntTyBits, DL->getTypeAllocSize(GV.getType()->getElementType())); + APInt Size(IntTyBits, DL.getTypeAllocSize(GV.getType()->getElementType())); return std::make_pair(align(Size, GV.getAlignment()), Zero); } @@ -593,19 +573,18 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitInstruction(Instruction &I) { return unknown(); } -ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator(const DataLayout *DL, - const TargetLibraryInfo *TLI, - LLVMContext &Context, - bool RoundToAlign) -: DL(DL), TLI(TLI), Context(Context), Builder(Context, TargetFolder(DL)), - RoundToAlign(RoundToAlign) { +ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator( + const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, + bool RoundToAlign) + : DL(DL), TLI(TLI), Context(Context), Builder(Context, TargetFolder(DL)), + RoundToAlign(RoundToAlign) { // IntTy and Zero must be set for each compute() since the address space may // be different for later objects. } SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute(Value *V) { // XXX - Are vectors of pointers possible here? - IntTy = cast(DL->getIntPtrType(V->getType())); + IntTy = cast(DL.getIntPtrType(V->getType())); Zero = ConstantInt::get(IntTy, 0); SizeOffsetEvalType Result = compute_(V); @@ -687,7 +666,7 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitAllocaInst(AllocaInst &I) { assert(I.isArrayAllocation()); Value *ArraySize = I.getArraySize(); Value *Size = ConstantInt::get(ArraySize->getType(), - DL->getTypeAllocSize(I.getAllocatedType())); + DL.getTypeAllocSize(I.getAllocatedType())); Size = Builder.CreateMul(Size, ArraySize); return std::make_pair(Size, Zero); } @@ -739,7 +718,7 @@ ObjectSizeOffsetEvaluator::visitGEPOperator(GEPOperator &GEP) { if (!bothKnown(PtrData)) return unknown(); - Value *Offset = EmitGEPOffset(&Builder, *DL, &GEP, /*NoAssumptions=*/true); + Value *Offset = EmitGEPOffset(&Builder, DL, &GEP, /*NoAssumptions=*/true); Offset = Builder.CreateAdd(PtrData.second, Offset); return std::make_pair(PtrData.first, Offset); } diff --git a/lib/Analysis/MemoryDependenceAnalysis.cpp b/lib/Analysis/MemoryDependenceAnalysis.cpp index 6d38863..716e3e6 100644 --- a/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/lib/Analysis/MemoryDependenceAnalysis.cpp @@ -93,8 +93,6 @@ void MemoryDependenceAnalysis::getAnalysisUsage(AnalysisUsage &AU) const { bool MemoryDependenceAnalysis::runOnFunction(Function &F) { AA = &getAnalysis(); AC = &getAnalysis().getAssumptionCache(F); - DataLayoutPass *DLP = getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; DominatorTreeWrapperPass *DTWP = getAnalysisIfAvailable(); DT = DTWP ? &DTWP->getDomTree() : nullptr; @@ -263,22 +261,17 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall, /// /// MemLocBase, MemLocOffset are lazily computed here the first time the /// base/offs of memloc is needed. -static bool -isLoadLoadClobberIfExtendedToFullWidth(const AliasAnalysis::Location &MemLoc, - const Value *&MemLocBase, - int64_t &MemLocOffs, - const LoadInst *LI, - const DataLayout *DL) { - // If we have no target data, we can't do this. - if (!DL) return false; +static bool isLoadLoadClobberIfExtendedToFullWidth( + const AliasAnalysis::Location &MemLoc, const Value *&MemLocBase, + int64_t &MemLocOffs, const LoadInst *LI) { + const DataLayout &DL = LI->getModule()->getDataLayout(); // If we haven't already computed the base/offset of MemLoc, do so now. if (!MemLocBase) MemLocBase = GetPointerBaseWithConstantOffset(MemLoc.Ptr, MemLocOffs, DL); - unsigned Size = MemoryDependenceAnalysis:: - getLoadLoadClobberFullWidthSize(MemLocBase, MemLocOffs, MemLoc.Size, - LI, *DL); + unsigned Size = MemoryDependenceAnalysis::getLoadLoadClobberFullWidthSize( + MemLocBase, MemLocOffs, MemLoc.Size, LI); return Size != 0; } @@ -289,10 +282,9 @@ isLoadLoadClobberIfExtendedToFullWidth(const AliasAnalysis::Location &MemLoc, /// 2) safe for the target, and 3) would provide the specified memory /// location value, then this function returns the size in bytes of the /// load width to use. If not, this returns zero. -unsigned MemoryDependenceAnalysis:: -getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs, - unsigned MemLocSize, const LoadInst *LI, - const DataLayout &DL) { +unsigned MemoryDependenceAnalysis::getLoadLoadClobberFullWidthSize( + const Value *MemLocBase, int64_t MemLocOffs, unsigned MemLocSize, + const LoadInst *LI) { // We can only extend simple integer loads. if (!isa(LI->getType()) || !LI->isSimple()) return 0; @@ -301,10 +293,12 @@ getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs, if (LI->getParent()->getParent()->hasFnAttribute(Attribute::SanitizeThread)) return 0; + const DataLayout &DL = LI->getModule()->getDataLayout(); + // Get the base of this load. int64_t LIOffs = 0; const Value *LIBase = - GetPointerBaseWithConstantOffset(LI->getPointerOperand(), LIOffs, &DL); + GetPointerBaseWithConstantOffset(LI->getPointerOperand(), LIOffs, DL); // If the two pointers are not based on the same pointer, we can't tell that // they are related. @@ -413,14 +407,19 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, // by every program that can detect any optimisation of that kind: either // it is racy (undefined) or there is a release followed by an acquire // between the pair of accesses under consideration. - bool HasSeenAcquire = false; + // If the load is invariant, we "know" that it doesn't alias *any* write. We + // do want to respect mustalias results since defs are useful for value + // forwarding, but any mayalias write can be assumed to be noalias. + // Arguably, this logic should be pushed inside AliasAnalysis itself. if (isLoad && QueryInst) { LoadInst *LI = dyn_cast(QueryInst); if (LI && LI->getMetadata(LLVMContext::MD_invariant_load) != nullptr) isInvariantLoad = true; } + const DataLayout &DL = BB->getModule()->getDataLayout(); + // Walk backwards through the basic block, looking for dependencies. while (ScanIt != BB->begin()) { Instruction *Inst = --ScanIt; @@ -472,12 +471,12 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, // Atomic loads have complications involved. // A Monotonic (or higher) load is OK if the query inst is itself not atomic. - // An Acquire (or higher) load sets the HasSeenAcquire flag, so that any - // release store will know to return getClobber. // FIXME: This is overly conservative. if (LI->isAtomic() && LI->getOrdering() > Unordered) { if (!QueryInst) return MemDepResult::getClobber(LI); + if (LI->getOrdering() != Monotonic) + return MemDepResult::getClobber(LI); if (auto *QueryLI = dyn_cast(QueryInst)) { if (!QueryLI->isSimple()) return MemDepResult::getClobber(LI); @@ -487,9 +486,6 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, } else if (QueryInst->mayReadOrWriteMemory()) { return MemDepResult::getClobber(LI); } - - if (isAtLeastAcquire(LI->getOrdering())) - HasSeenAcquire = true; } AliasAnalysis::Location LoadLoc = AA->getLocation(LI); @@ -505,12 +501,12 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, // location is 1 byte at P+1). If so, return it as a load/load // clobber result, allowing the client to decide to widen the load if // it wants to. - if (IntegerType *ITy = dyn_cast(LI->getType())) - if (LI->getAlignment()*8 > ITy->getPrimitiveSizeInBits() && + if (IntegerType *ITy = dyn_cast(LI->getType())) { + if (LI->getAlignment() * 8 > ITy->getPrimitiveSizeInBits() && isLoadLoadClobberIfExtendedToFullWidth(MemLoc, MemLocBase, - MemLocOffset, LI, DL)) + MemLocOffset, LI)) return MemDepResult::getClobber(Inst); - + } continue; } @@ -549,12 +545,12 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, if (StoreInst *SI = dyn_cast(Inst)) { // Atomic stores have complications involved. // A Monotonic store is OK if the query inst is itself not atomic. - // A Release (or higher) store further requires that no acquire load - // has been seen. // FIXME: This is overly conservative. if (!SI->isUnordered()) { if (!QueryInst) return MemDepResult::getClobber(SI); + if (SI->getOrdering() != Monotonic) + return MemDepResult::getClobber(SI); if (auto *QueryLI = dyn_cast(QueryInst)) { if (!QueryLI->isSimple()) return MemDepResult::getClobber(SI); @@ -564,9 +560,6 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, } else if (QueryInst->mayReadOrWriteMemory()) { return MemDepResult::getClobber(SI); } - - if (HasSeenAcquire && isAtLeastRelease(SI->getOrdering())) - return MemDepResult::getClobber(SI); } // FIXME: this is overly conservative. @@ -612,6 +605,8 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, if (AccessPtr == Inst || AA->isMustAlias(Inst, AccessPtr)) return MemDepResult::getDef(Inst); + if (isInvariantLoad) + continue; // Be conservative if the accessed pointer may alias the allocation. if (AA->alias(Inst, AccessPtr) != AliasAnalysis::NoAlias) return MemDepResult::getClobber(Inst); @@ -622,6 +617,9 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad, continue; } + if (isInvariantLoad) + continue; + // See if this instruction (e.g. a call or vaarg) mod/ref's the pointer. AliasAnalysis::ModRefResult MR = AA->getModRefInfo(Inst, MemLoc); // If necessary, perform additional analysis. @@ -923,8 +921,7 @@ getNonLocalPointerDependency(Instruction *QueryInst, const_cast(Loc.Ptr))); return; } - - + const DataLayout &DL = FromBB->getModule()->getDataLayout(); PHITransAddr Address(const_cast(Loc.Ptr), DL, AC); // This is the set of blocks we've inspected, and the pointer we consider in diff --git a/lib/Analysis/ModuleDebugInfoPrinter.cpp b/lib/Analysis/ModuleDebugInfoPrinter.cpp index f645558..cbc4700 100644 --- a/lib/Analysis/ModuleDebugInfoPrinter.cpp +++ b/lib/Analysis/ModuleDebugInfoPrinter.cpp @@ -55,28 +55,74 @@ bool ModuleDebugInfoPrinter::runOnModule(Module &M) { return false; } +static void printFile(raw_ostream &O, StringRef Filename, StringRef Directory, + unsigned Line = 0) { + if (Filename.empty()) + return; + + O << " from "; + if (!Directory.empty()) + O << Directory << "/"; + O << Filename; + if (Line) + O << ":" << Line; +} + void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const { + // Printing the nodes directly isn't particularly helpful (since they + // reference other nodes that won't be printed, particularly for the + // filenames), so just print a few useful things. for (DICompileUnit CU : Finder.compile_units()) { - O << "Compile Unit: "; - CU.print(O); + O << "Compile unit: "; + if (const char *Lang = LanguageString(CU.getLanguage())) + O << Lang; + else + O << "unknown-language(" << CU.getLanguage() << ")"; + printFile(O, CU.getFilename(), CU.getDirectory()); O << '\n'; } for (DISubprogram S : Finder.subprograms()) { - O << "Subprogram: "; - S.print(O); + O << "Subprogram: " << S.getName(); + printFile(O, S.getFilename(), S.getDirectory(), S.getLineNumber()); + if (!S.getLinkageName().empty()) + O << " ('" << S.getLinkageName() << "')"; O << '\n'; } for (DIGlobalVariable GV : Finder.global_variables()) { - O << "GlobalVariable: "; - GV.print(O); + O << "Global variable: " << GV.getName(); + printFile(O, GV.getFilename(), GV.getDirectory(), GV.getLineNumber()); + if (!GV.getLinkageName().empty()) + O << " ('" << GV.getLinkageName() << "')"; O << '\n'; } for (DIType T : Finder.types()) { - O << "Type: "; - T.print(O); + O << "Type:"; + if (!T.getName().empty()) + O << ' ' << T.getName(); + printFile(O, T.getFilename(), T.getDirectory(), T.getLineNumber()); + if (T.isBasicType()) { + DIBasicType BT(T.get()); + O << " "; + if (const char *Encoding = + dwarf::AttributeEncodingString(BT.getEncoding())) + O << Encoding; + else + O << "unknown-encoding(" << BT.getEncoding() << ')'; + } else { + O << ' '; + if (const char *Tag = dwarf::TagString(T.getTag())) + O << Tag; + else + O << "unknown-tag(" << T.getTag() << ")"; + } + if (T.isCompositeType()) { + DICompositeType CT(T.get()); + if (auto *S = CT.getIdentifier()) + O << " (identifier: '" << S->getString() << "')"; + } O << '\n'; } } diff --git a/lib/Analysis/NoAliasAnalysis.cpp b/lib/Analysis/NoAliasAnalysis.cpp index c214d3c..203e1da 100644 --- a/lib/Analysis/NoAliasAnalysis.cpp +++ b/lib/Analysis/NoAliasAnalysis.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/Pass.h" using namespace llvm; @@ -33,11 +34,11 @@ namespace { void getAnalysisUsage(AnalysisUsage &AU) const override {} - void initializePass() override { + bool doInitialization(Module &M) override { // Note: NoAA does not call InitializeAliasAnalysis because it's // special and does not support chaining. - DataLayoutPass *DLP = getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; + DL = &M.getDataLayout(); + return true; } AliasResult alias(const Location &LocA, const Location &LocB) override { diff --git a/lib/Analysis/PHITransAddr.cpp b/lib/Analysis/PHITransAddr.cpp index a534418..177684f 100644 --- a/lib/Analysis/PHITransAddr.cpp +++ b/lib/Analysis/PHITransAddr.cpp @@ -404,10 +404,9 @@ InsertPHITranslatedSubExpr(Value *InVal, BasicBlock *CurBB, GEPOps.push_back(OpVal); } - GetElementPtrInst *Result = - GetElementPtrInst::Create(GEPOps[0], makeArrayRef(GEPOps).slice(1), - InVal->getName()+".phi.trans.insert", - PredBB->getTerminator()); + GetElementPtrInst *Result = GetElementPtrInst::Create( + GEP->getSourceElementType(), GEPOps[0], makeArrayRef(GEPOps).slice(1), + InVal->getName() + ".phi.trans.insert", PredBB->getTerminator()); Result->setIsInBounds(GEP->isInBounds()); NewInsts.push_back(Result); return Result; diff --git a/lib/Analysis/RegionPass.cpp b/lib/Analysis/RegionPass.cpp index 6fa7b2e..cd1e944 100644 --- a/lib/Analysis/RegionPass.cpp +++ b/lib/Analysis/RegionPass.cpp @@ -17,6 +17,7 @@ #include "llvm/Analysis/RegionIterator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; #define DEBUG_TYPE "regionpassmgr" @@ -83,9 +84,11 @@ bool RGPassManager::runOnFunction(Function &F) { for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { RegionPass *P = (RegionPass*)getContainedPass(Index); - dumpPassInfo(P, EXECUTION_MSG, ON_REGION_MSG, - CurrentRegion->getNameStr()); - dumpRequiredSet(P); + if (isPassDebuggingExecutionsOrMore()) { + dumpPassInfo(P, EXECUTION_MSG, ON_REGION_MSG, + CurrentRegion->getNameStr()); + dumpRequiredSet(P); + } initializeAnalysisImpl(P); @@ -96,11 +99,13 @@ bool RGPassManager::runOnFunction(Function &F) { Changed |= P->runOnRegion(CurrentRegion, *this); } - if (Changed) - dumpPassInfo(P, MODIFICATION_MSG, ON_REGION_MSG, - skipThisRegion ? "" : - CurrentRegion->getNameStr()); - dumpPreservedSet(P); + if (isPassDebuggingExecutionsOrMore()) { + if (Changed) + dumpPassInfo(P, MODIFICATION_MSG, ON_REGION_MSG, + skipThisRegion ? "" : + CurrentRegion->getNameStr()); + dumpPreservedSet(P); + } if (!skipThisRegion) { // Manually check that this region is still healthy. This is done @@ -120,8 +125,8 @@ bool RGPassManager::runOnFunction(Function &F) { removeNotPreservedAnalysis(P); recordAvailableAnalysis(P); removeDeadPasses(P, - skipThisRegion ? "" : - CurrentRegion->getNameStr(), + (!isPassDebuggingExecutionsOrMore() || skipThisRegion) ? + "" : CurrentRegion->getNameStr(), ON_REGION_MSG); if (skipThisRegion) diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index 9e4eb11..4e713fb 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -1102,13 +1102,14 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, return getTruncateOrZeroExtend(SZ->getOperand(), Ty); // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can - // eliminate all the truncates. + // eliminate all the truncates, or we replace other casts with truncates. if (const SCEVAddExpr *SA = dyn_cast(Op)) { SmallVector Operands; bool hasTrunc = false; for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); - hasTrunc = isa(S); + if (!isa(SA->getOperand(i))) + hasTrunc = isa(S); Operands.push_back(S); } if (!hasTrunc) @@ -1117,13 +1118,14 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, } // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can - // eliminate all the truncates. + // eliminate all the truncates, or we replace other casts with truncates. if (const SCEVMulExpr *SM = dyn_cast(Op)) { SmallVector Operands; bool hasTrunc = false; for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); - hasTrunc = isa(S); + if (!isa(SM->getOperand(i))) + hasTrunc = isa(S); Operands.push_back(S); } if (!hasTrunc) @@ -1325,6 +1327,85 @@ static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, (SE->*GetExtendExpr)(PreStart, Ty)); } +// Try to prove away overflow by looking at "nearby" add recurrences. A +// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it +// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`. +// +// Formally: +// +// {S,+,X} == {S-T,+,X} + T +// => Ext({S,+,X}) == Ext({S-T,+,X} + T) +// +// If ({S-T,+,X} + T) does not overflow ... (1) +// +// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T) +// +// If {S-T,+,X} does not overflow ... (2) +// +// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T) +// == {Ext(S-T)+Ext(T),+,Ext(X)} +// +// If (S-T)+T does not overflow ... (3) +// +// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)} +// == {Ext(S),+,Ext(X)} == LHS +// +// Thus, if (1), (2) and (3) are true for some T, then +// Ext({S,+,X}) == {Ext(S),+,Ext(X)} +// +// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T) +// does not overflow" restricted to the 0th iteration. Therefore we only need +// to check for (1) and (2). +// +// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T +// is `Delta` (defined below). +// +template +bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start, + const SCEV *Step, + const Loop *L) { + auto WrapType = ExtendOpTraits::WrapType; + + // We restrict `Start` to a constant to prevent SCEV from spending too much + // time here. It is correct (but more expensive) to continue with a + // non-constant `Start` and do a general SCEV subtraction to compute + // `PreStart` below. + // + const SCEVConstant *StartC = dyn_cast(Start); + if (!StartC) + return false; + + APInt StartAI = StartC->getValue()->getValue(); + + for (unsigned Delta : {-2, -1, 1, 2}) { + const SCEV *PreStart = getConstant(StartAI - Delta); + + // Give up if we don't already have the add recurrence we need because + // actually constructing an add recurrence is relatively expensive. + const SCEVAddRecExpr *PreAR = [&]() { + FoldingSetNodeID ID; + ID.AddInteger(scAddRecExpr); + ID.AddPointer(PreStart); + ID.AddPointer(Step); + ID.AddPointer(L); + void *IP = nullptr; + return static_cast( + this->UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); + }(); + + if (PreAR && PreAR->getNoWrapFlags(WrapType)) { // proves (2) + const SCEV *DeltaS = getConstant(StartC->getType(), Delta); + ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; + const SCEV *Limit = ExtendOpTraits::getOverflowLimitForStep( + DeltaS, &Pred, this); + if (Limit && isKnownPredicate(Pred, PreAR, Limit)) // proves (1) + return true; + } + } + + return false; +} + const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty) { assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && @@ -1473,6 +1554,13 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, } } } + + if (proveNoWrapByVaryingStart(Start, Step, L)) { + const_cast(AR)->setNoWrapFlags(SCEV::FlagNUW); + return getAddRecExpr( + getExtendAddRecStart(AR, Ty, this), + getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); + } } // The cast wasn't folded; create an explicit cast node. @@ -1664,6 +1752,13 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, return getAddExpr(Start, getSignExtendExpr(NewAR, Ty)); } } + + if (proveNoWrapByVaryingStart(Start, Step, L)) { + const_cast(AR)->setNoWrapFlags(SCEV::FlagNSW); + return getAddRecExpr( + getExtendAddRecStart(AR, Ty, this), + getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); + } } // The cast wasn't folded; create an explicit cast node. @@ -3037,39 +3132,23 @@ const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, } const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) { - // If we have DataLayout, we can bypass creating a target-independent + // We can bypass creating a target-independent // constant expression and then folding it back into a ConstantInt. // This is just a compile-time optimization. - if (DL) - return getConstant(IntTy, DL->getTypeAllocSize(AllocTy)); - - Constant *C = ConstantExpr::getSizeOf(AllocTy); - if (ConstantExpr *CE = dyn_cast(C)) - if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI)) - C = Folded; - Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy)); - assert(Ty == IntTy && "Effective SCEV type doesn't match"); - return getTruncateOrZeroExtend(getSCEV(C), Ty); + return getConstant(IntTy, + F->getParent()->getDataLayout().getTypeAllocSize(AllocTy)); } const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo) { - // If we have DataLayout, we can bypass creating a target-independent + // We can bypass creating a target-independent // constant expression and then folding it back into a ConstantInt. // This is just a compile-time optimization. - if (DL) { - return getConstant(IntTy, - DL->getStructLayout(STy)->getElementOffset(FieldNo)); - } - - Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo); - if (ConstantExpr *CE = dyn_cast(C)) - if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI)) - C = Folded; - - Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy)); - return getTruncateOrZeroExtend(getSCEV(C), Ty); + return getConstant( + IntTy, + F->getParent()->getDataLayout().getStructLayout(STy)->getElementOffset( + FieldNo)); } const SCEV *ScalarEvolution::getUnknown(Value *V) { @@ -3111,19 +3190,7 @@ bool ScalarEvolution::isSCEVable(Type *Ty) const { /// for which isSCEVable must return true. uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const { assert(isSCEVable(Ty) && "Type is not SCEVable!"); - - // If we have a DataLayout, use it! - if (DL) - return DL->getTypeSizeInBits(Ty); - - // Integer types have fixed sizes. - if (Ty->isIntegerTy()) - return Ty->getPrimitiveSizeInBits(); - - // The only other support type is pointer. Without DataLayout, conservatively - // assume pointers are 64-bit. - assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!"); - return 64; + return F->getParent()->getDataLayout().getTypeSizeInBits(Ty); } /// getEffectiveSCEVType - Return a type with the same bitwidth as @@ -3139,12 +3206,7 @@ Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const { // The only other support type is pointer. assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"); - - if (DL) - return DL->getIntPtrType(Ty); - - // Without DataLayout, conservatively assume pointers are 64-bit. - return Type::getInt64Ty(getContext()); + return F->getParent()->getDataLayout().getIntPtrType(Ty); } const SCEV *ScalarEvolution::getCouldNotCompute() { @@ -3531,10 +3593,12 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { // If the increment doesn't overflow, then neither the addrec nor // the post-increment will overflow. if (const AddOperator *OBO = dyn_cast(BEValueV)) { - if (OBO->hasNoUnsignedWrap()) - Flags = setFlags(Flags, SCEV::FlagNUW); - if (OBO->hasNoSignedWrap()) - Flags = setFlags(Flags, SCEV::FlagNSW); + if (OBO->getOperand(0) == PN) { + if (OBO->hasNoUnsignedWrap()) + Flags = setFlags(Flags, SCEV::FlagNUW); + if (OBO->hasNoSignedWrap()) + Flags = setFlags(Flags, SCEV::FlagNSW); + } } else if (GEPOperator *GEP = dyn_cast(BEValueV)) { // If the increment is an inbounds GEP, then we know the address // space cannot be wrapped around. We cannot make any guarantee @@ -3542,7 +3606,7 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { // unsigned but we may have a negative index from the base // pointer. We can guarantee that no unsigned wrap occurs if the // indices form a positive value. - if (GEP->isInBounds()) { + if (GEP->isInBounds() && GEP->getOperand(0) == PN) { Flags = setFlags(Flags, SCEV::FlagNW); const SCEV *Ptr = getSCEV(GEP->getPointerOperand()); @@ -3608,7 +3672,8 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { // PHI's incoming blocks are in a different loop, in which case doing so // risks breaking LCSSA form. Instcombine would normally zap these, but // it doesn't have DominatorTree information, so it may miss cases. - if (Value *V = SimplifyInstruction(PN, DL, TLI, DT, AC)) + if (Value *V = + SimplifyInstruction(PN, F->getParent()->getDataLayout(), TLI, DT, AC)) if (LI->replacementPreservesLCSSAForm(PN, V)) return getSCEV(V); @@ -3740,7 +3805,8 @@ ScalarEvolution::GetMinTrailingZeros(const SCEV *S) { // For a SCEVUnknown, ask ValueTracking. unsigned BitWidth = getTypeSizeInBits(U->getType()); APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); - computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AC, nullptr, DT); + computeKnownBits(U->getValue(), Zeros, Ones, + F->getParent()->getDataLayout(), 0, AC, nullptr, DT); return Zeros.countTrailingOnes(); } @@ -3775,79 +3841,93 @@ static Optional GetRangeFromMetadata(Value *V) { return None; } -/// getUnsignedRange - Determine the unsigned range for a particular SCEV. +/// getRange - Determine the range for a particular SCEV. If SignHint is +/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges +/// with a "cleaner" unsigned (resp. signed) representation. /// ConstantRange -ScalarEvolution::getUnsignedRange(const SCEV *S) { +ScalarEvolution::getRange(const SCEV *S, + ScalarEvolution::RangeSignHint SignHint) { + DenseMap &Cache = + SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges + : SignedRanges; + // See if we've computed this range already. - DenseMap::iterator I = UnsignedRanges.find(S); - if (I != UnsignedRanges.end()) + DenseMap::iterator I = Cache.find(S); + if (I != Cache.end()) return I->second; if (const SCEVConstant *C = dyn_cast(S)) - return setUnsignedRange(C, ConstantRange(C->getValue()->getValue())); + return setRange(C, SignHint, ConstantRange(C->getValue()->getValue())); unsigned BitWidth = getTypeSizeInBits(S->getType()); ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); - // If the value has known zeros, the maximum unsigned value will have those - // known zeros as well. + // If the value has known zeros, the maximum value will have those known zeros + // as well. uint32_t TZ = GetMinTrailingZeros(S); - if (TZ != 0) - ConservativeResult = - ConstantRange(APInt::getMinValue(BitWidth), - APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); + if (TZ != 0) { + if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) + ConservativeResult = + ConstantRange(APInt::getMinValue(BitWidth), + APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); + else + ConservativeResult = ConstantRange( + APInt::getSignedMinValue(BitWidth), + APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); + } if (const SCEVAddExpr *Add = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(Add->getOperand(0)); + ConstantRange X = getRange(Add->getOperand(0), SignHint); for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) - X = X.add(getUnsignedRange(Add->getOperand(i))); - return setUnsignedRange(Add, ConservativeResult.intersectWith(X)); + X = X.add(getRange(Add->getOperand(i), SignHint)); + return setRange(Add, SignHint, ConservativeResult.intersectWith(X)); } if (const SCEVMulExpr *Mul = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(Mul->getOperand(0)); + ConstantRange X = getRange(Mul->getOperand(0), SignHint); for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) - X = X.multiply(getUnsignedRange(Mul->getOperand(i))); - return setUnsignedRange(Mul, ConservativeResult.intersectWith(X)); + X = X.multiply(getRange(Mul->getOperand(i), SignHint)); + return setRange(Mul, SignHint, ConservativeResult.intersectWith(X)); } if (const SCEVSMaxExpr *SMax = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(SMax->getOperand(0)); + ConstantRange X = getRange(SMax->getOperand(0), SignHint); for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) - X = X.smax(getUnsignedRange(SMax->getOperand(i))); - return setUnsignedRange(SMax, ConservativeResult.intersectWith(X)); + X = X.smax(getRange(SMax->getOperand(i), SignHint)); + return setRange(SMax, SignHint, ConservativeResult.intersectWith(X)); } if (const SCEVUMaxExpr *UMax = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(UMax->getOperand(0)); + ConstantRange X = getRange(UMax->getOperand(0), SignHint); for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) - X = X.umax(getUnsignedRange(UMax->getOperand(i))); - return setUnsignedRange(UMax, ConservativeResult.intersectWith(X)); + X = X.umax(getRange(UMax->getOperand(i), SignHint)); + return setRange(UMax, SignHint, ConservativeResult.intersectWith(X)); } if (const SCEVUDivExpr *UDiv = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(UDiv->getLHS()); - ConstantRange Y = getUnsignedRange(UDiv->getRHS()); - return setUnsignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); + ConstantRange X = getRange(UDiv->getLHS(), SignHint); + ConstantRange Y = getRange(UDiv->getRHS(), SignHint); + return setRange(UDiv, SignHint, + ConservativeResult.intersectWith(X.udiv(Y))); } if (const SCEVZeroExtendExpr *ZExt = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(ZExt->getOperand()); - return setUnsignedRange(ZExt, - ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); + ConstantRange X = getRange(ZExt->getOperand(), SignHint); + return setRange(ZExt, SignHint, + ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); } if (const SCEVSignExtendExpr *SExt = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(SExt->getOperand()); - return setUnsignedRange(SExt, - ConservativeResult.intersectWith(X.signExtend(BitWidth))); + ConstantRange X = getRange(SExt->getOperand(), SignHint); + return setRange(SExt, SignHint, + ConservativeResult.intersectWith(X.signExtend(BitWidth))); } if (const SCEVTruncateExpr *Trunc = dyn_cast(S)) { - ConstantRange X = getUnsignedRange(Trunc->getOperand()); - return setUnsignedRange(Trunc, - ConservativeResult.intersectWith(X.truncate(BitWidth))); + ConstantRange X = getRange(Trunc->getOperand(), SignHint); + return setRange(Trunc, SignHint, + ConservativeResult.intersectWith(X.truncate(BitWidth))); } if (const SCEVAddRecExpr *AddRec = dyn_cast(S)) { @@ -3860,143 +3940,6 @@ ScalarEvolution::getUnsignedRange(const SCEV *S) { ConservativeResult.intersectWith( ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0))); - // TODO: non-affine addrec - if (AddRec->isAffine()) { - Type *Ty = AddRec->getType(); - const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); - if (!isa(MaxBECount) && - getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { - MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); - - const SCEV *Start = AddRec->getStart(); - const SCEV *Step = AddRec->getStepRecurrence(*this); - - ConstantRange StartRange = getUnsignedRange(Start); - ConstantRange StepRange = getSignedRange(Step); - ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); - ConstantRange EndRange = - StartRange.add(MaxBECountRange.multiply(StepRange)); - - // Check for overflow. This must be done with ConstantRange arithmetic - // because we could be called from within the ScalarEvolution overflow - // checking code. - ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1); - ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); - ConstantRange ExtMaxBECountRange = - MaxBECountRange.zextOrTrunc(BitWidth*2+1); - ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1); - if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != - ExtEndRange) - return setUnsignedRange(AddRec, ConservativeResult); - - APInt Min = APIntOps::umin(StartRange.getUnsignedMin(), - EndRange.getUnsignedMin()); - APInt Max = APIntOps::umax(StartRange.getUnsignedMax(), - EndRange.getUnsignedMax()); - if (Min.isMinValue() && Max.isMaxValue()) - return setUnsignedRange(AddRec, ConservativeResult); - return setUnsignedRange(AddRec, - ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); - } - } - - return setUnsignedRange(AddRec, ConservativeResult); - } - - if (const SCEVUnknown *U = dyn_cast(S)) { - // Check if the IR explicitly contains !range metadata. - Optional MDRange = GetRangeFromMetadata(U->getValue()); - if (MDRange.hasValue()) - ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); - - // For a SCEVUnknown, ask ValueTracking. - APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); - computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AC, nullptr, DT); - if (Ones == ~Zeros + 1) - return setUnsignedRange(U, ConservativeResult); - return setUnsignedRange(U, - ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1))); - } - - return setUnsignedRange(S, ConservativeResult); -} - -/// getSignedRange - Determine the signed range for a particular SCEV. -/// -ConstantRange -ScalarEvolution::getSignedRange(const SCEV *S) { - // See if we've computed this range already. - DenseMap::iterator I = SignedRanges.find(S); - if (I != SignedRanges.end()) - return I->second; - - if (const SCEVConstant *C = dyn_cast(S)) - return setSignedRange(C, ConstantRange(C->getValue()->getValue())); - - unsigned BitWidth = getTypeSizeInBits(S->getType()); - ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); - - // If the value has known zeros, the maximum signed value will have those - // known zeros as well. - uint32_t TZ = GetMinTrailingZeros(S); - if (TZ != 0) - ConservativeResult = - ConstantRange(APInt::getSignedMinValue(BitWidth), - APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); - - if (const SCEVAddExpr *Add = dyn_cast(S)) { - ConstantRange X = getSignedRange(Add->getOperand(0)); - for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) - X = X.add(getSignedRange(Add->getOperand(i))); - return setSignedRange(Add, ConservativeResult.intersectWith(X)); - } - - if (const SCEVMulExpr *Mul = dyn_cast(S)) { - ConstantRange X = getSignedRange(Mul->getOperand(0)); - for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) - X = X.multiply(getSignedRange(Mul->getOperand(i))); - return setSignedRange(Mul, ConservativeResult.intersectWith(X)); - } - - if (const SCEVSMaxExpr *SMax = dyn_cast(S)) { - ConstantRange X = getSignedRange(SMax->getOperand(0)); - for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) - X = X.smax(getSignedRange(SMax->getOperand(i))); - return setSignedRange(SMax, ConservativeResult.intersectWith(X)); - } - - if (const SCEVUMaxExpr *UMax = dyn_cast(S)) { - ConstantRange X = getSignedRange(UMax->getOperand(0)); - for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) - X = X.umax(getSignedRange(UMax->getOperand(i))); - return setSignedRange(UMax, ConservativeResult.intersectWith(X)); - } - - if (const SCEVUDivExpr *UDiv = dyn_cast(S)) { - ConstantRange X = getSignedRange(UDiv->getLHS()); - ConstantRange Y = getSignedRange(UDiv->getRHS()); - return setSignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); - } - - if (const SCEVZeroExtendExpr *ZExt = dyn_cast(S)) { - ConstantRange X = getSignedRange(ZExt->getOperand()); - return setSignedRange(ZExt, - ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); - } - - if (const SCEVSignExtendExpr *SExt = dyn_cast(S)) { - ConstantRange X = getSignedRange(SExt->getOperand()); - return setSignedRange(SExt, - ConservativeResult.intersectWith(X.signExtend(BitWidth))); - } - - if (const SCEVTruncateExpr *Trunc = dyn_cast(S)) { - ConstantRange X = getSignedRange(Trunc->getOperand()); - return setSignedRange(Trunc, - ConservativeResult.intersectWith(X.truncate(BitWidth))); - } - - if (const SCEVAddRecExpr *AddRec = dyn_cast(S)) { // If there's no signed wrap, and all the operands have the same sign or // zero, the value won't ever change sign. if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) { @@ -4022,41 +3965,66 @@ ScalarEvolution::getSignedRange(const SCEV *S) { const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); if (!isa(MaxBECount) && getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { + + // Check for overflow. This must be done with ConstantRange arithmetic + // because we could be called from within the ScalarEvolution overflow + // checking code. + MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); + ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); + ConstantRange ZExtMaxBECountRange = + MaxBECountRange.zextOrTrunc(BitWidth * 2 + 1); const SCEV *Start = AddRec->getStart(); const SCEV *Step = AddRec->getStepRecurrence(*this); + ConstantRange StepSRange = getSignedRange(Step); + ConstantRange SExtStepSRange = StepSRange.sextOrTrunc(BitWidth * 2 + 1); + + ConstantRange StartURange = getUnsignedRange(Start); + ConstantRange EndURange = + StartURange.add(MaxBECountRange.multiply(StepSRange)); + + // Check for unsigned overflow. + ConstantRange ZExtStartURange = + StartURange.zextOrTrunc(BitWidth * 2 + 1); + ConstantRange ZExtEndURange = EndURange.zextOrTrunc(BitWidth * 2 + 1); + if (ZExtStartURange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) == + ZExtEndURange) { + APInt Min = APIntOps::umin(StartURange.getUnsignedMin(), + EndURange.getUnsignedMin()); + APInt Max = APIntOps::umax(StartURange.getUnsignedMax(), + EndURange.getUnsignedMax()); + bool IsFullRange = Min.isMinValue() && Max.isMaxValue(); + if (!IsFullRange) + ConservativeResult = + ConservativeResult.intersectWith(ConstantRange(Min, Max + 1)); + } - ConstantRange StartRange = getSignedRange(Start); - ConstantRange StepRange = getSignedRange(Step); - ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); - ConstantRange EndRange = - StartRange.add(MaxBECountRange.multiply(StepRange)); - - // Check for overflow. This must be done with ConstantRange arithmetic - // because we could be called from within the ScalarEvolution overflow - // checking code. - ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1); - ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); - ConstantRange ExtMaxBECountRange = - MaxBECountRange.zextOrTrunc(BitWidth*2+1); - ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1); - if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != - ExtEndRange) - return setSignedRange(AddRec, ConservativeResult); - - APInt Min = APIntOps::smin(StartRange.getSignedMin(), - EndRange.getSignedMin()); - APInt Max = APIntOps::smax(StartRange.getSignedMax(), - EndRange.getSignedMax()); - if (Min.isMinSignedValue() && Max.isMaxSignedValue()) - return setSignedRange(AddRec, ConservativeResult); - return setSignedRange(AddRec, - ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); + ConstantRange StartSRange = getSignedRange(Start); + ConstantRange EndSRange = + StartSRange.add(MaxBECountRange.multiply(StepSRange)); + + // Check for signed overflow. This must be done with ConstantRange + // arithmetic because we could be called from within the ScalarEvolution + // overflow checking code. + ConstantRange SExtStartSRange = + StartSRange.sextOrTrunc(BitWidth * 2 + 1); + ConstantRange SExtEndSRange = EndSRange.sextOrTrunc(BitWidth * 2 + 1); + if (SExtStartSRange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) == + SExtEndSRange) { + APInt Min = APIntOps::smin(StartSRange.getSignedMin(), + EndSRange.getSignedMin()); + APInt Max = APIntOps::smax(StartSRange.getSignedMax(), + EndSRange.getSignedMax()); + bool IsFullRange = Min.isMinSignedValue() && Max.isMaxSignedValue(); + if (!IsFullRange) + ConservativeResult = + ConservativeResult.intersectWith(ConstantRange(Min, Max + 1)); + } } } - return setSignedRange(AddRec, ConservativeResult); + return setRange(AddRec, SignHint, ConservativeResult); } if (const SCEVUnknown *U = dyn_cast(S)) { @@ -4065,18 +4033,31 @@ ScalarEvolution::getSignedRange(const SCEV *S) { if (MDRange.hasValue()) ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); - // For a SCEVUnknown, ask ValueTracking. - if (!U->getValue()->getType()->isIntegerTy() && !DL) - return setSignedRange(U, ConservativeResult); - unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, AC, nullptr, DT); - if (NS <= 1) - return setSignedRange(U, ConservativeResult); - return setSignedRange(U, ConservativeResult.intersectWith( - ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), - APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1))); + // Split here to avoid paying the compile-time cost of calling both + // computeKnownBits and ComputeNumSignBits. This restriction can be lifted + // if needed. + const DataLayout &DL = F->getParent()->getDataLayout(); + if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) { + // For a SCEVUnknown, ask ValueTracking. + APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); + computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AC, nullptr, DT); + if (Ones != ~Zeros + 1) + ConservativeResult = + ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1)); + } else { + assert(SignHint == ScalarEvolution::HINT_RANGE_SIGNED && + "generalize as needed!"); + unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, AC, nullptr, DT); + if (NS > 1) + ConservativeResult = ConservativeResult.intersectWith( + ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), + APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1)); + } + + return setRange(U, SignHint, ConservativeResult); } - return setSignedRange(S, ConservativeResult); + return setRange(S, SignHint, ConservativeResult); } /// createSCEV - We know that there is no SCEV for the specified value. @@ -4175,8 +4156,8 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { unsigned TZ = A.countTrailingZeros(); unsigned BitWidth = A.getBitWidth(); APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); - computeKnownBits(U->getOperand(0), KnownZero, KnownOne, DL, 0, AC, - nullptr, DT); + computeKnownBits(U->getOperand(0), KnownZero, KnownOne, + F->getParent()->getDataLayout(), 0, AC, nullptr, DT); APInt EffectiveMask = APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ); @@ -5327,12 +5308,9 @@ static bool canConstantEvolve(Instruction *I, const Loop *L) { if (!L->contains(I)) return false; if (isa(I)) { - if (L->getHeader() == I->getParent()) - return true; - else - // We don't currently keep track of the control flow needed to evaluate - // PHIs, so we cannot handle PHIs inside of loops. - return false; + // We don't currently keep track of the control flow needed to evaluate + // PHIs, so we cannot handle PHIs inside of loops. + return L->getHeader() == I->getParent(); } // If we won't be able to constant fold this expression even if the operands @@ -5403,7 +5381,7 @@ static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) { /// reason, return null. static Constant *EvaluateExpression(Value *V, const Loop *L, DenseMap &Vals, - const DataLayout *DL, + const DataLayout &DL, const TargetLibraryInfo *TLI) { // Convenient constant check, but redundant for recursive calls. if (Constant *C = dyn_cast(V)) return C; @@ -5492,6 +5470,7 @@ ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, unsigned NumIterations = BEs.getZExtValue(); // must be in range unsigned IterationNum = 0; + const DataLayout &DL = F->getParent()->getDataLayout(); for (; ; ++IterationNum) { if (IterationNum == NumIterations) return RetVal = CurrentIterVals[PN]; // Got exit value! @@ -5499,8 +5478,8 @@ ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, // Compute the value of the PHIs for the next iteration. // EvaluateExpression adds non-phi values to the CurrentIterVals map. DenseMap NextIterVals; - Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, - TLI); + Constant *NextPHI = + EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI); if (!NextPHI) return nullptr; // Couldn't evaluate! NextIterVals[PN] = NextPHI; @@ -5576,12 +5555,11 @@ const SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L, // Okay, we find a PHI node that defines the trip count of this loop. Execute // the loop symbolically to determine when the condition gets a value of // "ExitWhen". - unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis. + const DataLayout &DL = F->getParent()->getDataLayout(); for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){ - ConstantInt *CondVal = - dyn_cast_or_null(EvaluateExpression(Cond, L, CurrentIterVals, - DL, TLI)); + ConstantInt *CondVal = dyn_cast_or_null( + EvaluateExpression(Cond, L, CurrentIterVals, DL, TLI)); // Couldn't symbolically evaluate. if (!CondVal) return getCouldNotCompute(); @@ -5814,16 +5792,16 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { // Check to see if getSCEVAtScope actually made an improvement. if (MadeImprovement) { Constant *C = nullptr; + const DataLayout &DL = F->getParent()->getDataLayout(); if (const CmpInst *CI = dyn_cast(I)) - C = ConstantFoldCompareInstOperands(CI->getPredicate(), - Operands[0], Operands[1], DL, - TLI); + C = ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], + Operands[1], DL, TLI); else if (const LoadInst *LI = dyn_cast(I)) { if (!LI->isVolatile()) C = ConstantFoldLoadFromConstPtr(Operands[0], DL); } else - C = ConstantFoldInstOperands(I->getOpcode(), I->getType(), - Operands, DL, TLI); + C = ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, + DL, TLI); if (!C) return V; return getSCEV(C); } @@ -6105,7 +6083,7 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit) { dyn_cast(ConstantExpr::getICmp(CmpInst::ICMP_ULT, R1->getValue(), R2->getValue()))) { - if (CB->getZExtValue() == false) + if (!CB->getZExtValue()) std::swap(R1, R2); // R1 is the minimum root now. // We can only use this value if the chrec ends up with an exact zero @@ -6815,15 +6793,6 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, ICmpInst *ICI = dyn_cast(FoundCondValue); if (!ICI) return false; - // Bail if the ICmp's operands' types are wider than the needed type - // before attempting to call getSCEV on them. This avoids infinite - // recursion, since the analysis of widening casts can require loop - // exit condition information for overflow checking, which would - // lead back here. - if (getTypeSizeInBits(LHS->getType()) < - getTypeSizeInBits(ICI->getOperand(0)->getType())) - return false; - // Now that we found a conditional branch that dominates the loop or controls // the loop latch. Check to see if it is the comparison we are looking for. ICmpInst::Predicate FoundPred; @@ -6835,9 +6804,17 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); - // Balance the types. The case where FoundLHS' type is wider than - // LHS' type is checked for above. - if (getTypeSizeInBits(LHS->getType()) > + // Balance the types. + if (getTypeSizeInBits(LHS->getType()) < + getTypeSizeInBits(FoundLHS->getType())) { + if (CmpInst::isSigned(Pred)) { + LHS = getSignExtendExpr(LHS, FoundLHS->getType()); + RHS = getSignExtendExpr(RHS, FoundLHS->getType()); + } else { + LHS = getZeroExtendExpr(LHS, FoundLHS->getType()); + RHS = getZeroExtendExpr(RHS, FoundLHS->getType()); + } + } else if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(FoundLHS->getType())) { if (CmpInst::isSigned(FoundPred)) { FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType()); @@ -6963,6 +6940,9 @@ bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS, const SCEV *FoundRHS) { + if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS)) + return true; + return isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS) || // ~x < ~y --> x > y @@ -7100,6 +7080,47 @@ ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, return false; } +/// isImpliedCondOperandsViaRanges - helper function for isImpliedCondOperands. +/// Tries to get cases like "X `sgt` 0 => X - 1 `sgt` -1". +bool ScalarEvolution::isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred, + const SCEV *LHS, + const SCEV *RHS, + const SCEV *FoundLHS, + const SCEV *FoundRHS) { + if (!isa(RHS) || !isa(FoundRHS)) + // The restriction on `FoundRHS` be lifted easily -- it exists only to + // reduce the compile time impact of this optimization. + return false; + + const SCEVAddExpr *AddLHS = dyn_cast(LHS); + if (!AddLHS || AddLHS->getOperand(1) != FoundLHS || + !isa(AddLHS->getOperand(0))) + return false; + + APInt ConstFoundRHS = cast(FoundRHS)->getValue()->getValue(); + + // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the + // antecedent "`FoundLHS` `Pred` `FoundRHS`". + ConstantRange FoundLHSRange = + ConstantRange::makeAllowedICmpRegion(Pred, ConstFoundRHS); + + // Since `LHS` is `FoundLHS` + `AddLHS->getOperand(0)`, we can compute a range + // for `LHS`: + APInt Addend = + cast(AddLHS->getOperand(0))->getValue()->getValue(); + ConstantRange LHSRange = FoundLHSRange.add(ConstantRange(Addend)); + + // We can also compute the range of values for `LHS` that satisfy the + // consequent, "`LHS` `Pred` `RHS`": + APInt ConstRHS = cast(RHS)->getValue()->getValue(); + ConstantRange SatisfyingLHSRange = + ConstantRange::makeSatisfyingICmpRegion(Pred, ConstRHS); + + // The antecedent implies the consequent if every value of `LHS` that + // satisfies the antecedent also satisfies the consequent. + return SatisfyingLHSRange.contains(LHSRange); +} + // Verify if an linear IV with positive stride can overflow when in a // less-than comparison, knowing the invariant term of the comparison, the // stride and the knowledge of NSW/NUW flags on the recurrence. @@ -7428,7 +7449,7 @@ const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, if (ConstantInt *CB = dyn_cast(ConstantExpr::getICmp(ICmpInst::ICMP_ULT, R1->getValue(), R2->getValue()))) { - if (CB->getZExtValue() == false) + if (!CB->getZExtValue()) std::swap(R1, R2); // R1 is the minimum root now. // Make sure the root is not off by one. The returned iteration should @@ -7956,8 +7977,6 @@ bool ScalarEvolution::runOnFunction(Function &F) { this->F = &F; AC = &getAnalysis().getAssumptionCache(F); LI = &getAnalysis().getLoopInfo(); - DataLayoutPass *DLP = getAnalysisIfAvailable(); - DL = DLP ? &DLP->getDataLayout() : nullptr; TLI = &getAnalysis().getTLI(); DT = &getAnalysis().getDomTree(); return false; @@ -8058,6 +8077,12 @@ void ScalarEvolution::print(raw_ostream &OS, const Module *) const { OS << " --> "; const SCEV *SV = SE.getSCEV(&*I); SV->print(OS); + if (!isa(SV)) { + OS << " U: "; + SE.getUnsignedRange(SV).print(OS); + OS << " S: "; + SE.getSignedRange(SV).print(OS); + } const Loop *L = LI->getLoopFor((*I).getParent()); @@ -8065,6 +8090,12 @@ void ScalarEvolution::print(raw_ostream &OS, const Module *) const { if (AtUse != SV) { OS << " --> "; AtUse->print(OS); + if (!isa(AtUse)) { + OS << " U: "; + SE.getUnsignedRange(AtUse).print(OS); + OS << " S: "; + SE.getSignedRange(AtUse).print(OS); + } } if (L) { diff --git a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp index 5c339ee..ccec0a8 100644 --- a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp +++ b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp @@ -22,6 +22,7 @@ #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/IR/Module.h" #include "llvm/Pass.h" using namespace llvm; @@ -79,7 +80,7 @@ ScalarEvolutionAliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const { bool ScalarEvolutionAliasAnalysis::runOnFunction(Function &F) { - InitializeAliasAnalysis(this); + InitializeAliasAnalysis(this, &F.getParent()->getDataLayout()); SE = &getAnalysis(); return false; } diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp index 2625cf3..a73ec9e 100644 --- a/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/lib/Analysis/ScalarEvolutionExpander.cpp @@ -24,6 +24,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -204,11 +205,9 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, /// TODO: When ScalarEvolution gets a SCEVSDivExpr, this can be made /// unnecessary; in its place, just signed-divide Ops[i] by the scale and /// check to see if the divide was folded. -static bool FactorOutConstant(const SCEV *&S, - const SCEV *&Remainder, - const SCEV *Factor, - ScalarEvolution &SE, - const DataLayout *DL) { +static bool FactorOutConstant(const SCEV *&S, const SCEV *&Remainder, + const SCEV *Factor, ScalarEvolution &SE, + const DataLayout &DL) { // Everything is divisible by one. if (Factor->isOne()) return true; @@ -248,35 +247,17 @@ static bool FactorOutConstant(const SCEV *&S, // In a Mul, check if there is a constant operand which is a multiple // of the given factor. if (const SCEVMulExpr *M = dyn_cast(S)) { - if (DL) { - // With DataLayout, the size is known. Check if there is a constant - // operand which is a multiple of the given factor. If so, we can - // factor it. - const SCEVConstant *FC = cast(Factor); - if (const SCEVConstant *C = dyn_cast(M->getOperand(0))) - if (!C->getValue()->getValue().srem(FC->getValue()->getValue())) { - SmallVector NewMulOps(M->op_begin(), M->op_end()); - NewMulOps[0] = - SE.getConstant(C->getValue()->getValue().sdiv( - FC->getValue()->getValue())); - S = SE.getMulExpr(NewMulOps); - return true; - } - } else { - // Without DataLayout, check if Factor can be factored out of any of the - // Mul's operands. If so, we can just remove it. - for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { - const SCEV *SOp = M->getOperand(i); - const SCEV *Remainder = SE.getConstant(SOp->getType(), 0); - if (FactorOutConstant(SOp, Remainder, Factor, SE, DL) && - Remainder->isZero()) { - SmallVector NewMulOps(M->op_begin(), M->op_end()); - NewMulOps[i] = SOp; - S = SE.getMulExpr(NewMulOps); - return true; - } + // Size is known, check if there is a constant operand which is a multiple + // of the given factor. If so, we can factor it. + const SCEVConstant *FC = cast(Factor); + if (const SCEVConstant *C = dyn_cast(M->getOperand(0))) + if (!C->getValue()->getValue().srem(FC->getValue()->getValue())) { + SmallVector NewMulOps(M->op_begin(), M->op_end()); + NewMulOps[0] = SE.getConstant( + C->getValue()->getValue().sdiv(FC->getValue()->getValue())); + S = SE.getMulExpr(NewMulOps); + return true; } - } } // In an AddRec, check if both start and step are divisible. @@ -393,7 +374,8 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, PointerType *PTy, Type *Ty, Value *V) { - Type *ElTy = PTy->getElementType(); + Type *OriginalElTy = PTy->getElementType(); + Type *ElTy = OriginalElTy; SmallVector GepIndices; SmallVector Ops(op_begin, op_end); bool AnyNonZeroIndices = false; @@ -402,9 +384,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, // without the other. SplitAddRecs(Ops, Ty, SE); - Type *IntPtrTy = SE.DL - ? SE.DL->getIntPtrType(PTy) - : Type::getInt64Ty(PTy->getContext()); + Type *IntPtrTy = DL.getIntPtrType(PTy); // Descend down the pointer's type and attempt to convert the other // operands into GEP indices, at each level. The first index in a GEP @@ -422,7 +402,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, for (unsigned i = 0, e = Ops.size(); i != e; ++i) { const SCEV *Op = Ops[i]; const SCEV *Remainder = SE.getConstant(Ty, 0); - if (FactorOutConstant(Op, Remainder, ElSize, SE, SE.DL)) { + if (FactorOutConstant(Op, Remainder, ElSize, SE, DL)) { // Op now has ElSize factored out. ScaledOps.push_back(Op); if (!Remainder->isZero()) @@ -456,43 +436,25 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, bool FoundFieldNo = false; // An empty struct has no fields. if (STy->getNumElements() == 0) break; - if (SE.DL) { - // With DataLayout, field offsets are known. See if a constant offset - // falls within any of the struct fields. - if (Ops.empty()) break; - if (const SCEVConstant *C = dyn_cast(Ops[0])) - if (SE.getTypeSizeInBits(C->getType()) <= 64) { - const StructLayout &SL = *SE.DL->getStructLayout(STy); - uint64_t FullOffset = C->getValue()->getZExtValue(); - if (FullOffset < SL.getSizeInBytes()) { - unsigned ElIdx = SL.getElementContainingOffset(FullOffset); - GepIndices.push_back( - ConstantInt::get(Type::getInt32Ty(Ty->getContext()), ElIdx)); - ElTy = STy->getTypeAtIndex(ElIdx); - Ops[0] = + // Field offsets are known. See if a constant offset falls within any of + // the struct fields. + if (Ops.empty()) + break; + if (const SCEVConstant *C = dyn_cast(Ops[0])) + if (SE.getTypeSizeInBits(C->getType()) <= 64) { + const StructLayout &SL = *DL.getStructLayout(STy); + uint64_t FullOffset = C->getValue()->getZExtValue(); + if (FullOffset < SL.getSizeInBytes()) { + unsigned ElIdx = SL.getElementContainingOffset(FullOffset); + GepIndices.push_back( + ConstantInt::get(Type::getInt32Ty(Ty->getContext()), ElIdx)); + ElTy = STy->getTypeAtIndex(ElIdx); + Ops[0] = SE.getConstant(Ty, FullOffset - SL.getElementOffset(ElIdx)); - AnyNonZeroIndices = true; - FoundFieldNo = true; - } - } - } else { - // Without DataLayout, just check for an offsetof expression of the - // appropriate struct type. - for (unsigned i = 0, e = Ops.size(); i != e; ++i) - if (const SCEVUnknown *U = dyn_cast(Ops[i])) { - Type *CTy; - Constant *FieldNo; - if (U->isOffsetOf(CTy, FieldNo) && CTy == STy) { - GepIndices.push_back(FieldNo); - ElTy = - STy->getTypeAtIndex(cast(FieldNo)->getZExtValue()); - Ops[i] = SE.getConstant(Ty, 0); - AnyNonZeroIndices = true; - FoundFieldNo = true; - break; - } + AnyNonZeroIndices = true; + FoundFieldNo = true; } - } + } // If no struct field offsets were found, tentatively assume that // field zero was selected (since the zero offset would obviously // be folded away). @@ -597,7 +559,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, Value *Casted = V; if (V->getType() != PTy) Casted = InsertNoopCastOfTo(Casted, PTy); - Value *GEP = Builder.CreateGEP(Casted, + Value *GEP = Builder.CreateGEP(OriginalElTy, Casted, GepIndices, "scevgep"); Ops.push_back(SE.getUnknown(GEP)); @@ -1746,7 +1708,7 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT, // Fold constant phis. They may be congruent to other constant phis and // would confuse the logic below that expects proper IVs. - if (Value *V = SimplifyInstruction(Phi, SE.DL, SE.TLI, SE.DT, SE.AC)) { + if (Value *V = SimplifyInstruction(Phi, DL, SE.TLI, SE.DT, SE.AC)) { Phi->replaceAllUsesWith(V); DeadInsts.push_back(Phi); ++NumElim; @@ -1811,9 +1773,12 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT, << *IsomorphicInc << '\n'); Value *NewInc = OrigInc; if (OrigInc->getType() != IsomorphicInc->getType()) { - Instruction *IP = isa(OrigInc) - ? (Instruction*)L->getHeader()->getFirstInsertionPt() - : OrigInc->getNextNode(); + Instruction *IP = nullptr; + if (PHINode *PN = dyn_cast(OrigInc)) + IP = PN->getParent()->getFirstInsertionPt(); + else + IP = OrigInc->getNextNode(); + IRBuilder<> Builder(IP); Builder.SetCurrentDebugLocation(IsomorphicInc->getDebugLoc()); NewInc = Builder. diff --git a/lib/Analysis/ScopedNoAliasAA.cpp b/lib/Analysis/ScopedNoAliasAA.cpp index c6ea3af..02f8b0b 100644 --- a/lib/Analysis/ScopedNoAliasAA.cpp +++ b/lib/Analysis/ScopedNoAliasAA.cpp @@ -80,7 +80,7 @@ public: initializeScopedNoAliasAAPass(*PassRegistry::getPassRegistry()); } - void initializePass() override { InitializeAliasAnalysis(this); } + bool doInitialization(Module &M) override; /// getAdjustedAnalysisPointer - This method is used when a pass implements /// an analysis interface through multiple inheritance. If needed, it @@ -119,6 +119,11 @@ ImmutablePass *llvm::createScopedNoAliasAAPass() { return new ScopedNoAliasAA(); } +bool ScopedNoAliasAA::doInitialization(Module &M) { + InitializeAliasAnalysis(this, &M.getDataLayout()); + return true; +} + void ScopedNoAliasAA::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); diff --git a/lib/Analysis/TargetLibraryInfo.cpp b/lib/Analysis/TargetLibraryInfo.cpp index 91041fc..7e574d5 100644 --- a/lib/Analysis/TargetLibraryInfo.cpp +++ b/lib/Analysis/TargetLibraryInfo.cpp @@ -13,341 +13,22 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/ADT/Triple.h" +#include "llvm/Support/CommandLine.h" using namespace llvm; -const char* TargetLibraryInfoImpl::StandardNames[LibFunc::NumLibFuncs] = - { - "_IO_getc", - "_IO_putc", - "_ZdaPv", - "_ZdaPvRKSt9nothrow_t", - "_ZdaPvj", - "_ZdaPvm", - "_ZdlPv", - "_ZdlPvRKSt9nothrow_t", - "_ZdlPvj", - "_ZdlPvm", - "_Znaj", - "_ZnajRKSt9nothrow_t", - "_Znam", - "_ZnamRKSt9nothrow_t", - "_Znwj", - "_ZnwjRKSt9nothrow_t", - "_Znwm", - "_ZnwmRKSt9nothrow_t", - "__cospi", - "__cospif", - "__cxa_atexit", - "__cxa_guard_abort", - "__cxa_guard_acquire", - "__cxa_guard_release", - "__isoc99_scanf", - "__isoc99_sscanf", - "__memcpy_chk", - "__memmove_chk", - "__memset_chk", - "__sincospi_stret", - "__sincospif_stret", - "__sinpi", - "__sinpif", - "__sqrt_finite", - "__sqrtf_finite", - "__sqrtl_finite", - "__stpcpy_chk", - "__stpncpy_chk", - "__strcpy_chk", - "__strdup", - "__strncpy_chk", - "__strndup", - "__strtok_r", - "abs", - "access", - "acos", - "acosf", - "acosh", - "acoshf", - "acoshl", - "acosl", - "asin", - "asinf", - "asinh", - "asinhf", - "asinhl", - "asinl", - "atan", - "atan2", - "atan2f", - "atan2l", - "atanf", - "atanh", - "atanhf", - "atanhl", - "atanl", - "atof", - "atoi", - "atol", - "atoll", - "bcmp", - "bcopy", - "bzero", - "calloc", - "cbrt", - "cbrtf", - "cbrtl", - "ceil", - "ceilf", - "ceill", - "chmod", - "chown", - "clearerr", - "closedir", - "copysign", - "copysignf", - "copysignl", - "cos", - "cosf", - "cosh", - "coshf", - "coshl", - "cosl", - "ctermid", - "exp", - "exp10", - "exp10f", - "exp10l", - "exp2", - "exp2f", - "exp2l", - "expf", - "expl", - "expm1", - "expm1f", - "expm1l", - "fabs", - "fabsf", - "fabsl", - "fclose", - "fdopen", - "feof", - "ferror", - "fflush", - "ffs", - "ffsl", - "ffsll", - "fgetc", - "fgetpos", - "fgets", - "fileno", - "fiprintf", - "flockfile", - "floor", - "floorf", - "floorl", - "fmax", - "fmaxf", - "fmaxl", - "fmin", - "fminf", - "fminl", - "fmod", - "fmodf", - "fmodl", - "fopen", - "fopen64", - "fprintf", - "fputc", - "fputs", - "fread", - "free", - "frexp", - "frexpf", - "frexpl", - "fscanf", - "fseek", - "fseeko", - "fseeko64", - "fsetpos", - "fstat", - "fstat64", - "fstatvfs", - "fstatvfs64", - "ftell", - "ftello", - "ftello64", - "ftrylockfile", - "funlockfile", - "fwrite", - "getc", - "getc_unlocked", - "getchar", - "getenv", - "getitimer", - "getlogin_r", - "getpwnam", - "gets", - "gettimeofday", - "htonl", - "htons", - "iprintf", - "isascii", - "isdigit", - "labs", - "lchown", - "ldexp", - "ldexpf", - "ldexpl", - "llabs", - "log", - "log10", - "log10f", - "log10l", - "log1p", - "log1pf", - "log1pl", - "log2", - "log2f", - "log2l", - "logb", - "logbf", - "logbl", - "logf", - "logl", - "lstat", - "lstat64", - "malloc", - "memalign", - "memccpy", - "memchr", - "memcmp", - "memcpy", - "memmove", - "memrchr", - "memset", - "memset_pattern16", - "mkdir", - "mktime", - "modf", - "modff", - "modfl", - "nearbyint", - "nearbyintf", - "nearbyintl", - "ntohl", - "ntohs", - "open", - "open64", - "opendir", - "pclose", - "perror", - "popen", - "posix_memalign", - "pow", - "powf", - "powl", - "pread", - "printf", - "putc", - "putchar", - "puts", - "pwrite", - "qsort", - "read", - "readlink", - "realloc", - "reallocf", - "realpath", - "remove", - "rename", - "rewind", - "rint", - "rintf", - "rintl", - "rmdir", - "round", - "roundf", - "roundl", - "scanf", - "setbuf", - "setitimer", - "setvbuf", - "sin", - "sinf", - "sinh", - "sinhf", - "sinhl", - "sinl", - "siprintf", - "snprintf", - "sprintf", - "sqrt", - "sqrtf", - "sqrtl", - "sscanf", - "stat", - "stat64", - "statvfs", - "statvfs64", - "stpcpy", - "stpncpy", - "strcasecmp", - "strcat", - "strchr", - "strcmp", - "strcoll", - "strcpy", - "strcspn", - "strdup", - "strlen", - "strncasecmp", - "strncat", - "strncmp", - "strncpy", - "strndup", - "strnlen", - "strpbrk", - "strrchr", - "strspn", - "strstr", - "strtod", - "strtof", - "strtok", - "strtok_r", - "strtol", - "strtold", - "strtoll", - "strtoul", - "strtoull", - "strxfrm", - "system", - "tan", - "tanf", - "tanh", - "tanhf", - "tanhl", - "tanl", - "times", - "tmpfile", - "tmpfile64", - "toascii", - "trunc", - "truncf", - "truncl", - "uname", - "ungetc", - "unlink", - "unsetenv", - "utime", - "utimes", - "valloc", - "vfprintf", - "vfscanf", - "vprintf", - "vscanf", - "vsnprintf", - "vsprintf", - "vsscanf", - "write" - }; +static cl::opt ClVectorLibrary( + "vector-library", cl::Hidden, cl::desc("Vector functions library"), + cl::init(TargetLibraryInfoImpl::NoLibrary), + cl::values(clEnumValN(TargetLibraryInfoImpl::NoLibrary, "none", + "No vector functions library"), + clEnumValN(TargetLibraryInfoImpl::Accelerate, "Accelerate", + "Accelerate framework"), + clEnumValEnd)); + +const char *const TargetLibraryInfoImpl::StandardNames[LibFunc::NumLibFuncs] = { +#define TLI_DEFINE_STRING +#include "llvm/Analysis/TargetLibraryInfo.def" +}; static bool hasSinCosPiStret(const Triple &T) { // Only Darwin variants have _stret versions of combined trig functions. @@ -371,7 +52,7 @@ static bool hasSinCosPiStret(const Triple &T) { /// specified target triple. This should be carefully written so that a missing /// target triple gets a sane set of defaults. static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, - const char **StandardNames) { + const char *const *StandardNames) { #ifndef NDEBUG // Verify that the StandardNames array is in alphabetical order. for (unsigned F = 1; F < LibFunc::NumLibFuncs; ++F) { @@ -674,6 +355,8 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setUnavailable(LibFunc::statvfs64); TLI.setUnavailable(LibFunc::tmpfile64); } + + TLI.addVectorizableFunctionsFromVecLib(ClVectorLibrary); } TargetLibraryInfoImpl::TargetLibraryInfoImpl() { @@ -693,12 +376,16 @@ TargetLibraryInfoImpl::TargetLibraryInfoImpl(const Triple &T) { TargetLibraryInfoImpl::TargetLibraryInfoImpl(const TargetLibraryInfoImpl &TLI) : CustomNames(TLI.CustomNames) { memcpy(AvailableArray, TLI.AvailableArray, sizeof(AvailableArray)); + VectorDescs = TLI.VectorDescs; + ScalarDescs = TLI.ScalarDescs; } TargetLibraryInfoImpl::TargetLibraryInfoImpl(TargetLibraryInfoImpl &&TLI) : CustomNames(std::move(TLI.CustomNames)) { std::move(std::begin(TLI.AvailableArray), std::end(TLI.AvailableArray), AvailableArray); + VectorDescs = TLI.VectorDescs; + ScalarDescs = TLI.ScalarDescs; } TargetLibraryInfoImpl &TargetLibraryInfoImpl::operator=(const TargetLibraryInfoImpl &TLI) { @@ -714,40 +401,32 @@ TargetLibraryInfoImpl &TargetLibraryInfoImpl::operator=(TargetLibraryInfoImpl && return *this; } -namespace { -struct StringComparator { - /// Compare two strings and return true if LHS is lexicographically less than - /// RHS. Requires that RHS doesn't contain any zero bytes. - bool operator()(const char *LHS, StringRef RHS) const { - // Compare prefixes with strncmp. If prefixes match we know that LHS is - // greater or equal to RHS as RHS can't contain any '\0'. - return std::strncmp(LHS, RHS.data(), RHS.size()) < 0; - } - - // Provided for compatibility with MSVC's debug mode. - bool operator()(StringRef LHS, const char *RHS) const { return LHS < RHS; } - bool operator()(StringRef LHS, StringRef RHS) const { return LHS < RHS; } - bool operator()(const char *LHS, const char *RHS) const { - return std::strcmp(LHS, RHS) < 0; - } -}; -} - -bool TargetLibraryInfoImpl::getLibFunc(StringRef funcName, - LibFunc::Func &F) const { - const char **Start = &StandardNames[0]; - const char **End = &StandardNames[LibFunc::NumLibFuncs]; - +static StringRef sanitizeFunctionName(StringRef funcName) { // Filter out empty names and names containing null bytes, those can't be in // our table. if (funcName.empty() || funcName.find('\0') != StringRef::npos) - return false; + return StringRef(); // Check for \01 prefix that is used to mangle __asm declarations and // strip it if present. if (funcName.front() == '\01') funcName = funcName.substr(1); - const char **I = std::lower_bound(Start, End, funcName, StringComparator()); + return funcName; +} + +bool TargetLibraryInfoImpl::getLibFunc(StringRef funcName, + LibFunc::Func &F) const { + const char *const *Start = &StandardNames[0]; + const char *const *End = &StandardNames[LibFunc::NumLibFuncs]; + + funcName = sanitizeFunctionName(funcName); + if (funcName.empty()) + return false; + + const char *const *I = std::lower_bound( + Start, End, funcName, [](const char *LHS, StringRef RHS) { + return std::strncmp(LHS, RHS.data(), RHS.size()) < 0; + }); if (I != End && *I == funcName) { F = (LibFunc::Func)(I - Start); return true; @@ -759,6 +438,94 @@ void TargetLibraryInfoImpl::disableAllFunctions() { memset(AvailableArray, 0, sizeof(AvailableArray)); } +static bool compareByScalarFnName(const VecDesc &LHS, const VecDesc &RHS) { + return std::strncmp(LHS.ScalarFnName, RHS.ScalarFnName, + std::strlen(RHS.ScalarFnName)) < 0; +} + +static bool compareByVectorFnName(const VecDesc &LHS, const VecDesc &RHS) { + return std::strncmp(LHS.VectorFnName, RHS.VectorFnName, + std::strlen(RHS.VectorFnName)) < 0; +} + +static bool compareWithScalarFnName(const VecDesc &LHS, StringRef S) { + return std::strncmp(LHS.ScalarFnName, S.data(), S.size()) < 0; +} + +static bool compareWithVectorFnName(const VecDesc &LHS, StringRef S) { + return std::strncmp(LHS.VectorFnName, S.data(), S.size()) < 0; +} + +void TargetLibraryInfoImpl::addVectorizableFunctions(ArrayRef Fns) { + VectorDescs.insert(VectorDescs.end(), Fns.begin(), Fns.end()); + std::sort(VectorDescs.begin(), VectorDescs.end(), compareByScalarFnName); + + ScalarDescs.insert(ScalarDescs.end(), Fns.begin(), Fns.end()); + std::sort(ScalarDescs.begin(), ScalarDescs.end(), compareByVectorFnName); +} + +void TargetLibraryInfoImpl::addVectorizableFunctionsFromVecLib( + enum VectorLibrary VecLib) { + switch (VecLib) { + case Accelerate: { + const VecDesc VecFuncs[] = { + {"expf", "vexpf", 4}, + {"llvm.exp.f32", "vexpf", 4}, + {"logf", "vlogf", 4}, + {"llvm.log.f32", "vlogf", 4}, + {"sqrtf", "vsqrtf", 4}, + {"llvm.sqrt.f32", "vsqrtf", 4}, + {"fabsf", "vfabsf", 4}, + {"llvm.fabs.f32", "vfabsf", 4}, + }; + addVectorizableFunctions(VecFuncs); + break; + } + case NoLibrary: + break; + } +} + +bool TargetLibraryInfoImpl::isFunctionVectorizable(StringRef funcName) const { + funcName = sanitizeFunctionName(funcName); + if (funcName.empty()) + return false; + + std::vector::const_iterator I = std::lower_bound( + VectorDescs.begin(), VectorDescs.end(), funcName, + compareWithScalarFnName); + return I != VectorDescs.end() && StringRef(I->ScalarFnName) == funcName; +} + +StringRef TargetLibraryInfoImpl::getVectorizedFunction(StringRef F, + unsigned VF) const { + F = sanitizeFunctionName(F); + if (F.empty()) + return F; + std::vector::const_iterator I = std::lower_bound( + VectorDescs.begin(), VectorDescs.end(), F, compareWithScalarFnName); + while (I != VectorDescs.end() && StringRef(I->ScalarFnName) == F) { + if (I->VectorizationFactor == VF) + return I->VectorFnName; + ++I; + } + return StringRef(); +} + +StringRef TargetLibraryInfoImpl::getScalarizedFunction(StringRef F, + unsigned &VF) const { + F = sanitizeFunctionName(F); + if (F.empty()) + return F; + + std::vector::const_iterator I = std::lower_bound( + ScalarDescs.begin(), ScalarDescs.end(), F, compareWithVectorFnName); + if (I == VectorDescs.end() || StringRef(I->VectorFnName) != F) + return StringRef(); + VF = I->VectorizationFactor; + return I->ScalarFnName; +} + TargetLibraryInfo TargetLibraryAnalysis::run(Module &M) { if (PresetInfoImpl) return TargetLibraryInfo(*PresetInfoImpl); diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp index 7ff29b0..f51c7f54 100644 --- a/lib/Analysis/TargetTransformInfo.cpp +++ b/lib/Analysis/TargetTransformInfo.cpp @@ -143,6 +143,10 @@ bool TargetTransformInfo::shouldBuildLookupTables() const { return TTIImpl->shouldBuildLookupTables(); } +bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const { + return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); +} + TargetTransformInfo::PopcntSupportKind TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const { return TTIImpl->getPopcntSupport(IntTyWidthInBit); @@ -233,6 +237,11 @@ TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, return TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys); } +unsigned TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, + ArrayRef Tys) const { + return TTIImpl->getCallInstrCost(F, RetTy, Tys); +} + unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { return TTIImpl->getNumberOfParts(Tp); } @@ -277,7 +286,7 @@ TargetIRAnalysis::Result TargetIRAnalysis::run(Function &F) { char TargetIRAnalysis::PassID; TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(Function &F) { - return Result(F.getParent()->getDataLayout()); + return Result(&F.getParent()->getDataLayout()); } // Register the basic pass. diff --git a/lib/Analysis/TypeBasedAliasAnalysis.cpp b/lib/Analysis/TypeBasedAliasAnalysis.cpp index ff89558..1158725 100644 --- a/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/lib/Analysis/TypeBasedAliasAnalysis.cpp @@ -129,6 +129,7 @@ #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" +#include "llvm/ADT/SetVector.h" using namespace llvm; // A handy option for disabling TBAA functionality. The same effect can also be @@ -282,9 +283,7 @@ namespace { initializeTypeBasedAliasAnalysisPass(*PassRegistry::getPassRegistry()); } - void initializePass() override { - InitializeAliasAnalysis(this); - } + bool doInitialization(Module &M) override; /// getAdjustedAnalysisPointer - This method is used when a pass implements /// an analysis interface through multiple inheritance. If needed, it @@ -321,6 +320,11 @@ ImmutablePass *llvm::createTypeBasedAliasAnalysisPass() { return new TypeBasedAliasAnalysis(); } +bool TypeBasedAliasAnalysis::doInitialization(Module &M) { + InitializeAliasAnalysis(this, &M.getDataLayout()); + return true; +} + void TypeBasedAliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); @@ -575,18 +579,22 @@ MDNode *MDNode::getMostGenericTBAA(MDNode *A, MDNode *B) { if (!B) return nullptr; } - SmallVector PathA; + SmallSetVector PathA; MDNode *T = A; while (T) { - PathA.push_back(T); + if (PathA.count(T)) + report_fatal_error("Cycle found in TBAA metadata."); + PathA.insert(T); T = T->getNumOperands() >= 2 ? cast_or_null(T->getOperand(1)) : nullptr; } - SmallVector PathB; + SmallSetVector PathB; T = B; while (T) { - PathB.push_back(T); + if (PathB.count(T)) + report_fatal_error("Cycle found in TBAA metadata."); + PathB.insert(T); T = T->getNumOperands() >= 2 ? cast_or_null(T->getOperand(1)) : nullptr; } diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp index 0458d28..f329e3a 100644 --- a/lib/Analysis/ValueTracking.cpp +++ b/lib/Analysis/ValueTracking.cpp @@ -39,13 +39,41 @@ using namespace llvm::PatternMatch; const unsigned MaxDepth = 6; +/// Enable an experimental feature to leverage information about dominating +/// conditions to compute known bits. The individual options below control how +/// hard we search. The defaults are choosen to be fairly aggressive. If you +/// run into compile time problems when testing, scale them back and report +/// your findings. +static cl::opt EnableDomConditions("value-tracking-dom-conditions", + cl::Hidden, cl::init(false)); + +// This is expensive, so we only do it for the top level query value. +// (TODO: evaluate cost vs profit, consider higher thresholds) +static cl::opt DomConditionsMaxDepth("dom-conditions-max-depth", + cl::Hidden, cl::init(1)); + +/// How many dominating blocks should be scanned looking for dominating +/// conditions? +static cl::opt DomConditionsMaxDomBlocks("dom-conditions-dom-blocks", + cl::Hidden, + cl::init(20000)); + +// Controls the number of uses of the value searched for possible +// dominating comparisons. +static cl::opt DomConditionsMaxUses("dom-conditions-max-uses", + cl::Hidden, cl::init(2000)); + +// If true, don't consider only compares whose only use is a branch. +static cl::opt DomConditionsSingleCmpUse("dom-conditions-single-cmp-use", + cl::Hidden, cl::init(false)); + /// Returns the bitwidth of the given scalar or pointer type (if unknown returns /// 0). For vector types, returns the element type's bitwidth. -static unsigned getBitWidth(Type *Ty, const DataLayout *TD) { +static unsigned getBitWidth(Type *Ty, const DataLayout &DL) { if (unsigned BitWidth = Ty->getScalarSizeInBits()) return BitWidth; - return TD ? TD->getPointerTypeSizeInBits(Ty) : 0; + return DL.getPointerTypeSizeInBits(Ty); } // Many of these functions have internal versions that take an assumption @@ -97,73 +125,73 @@ static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) { } static void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, - const DataLayout *TD, unsigned Depth, - const Query &Q); + const DataLayout &DL, unsigned Depth, + const Query &Q); void llvm::computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, - const DataLayout *TD, unsigned Depth, + const DataLayout &DL, unsigned Depth, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { - ::computeKnownBits(V, KnownZero, KnownOne, TD, Depth, + ::computeKnownBits(V, KnownZero, KnownOne, DL, Depth, Query(AC, safeCxtI(V, CxtI), DT)); } static void ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne, - const DataLayout *TD, unsigned Depth, - const Query &Q); + const DataLayout &DL, unsigned Depth, + const Query &Q); void llvm::ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne, - const DataLayout *TD, unsigned Depth, + const DataLayout &DL, unsigned Depth, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { - ::ComputeSignBit(V, KnownZero, KnownOne, TD, Depth, + ::ComputeSignBit(V, KnownZero, KnownOne, DL, Depth, Query(AC, safeCxtI(V, CxtI), DT)); } static bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, - const Query &Q); + const Query &Q, const DataLayout &DL); -bool llvm::isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, - AssumptionCache *AC, const Instruction *CxtI, +bool llvm::isKnownToBeAPowerOfTwo(Value *V, const DataLayout &DL, bool OrZero, + unsigned Depth, AssumptionCache *AC, + const Instruction *CxtI, const DominatorTree *DT) { return ::isKnownToBeAPowerOfTwo(V, OrZero, Depth, - Query(AC, safeCxtI(V, CxtI), DT)); + Query(AC, safeCxtI(V, CxtI), DT), DL); } -static bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, +static bool isKnownNonZero(Value *V, const DataLayout &DL, unsigned Depth, const Query &Q); -bool llvm::isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, +bool llvm::isKnownNonZero(Value *V, const DataLayout &DL, unsigned Depth, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { - return ::isKnownNonZero(V, TD, Depth, Query(AC, safeCxtI(V, CxtI), DT)); + return ::isKnownNonZero(V, DL, Depth, Query(AC, safeCxtI(V, CxtI), DT)); } -static bool MaskedValueIsZero(Value *V, const APInt &Mask, - const DataLayout *TD, unsigned Depth, - const Query &Q); +static bool MaskedValueIsZero(Value *V, const APInt &Mask, const DataLayout &DL, + unsigned Depth, const Query &Q); -bool llvm::MaskedValueIsZero(Value *V, const APInt &Mask, const DataLayout *TD, +bool llvm::MaskedValueIsZero(Value *V, const APInt &Mask, const DataLayout &DL, unsigned Depth, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { - return ::MaskedValueIsZero(V, Mask, TD, Depth, + return ::MaskedValueIsZero(V, Mask, DL, Depth, Query(AC, safeCxtI(V, CxtI), DT)); } -static unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, +static unsigned ComputeNumSignBits(Value *V, const DataLayout &DL, unsigned Depth, const Query &Q); -unsigned llvm::ComputeNumSignBits(Value *V, const DataLayout *TD, +unsigned llvm::ComputeNumSignBits(Value *V, const DataLayout &DL, unsigned Depth, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { - return ::ComputeNumSignBits(V, TD, Depth, Query(AC, safeCxtI(V, CxtI), DT)); + return ::ComputeNumSignBits(V, DL, Depth, Query(AC, safeCxtI(V, CxtI), DT)); } static void computeKnownBitsAddSub(bool Add, Value *Op0, Value *Op1, bool NSW, APInt &KnownZero, APInt &KnownOne, APInt &KnownZero2, APInt &KnownOne2, - const DataLayout *TD, unsigned Depth, + const DataLayout &DL, unsigned Depth, const Query &Q) { if (!Add) { if (ConstantInt *CLHS = dyn_cast(Op0)) { @@ -175,7 +203,7 @@ static void computeKnownBitsAddSub(bool Add, Value *Op0, Value *Op1, bool NSW, unsigned NLZ = (CLHS->getValue()+1).countLeadingZeros(); // NLZ can't be BitWidth with no sign bit APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); - computeKnownBits(Op1, KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(Op1, KnownZero2, KnownOne2, DL, Depth + 1, Q); // If all of the MaskV bits are known to be zero, then we know the // output top bits are zero, because we now know that the output is @@ -194,8 +222,8 @@ static void computeKnownBitsAddSub(bool Add, Value *Op0, Value *Op1, bool NSW, // If an initial sequence of bits in the result is not needed, the // corresponding bits in the operands are not needed. APInt LHSKnownZero(BitWidth, 0), LHSKnownOne(BitWidth, 0); - computeKnownBits(Op0, LHSKnownZero, LHSKnownOne, TD, Depth+1, Q); - computeKnownBits(Op1, KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(Op0, LHSKnownZero, LHSKnownOne, DL, Depth + 1, Q); + computeKnownBits(Op1, KnownZero2, KnownOne2, DL, Depth + 1, Q); // Carry in a 1 for a subtract, rather than a 0. APInt CarryIn(BitWidth, 0); @@ -243,11 +271,11 @@ static void computeKnownBitsAddSub(bool Add, Value *Op0, Value *Op1, bool NSW, static void computeKnownBitsMul(Value *Op0, Value *Op1, bool NSW, APInt &KnownZero, APInt &KnownOne, APInt &KnownZero2, APInt &KnownOne2, - const DataLayout *TD, unsigned Depth, + const DataLayout &DL, unsigned Depth, const Query &Q) { unsigned BitWidth = KnownZero.getBitWidth(); - computeKnownBits(Op1, KnownZero, KnownOne, TD, Depth+1, Q); - computeKnownBits(Op0, KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(Op1, KnownZero, KnownOne, DL, Depth + 1, Q); + computeKnownBits(Op0, KnownZero2, KnownOne2, DL, Depth + 1, Q); bool isKnownNegative = false; bool isKnownNonNegative = false; @@ -268,9 +296,9 @@ static void computeKnownBitsMul(Value *Op0, Value *Op1, bool NSW, // negative or zero. if (!isKnownNonNegative) isKnownNegative = (isKnownNegativeOp1 && isKnownNonNegativeOp0 && - isKnownNonZero(Op0, TD, Depth, Q)) || + isKnownNonZero(Op0, DL, Depth, Q)) || (isKnownNegativeOp0 && isKnownNonNegativeOp1 && - isKnownNonZero(Op1, TD, Depth, Q)); + isKnownNonZero(Op1, DL, Depth, Q)); } } @@ -382,8 +410,7 @@ static bool isAssumeLikeIntrinsic(const Instruction *I) { return false; } -static bool isValidAssumeForContext(Value *V, const Query &Q, - const DataLayout *DL) { +static bool isValidAssumeForContext(Value *V, const Query &Q) { Instruction *Inv = cast(V); // There are two restrictions on the use of an assume: @@ -403,8 +430,7 @@ static bool isValidAssumeForContext(Value *V, const Query &Q, for (BasicBlock::const_iterator I = std::next(BasicBlock::const_iterator(Q.CxtI)), IE(Inv); I != IE; ++I) - if (!isSafeToSpeculativelyExecute(I, DL) && - !isAssumeLikeIntrinsic(I)) + if (!isSafeToSpeculativelyExecute(I) && !isAssumeLikeIntrinsic(I)) return false; return !isEphemeralValueOf(Inv, Q.CxtI); @@ -428,8 +454,7 @@ static bool isValidAssumeForContext(Value *V, const Query &Q, for (BasicBlock::const_iterator I = std::next(BasicBlock::const_iterator(Q.CxtI)), IE(Inv); I != IE; ++I) - if (!isSafeToSpeculativelyExecute(I, DL) && - !isAssumeLikeIntrinsic(I)) + if (!isSafeToSpeculativelyExecute(I) && !isAssumeLikeIntrinsic(I)) return false; return !isEphemeralValueOf(Inv, Q.CxtI); @@ -440,10 +465,9 @@ static bool isValidAssumeForContext(Value *V, const Query &Q, bool llvm::isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, - const DataLayout *DL, const DominatorTree *DT) { - return ::isValidAssumeForContext(const_cast(I), - Query(nullptr, CxtI, DT), DL); + return ::isValidAssumeForContext(const_cast(I), + Query(nullptr, CxtI, DT)); } template @@ -474,9 +498,181 @@ m_c_Xor(const LHS &L, const RHS &R) { return m_CombineOr(m_Xor(L, R), m_Xor(R, L)); } +/// Compute known bits in 'V' under the assumption that the condition 'Cmp' is +/// true (at the context instruction.) This is mostly a utility function for +/// the prototype dominating conditions reasoning below. +static void computeKnownBitsFromTrueCondition(Value *V, ICmpInst *Cmp, + APInt &KnownZero, + APInt &KnownOne, + const DataLayout &DL, + unsigned Depth, const Query &Q) { + Value *LHS = Cmp->getOperand(0); + Value *RHS = Cmp->getOperand(1); + // TODO: We could potentially be more aggressive here. This would be worth + // evaluating. If we can, explore commoning this code with the assume + // handling logic. + if (LHS != V && RHS != V) + return; + + const unsigned BitWidth = KnownZero.getBitWidth(); + + switch (Cmp->getPredicate()) { + default: + // We know nothing from this condition + break; + // TODO: implement unsigned bound from below (known one bits) + // TODO: common condition check implementations with assumes + // TODO: implement other patterns from assume (e.g. V & B == A) + case ICmpInst::ICMP_SGT: + if (LHS == V) { + APInt KnownZeroTemp(BitWidth, 0), KnownOneTemp(BitWidth, 0); + computeKnownBits(RHS, KnownZeroTemp, KnownOneTemp, DL, Depth + 1, Q); + if (KnownOneTemp.isAllOnesValue() || KnownZeroTemp.isNegative()) { + // We know that the sign bit is zero. + KnownZero |= APInt::getSignBit(BitWidth); + } + } + break; + case ICmpInst::ICMP_EQ: + if (LHS == V) + computeKnownBits(RHS, KnownZero, KnownOne, DL, Depth + 1, Q); + else if (RHS == V) + computeKnownBits(LHS, KnownZero, KnownOne, DL, Depth + 1, Q); + else + llvm_unreachable("missing use?"); + break; + case ICmpInst::ICMP_ULE: + if (LHS == V) { + APInt KnownZeroTemp(BitWidth, 0), KnownOneTemp(BitWidth, 0); + computeKnownBits(RHS, KnownZeroTemp, KnownOneTemp, DL, Depth + 1, Q); + // The known zero bits carry over + unsigned SignBits = KnownZeroTemp.countLeadingOnes(); + KnownZero |= APInt::getHighBitsSet(BitWidth, SignBits); + } + break; + case ICmpInst::ICMP_ULT: + if (LHS == V) { + APInt KnownZeroTemp(BitWidth, 0), KnownOneTemp(BitWidth, 0); + computeKnownBits(RHS, KnownZeroTemp, KnownOneTemp, DL, Depth + 1, Q); + // Whatever high bits in rhs are zero are known to be zero (if rhs is a + // power of 2, then one more). + unsigned SignBits = KnownZeroTemp.countLeadingOnes(); + if (isKnownToBeAPowerOfTwo(RHS, false, Depth + 1, Query(Q, Cmp), DL)) + SignBits++; + KnownZero |= APInt::getHighBitsSet(BitWidth, SignBits); + } + break; + }; +} + +/// Compute known bits in 'V' from conditions which are known to be true along +/// all paths leading to the context instruction. In particular, look for +/// cases where one branch of an interesting condition dominates the context +/// instruction. This does not do general dataflow. +/// NOTE: This code is EXPERIMENTAL and currently off by default. +static void computeKnownBitsFromDominatingCondition(Value *V, APInt &KnownZero, + APInt &KnownOne, + const DataLayout &DL, + unsigned Depth, + const Query &Q) { + // Need both the dominator tree and the query location to do anything useful + if (!Q.DT || !Q.CxtI) + return; + Instruction *Cxt = const_cast(Q.CxtI); + + // Avoid useless work + if (auto VI = dyn_cast(V)) + if (VI->getParent() == Cxt->getParent()) + return; + + // Note: We currently implement two options. It's not clear which of these + // will survive long term, we need data for that. + // Option 1 - Try walking the dominator tree looking for conditions which + // might apply. This works well for local conditions (loop guards, etc..), + // but not as well for things far from the context instruction (presuming a + // low max blocks explored). If we can set an high enough limit, this would + // be all we need. + // Option 2 - We restrict out search to those conditions which are uses of + // the value we're interested in. This is independent of dom structure, + // but is slightly less powerful without looking through lots of use chains. + // It does handle conditions far from the context instruction (e.g. early + // function exits on entry) really well though. + + // Option 1 - Search the dom tree + unsigned NumBlocksExplored = 0; + BasicBlock *Current = Cxt->getParent(); + while (true) { + // Stop searching if we've gone too far up the chain + if (NumBlocksExplored >= DomConditionsMaxDomBlocks) + break; + NumBlocksExplored++; + + if (!Q.DT->getNode(Current)->getIDom()) + break; + Current = Q.DT->getNode(Current)->getIDom()->getBlock(); + if (!Current) + // found function entry + break; + + BranchInst *BI = dyn_cast(Current->getTerminator()); + if (!BI || BI->isUnconditional()) + continue; + ICmpInst *Cmp = dyn_cast(BI->getCondition()); + if (!Cmp) + continue; + + // We're looking for conditions that are guaranteed to hold at the context + // instruction. Finding a condition where one path dominates the context + // isn't enough because both the true and false cases could merge before + // the context instruction we're actually interested in. Instead, we need + // to ensure that the taken *edge* dominates the context instruction. + BasicBlock *BB0 = BI->getSuccessor(0); + BasicBlockEdge Edge(BI->getParent(), BB0); + if (!Edge.isSingleEdge() || !Q.DT->dominates(Edge, Q.CxtI->getParent())) + continue; + + computeKnownBitsFromTrueCondition(V, Cmp, KnownZero, KnownOne, DL, Depth, + Q); + } + + // Option 2 - Search the other uses of V + unsigned NumUsesExplored = 0; + for (auto U : V->users()) { + // Avoid massive lists + if (NumUsesExplored >= DomConditionsMaxUses) + break; + NumUsesExplored++; + // Consider only compare instructions uniquely controlling a branch + ICmpInst *Cmp = dyn_cast(U); + if (!Cmp) + continue; + + if (DomConditionsSingleCmpUse && !Cmp->hasOneUse()) + continue; + + for (auto *CmpU : Cmp->users()) { + BranchInst *BI = dyn_cast(CmpU); + if (!BI || BI->isUnconditional()) + continue; + // We're looking for conditions that are guaranteed to hold at the + // context instruction. Finding a condition where one path dominates + // the context isn't enough because both the true and false cases could + // merge before the context instruction we're actually interested in. + // Instead, we need to ensure that the taken *edge* dominates the context + // instruction. + BasicBlock *BB0 = BI->getSuccessor(0); + BasicBlockEdge Edge(BI->getParent(), BB0); + if (!Edge.isSingleEdge() || !Q.DT->dominates(Edge, Q.CxtI->getParent())) + continue; + + computeKnownBitsFromTrueCondition(V, Cmp, KnownZero, KnownOne, DL, Depth, + Q); + } + } +} + static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, - APInt &KnownOne, - const DataLayout *DL, + APInt &KnownOne, const DataLayout &DL, unsigned Depth, const Query &Q) { // Use of assumptions is context-sensitive. If we don't have a context, we // cannot use them! @@ -504,8 +700,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, Value *Arg = I->getArgOperand(0); - if (Arg == V && - isValidAssumeForContext(I, Q, DL)) { + if (Arg == V && isValidAssumeForContext(I, Q)) { assert(BitWidth == 1 && "assume operand is not i1?"); KnownZero.clearAllBits(); KnownOne.setAllBits(); @@ -525,15 +720,15 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, ConstantInt *C; // assume(v = a) if (match(Arg, m_c_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); KnownZero |= RHSKnownZero; KnownOne |= RHSKnownOne; // assume(v & b = a) - } else if (match(Arg, m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + } else if (match(Arg, + m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) && + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); APInt MaskKnownZero(BitWidth, 0), MaskKnownOne(BitWidth, 0); @@ -546,7 +741,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // assume(~(v & b) = a) } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); APInt MaskKnownZero(BitWidth, 0), MaskKnownOne(BitWidth, 0); @@ -557,9 +752,9 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, KnownZero |= RHSKnownOne & MaskKnownOne; KnownOne |= RHSKnownZero & MaskKnownOne; // assume(v | b = a) - } else if (match(Arg, m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + } else if (match(Arg, + m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) && + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0); @@ -572,7 +767,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // assume(~(v | b) = a) } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0); @@ -583,9 +778,9 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, KnownZero |= RHSKnownOne & BKnownZero; KnownOne |= RHSKnownZero & BKnownZero; // assume(v ^ b = a) - } else if (match(Arg, m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + } else if (match(Arg, + m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) && + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0); @@ -601,7 +796,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // assume(~(v ^ b) = a) } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0); @@ -617,7 +812,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // assume(v << c = a) } else if (match(Arg, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); // For those bits in RHS that are known, we can propagate them to known @@ -627,7 +822,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // assume(~(v << c) = a) } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); // For those bits in RHS that are known, we can propagate them inverted @@ -637,10 +832,9 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // assume(v >> c = a) } else if (match(Arg, m_c_ICmp(Pred, m_CombineOr(m_LShr(m_V, m_ConstantInt(C)), - m_AShr(m_V, - m_ConstantInt(C))), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + m_AShr(m_V, m_ConstantInt(C))), + m_Value(A))) && + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); // For those bits in RHS that are known, we can propagate them to known @@ -649,10 +843,10 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, KnownOne |= RHSKnownOne << C->getZExtValue(); // assume(~(v >> c) = a) } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_CombineOr( - m_LShr(m_V, m_ConstantInt(C)), - m_AShr(m_V, m_ConstantInt(C)))), + m_LShr(m_V, m_ConstantInt(C)), + m_AShr(m_V, m_ConstantInt(C)))), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); // For those bits in RHS that are known, we can propagate them inverted @@ -661,8 +855,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, KnownOne |= RHSKnownZero << C->getZExtValue(); // assume(v >=_s c) where c is non-negative } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SGE && - isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_SGE && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); @@ -672,8 +865,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, } // assume(v >_s c) where c is at least -1. } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SGT && - isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_SGT && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); @@ -683,8 +875,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, } // assume(v <=_s c) where c is negative } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SLE && - isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_SLE && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); @@ -694,8 +885,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, } // assume(v <_s c) where c is non-positive } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SLT && - isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_SLT && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); @@ -705,8 +895,7 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, } // assume(v <=_u c) } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_ULE && - isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_ULE && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); @@ -715,14 +904,13 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, APInt::getHighBitsSet(BitWidth, RHSKnownZero.countLeadingOnes()); // assume(v <_u c) } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_ULT && - isValidAssumeForContext(I, Q, DL)) { + Pred == ICmpInst::ICMP_ULT && isValidAssumeForContext(I, Q)) { APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0); computeKnownBits(A, RHSKnownZero, RHSKnownOne, DL, Depth+1, Query(Q, I)); // Whatever high bits in c are zero are known to be zero (if c is a power // of 2, then one more). - if (isKnownToBeAPowerOfTwo(A, false, Depth+1, Query(Q, I))) + if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, I), DL)) KnownZero |= APInt::getHighBitsSet(BitWidth, RHSKnownZero.countLeadingOnes()+1); else @@ -743,13 +931,12 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, /// this won't lose us code quality. /// /// This function is defined on values with integer type, values with pointer -/// type (but only if TD is non-null), and vectors of integers. In the case +/// type, and vectors of integers. In the case /// where V is a vector, known zero, and known one values are the /// same width as the vector element, and the bit is set only if it is true /// for all of the elements in the vector. void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, - const DataLayout *TD, unsigned Depth, - const Query &Q) { + const DataLayout &DL, unsigned Depth, const Query &Q) { assert(V && "No Value?"); assert(Depth <= MaxDepth && "Limit Search Depth"); unsigned BitWidth = KnownZero.getBitWidth(); @@ -757,8 +944,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, assert((V->getType()->isIntOrIntVectorTy() || V->getType()->getScalarType()->isPointerTy()) && "Not integer or pointer type!"); - assert((!TD || - TD->getTypeSizeInBits(V->getType()->getScalarType()) == BitWidth) && + assert((DL.getTypeSizeInBits(V->getType()->getScalarType()) == BitWidth) && (!V->getType()->isIntOrIntVectorTy() || V->getType()->getScalarSizeInBits() == BitWidth) && KnownZero.getBitWidth() == BitWidth && @@ -797,7 +983,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // The address of an aligned GlobalValue has trailing zeros. if (auto *GO = dyn_cast(V)) { unsigned Align = GO->getAlignment(); - if (Align == 0 && TD) { + if (Align == 0) { if (auto *GVar = dyn_cast(GO)) { Type *ObjectType = GVar->getType()->getElementType(); if (ObjectType->isSized()) { @@ -805,9 +991,9 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // it the preferred alignment. Otherwise, we have to assume that it // may only have the minimum ABI alignment. if (!GVar->isDeclaration() && !GVar->isWeakForLinker()) - Align = TD->getPreferredAlignment(GVar); + Align = DL.getPreferredAlignment(GVar); else - Align = TD->getABITypeAlignment(ObjectType); + Align = DL.getABITypeAlignment(ObjectType); } } } @@ -823,11 +1009,11 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, if (Argument *A = dyn_cast(V)) { unsigned Align = A->getType()->isPointerTy() ? A->getParamAlignment() : 0; - if (!Align && TD && A->hasStructRetAttr()) { + if (!Align && A->hasStructRetAttr()) { // An sret parameter has at least the ABI alignment of the return type. Type *EltTy = cast(A->getType())->getElementType(); if (EltTy->isSized()) - Align = TD->getABITypeAlignment(EltTy); + Align = DL.getABITypeAlignment(EltTy); } if (Align) @@ -838,7 +1024,12 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // Don't give up yet... there might be an assumption that provides more // information... - computeKnownBitsFromAssume(V, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBitsFromAssume(V, KnownZero, KnownOne, DL, Depth, Q); + + // Or a dominating condition for that matter + if (EnableDomConditions && Depth <= DomConditionsMaxDepth) + computeKnownBitsFromDominatingCondition(V, KnownZero, KnownOne, DL, + Depth, Q); return; } @@ -854,12 +1045,18 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // the bits of its aliasee. if (GlobalAlias *GA = dyn_cast(V)) { if (!GA->mayBeOverridden()) - computeKnownBits(GA->getAliasee(), KnownZero, KnownOne, TD, Depth + 1, Q); + computeKnownBits(GA->getAliasee(), KnownZero, KnownOne, DL, Depth + 1, Q); return; } // Check whether a nearby assume intrinsic can determine some known bits. - computeKnownBitsFromAssume(V, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBitsFromAssume(V, KnownZero, KnownOne, DL, Depth, Q); + + // Check whether there's a dominating condition which implies something about + // this value at the given context. + if (EnableDomConditions && Depth <= DomConditionsMaxDepth) + computeKnownBitsFromDominatingCondition(V, KnownZero, KnownOne, DL, Depth, + Q); Operator *I = dyn_cast(V); if (!I) return; @@ -873,8 +1070,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, break; case Instruction::And: { // If either the LHS or the RHS are Zero, the result is zero. - computeKnownBits(I->getOperand(1), KnownZero, KnownOne, TD, Depth+1, Q); - computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(1), KnownZero, KnownOne, DL, Depth + 1, Q); + computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, DL, Depth + 1, Q); // Output known-1 bits are only known if set in both the LHS & RHS. KnownOne &= KnownOne2; @@ -883,8 +1080,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, break; } case Instruction::Or: { - computeKnownBits(I->getOperand(1), KnownZero, KnownOne, TD, Depth+1, Q); - computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(1), KnownZero, KnownOne, DL, Depth + 1, Q); + computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, DL, Depth + 1, Q); // Output known-0 bits are only known if clear in both the LHS & RHS. KnownZero &= KnownZero2; @@ -893,8 +1090,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, break; } case Instruction::Xor: { - computeKnownBits(I->getOperand(1), KnownZero, KnownOne, TD, Depth+1, Q); - computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(1), KnownZero, KnownOne, DL, Depth + 1, Q); + computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, DL, Depth + 1, Q); // Output known-0 bits are known if clear or set in both the LHS & RHS. APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); @@ -905,21 +1102,20 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, } case Instruction::Mul: { bool NSW = cast(I)->hasNoSignedWrap(); - computeKnownBitsMul(I->getOperand(0), I->getOperand(1), NSW, - KnownZero, KnownOne, KnownZero2, KnownOne2, TD, - Depth, Q); + computeKnownBitsMul(I->getOperand(0), I->getOperand(1), NSW, KnownZero, + KnownOne, KnownZero2, KnownOne2, DL, Depth, Q); break; } case Instruction::UDiv: { // For the purposes of computing leading zeros we can conservatively // treat a udiv as a logical right shift by the power of 2 known to // be less than the denominator. - computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, DL, Depth + 1, Q); unsigned LeadZ = KnownZero2.countLeadingOnes(); KnownOne2.clearAllBits(); KnownZero2.clearAllBits(); - computeKnownBits(I->getOperand(1), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(1), KnownZero2, KnownOne2, DL, Depth + 1, Q); unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); if (RHSUnknownLeadingOnes != BitWidth) LeadZ = std::min(BitWidth, @@ -929,8 +1125,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, break; } case Instruction::Select: - computeKnownBits(I->getOperand(2), KnownZero, KnownOne, TD, Depth+1, Q); - computeKnownBits(I->getOperand(1), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(2), KnownZero, KnownOne, DL, Depth + 1, Q); + computeKnownBits(I->getOperand(1), KnownZero2, KnownOne2, DL, Depth + 1, Q); // Only known if known in both the LHS and RHS. KnownOne &= KnownOne2; @@ -946,8 +1142,6 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, case Instruction::PtrToInt: case Instruction::IntToPtr: case Instruction::AddrSpaceCast: // Pointers could be different sizes. - // We can't handle these if we don't know the pointer size. - if (!TD) break; // FALL THROUGH and handle them the same as zext/trunc. case Instruction::ZExt: case Instruction::Trunc: { @@ -956,17 +1150,12 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, unsigned SrcBitWidth; // Note that we handle pointer operands here because of inttoptr/ptrtoint // which fall through here. - if(TD) { - SrcBitWidth = TD->getTypeSizeInBits(SrcTy->getScalarType()); - } else { - SrcBitWidth = SrcTy->getScalarSizeInBits(); - if (!SrcBitWidth) break; - } + SrcBitWidth = DL.getTypeSizeInBits(SrcTy->getScalarType()); assert(SrcBitWidth && "SrcBitWidth can't be zero"); KnownZero = KnownZero.zextOrTrunc(SrcBitWidth); KnownOne = KnownOne.zextOrTrunc(SrcBitWidth); - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); KnownZero = KnownZero.zextOrTrunc(BitWidth); KnownOne = KnownOne.zextOrTrunc(BitWidth); // Any top bits are known to be zero. @@ -980,7 +1169,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // TODO: For now, not handling conversions like: // (bitcast i64 %x to <2 x i32>) !I->getType()->isVectorTy()) { - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); break; } break; @@ -991,7 +1180,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, KnownZero = KnownZero.trunc(SrcBitWidth); KnownOne = KnownOne.trunc(SrcBitWidth); - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); KnownZero = KnownZero.zext(BitWidth); KnownOne = KnownOne.zext(BitWidth); @@ -1007,7 +1196,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 if (ConstantInt *SA = dyn_cast(I->getOperand(1))) { uint64_t ShiftAmt = SA->getLimitedValue(BitWidth); - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); KnownZero <<= ShiftAmt; KnownOne <<= ShiftAmt; KnownZero |= APInt::getLowBitsSet(BitWidth, ShiftAmt); // low bits known 0 @@ -1020,7 +1209,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, uint64_t ShiftAmt = SA->getLimitedValue(BitWidth); // Unsigned shift right. - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); KnownZero = APIntOps::lshr(KnownZero, ShiftAmt); KnownOne = APIntOps::lshr(KnownOne, ShiftAmt); // high bits known zero. @@ -1034,7 +1223,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, uint64_t ShiftAmt = SA->getLimitedValue(BitWidth-1); // Signed shift right. - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); KnownZero = APIntOps::lshr(KnownZero, ShiftAmt); KnownOne = APIntOps::lshr(KnownOne, ShiftAmt); @@ -1048,15 +1237,15 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, case Instruction::Sub: { bool NSW = cast(I)->hasNoSignedWrap(); computeKnownBitsAddSub(false, I->getOperand(0), I->getOperand(1), NSW, - KnownZero, KnownOne, KnownZero2, KnownOne2, TD, - Depth, Q); + KnownZero, KnownOne, KnownZero2, KnownOne2, DL, + Depth, Q); break; } case Instruction::Add: { bool NSW = cast(I)->hasNoSignedWrap(); computeKnownBitsAddSub(true, I->getOperand(0), I->getOperand(1), NSW, - KnownZero, KnownOne, KnownZero2, KnownOne2, TD, - Depth, Q); + KnownZero, KnownOne, KnownZero2, KnownOne2, DL, + Depth, Q); break; } case Instruction::SRem: @@ -1064,8 +1253,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, APInt RA = Rem->getValue().abs(); if (RA.isPowerOf2()) { APInt LowBits = RA - 1; - computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, TD, - Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero2, KnownOne2, DL, Depth + 1, + Q); // The low bits of the first operand are unchanged by the srem. KnownZero = KnownZero2 & LowBits; @@ -1089,8 +1278,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // remainder is zero. if (KnownZero.isNonNegative()) { APInt LHSKnownZero(BitWidth, 0), LHSKnownOne(BitWidth, 0); - computeKnownBits(I->getOperand(0), LHSKnownZero, LHSKnownOne, TD, - Depth+1, Q); + computeKnownBits(I->getOperand(0), LHSKnownZero, LHSKnownOne, DL, + Depth + 1, Q); // If it's known zero, our sign bit is also zero. if (LHSKnownZero.isNegative()) KnownZero.setBit(BitWidth - 1); @@ -1102,8 +1291,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, APInt RA = Rem->getValue(); if (RA.isPowerOf2()) { APInt LowBits = (RA - 1); - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, - Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, + Q); KnownZero |= ~LowBits; KnownOne &= LowBits; break; @@ -1112,8 +1301,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // Since the result is less than or equal to either operand, any leading // zero bits in either operand must also exist in the result. - computeKnownBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); - computeKnownBits(I->getOperand(1), KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(I->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, Q); + computeKnownBits(I->getOperand(1), KnownZero2, KnownOne2, DL, Depth + 1, Q); unsigned Leaders = std::max(KnownZero.countLeadingOnes(), KnownZero2.countLeadingOnes()); @@ -1125,8 +1314,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, case Instruction::Alloca: { AllocaInst *AI = cast(V); unsigned Align = AI->getAlignment(); - if (Align == 0 && TD) - Align = TD->getABITypeAlignment(AI->getType()->getElementType()); + if (Align == 0) + Align = DL.getABITypeAlignment(AI->getType()->getElementType()); if (Align > 0) KnownZero = APInt::getLowBitsSet(BitWidth, countTrailingZeros(Align)); @@ -1136,8 +1325,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, // Analyze all of the subscripts of this getelementptr instruction // to determine if we can prove known low zero bits. APInt LocalKnownZero(BitWidth, 0), LocalKnownOne(BitWidth, 0); - computeKnownBits(I->getOperand(0), LocalKnownZero, LocalKnownOne, TD, - Depth+1, Q); + computeKnownBits(I->getOperand(0), LocalKnownZero, LocalKnownOne, DL, + Depth + 1, Q); unsigned TrailZ = LocalKnownZero.countTrailingOnes(); gep_type_iterator GTI = gep_type_begin(I); @@ -1145,10 +1334,6 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, Value *Index = I->getOperand(i); if (StructType *STy = dyn_cast(*GTI)) { // Handle struct member offset arithmetic. - if (!TD) { - TrailZ = 0; - break; - } // Handle case when index is vector zeroinitializer Constant *CIndex = cast(Index); @@ -1159,7 +1344,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, Index = CIndex->getSplatValue(); unsigned Idx = cast(Index)->getZExtValue(); - const StructLayout *SL = TD->getStructLayout(STy); + const StructLayout *SL = DL.getStructLayout(STy); uint64_t Offset = SL->getElementOffset(Idx); TrailZ = std::min(TrailZ, countTrailingZeros(Offset)); @@ -1171,9 +1356,10 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, break; } unsigned GEPOpiBits = Index->getType()->getScalarSizeInBits(); - uint64_t TypeSize = TD ? TD->getTypeAllocSize(IndexedTy) : 1; + uint64_t TypeSize = DL.getTypeAllocSize(IndexedTy); LocalKnownZero = LocalKnownOne = APInt(GEPOpiBits, 0); - computeKnownBits(Index, LocalKnownZero, LocalKnownOne, TD, Depth+1, Q); + computeKnownBits(Index, LocalKnownZero, LocalKnownOne, DL, Depth + 1, + Q); TrailZ = std::min(TrailZ, unsigned(countTrailingZeros(TypeSize) + LocalKnownZero.countTrailingOnes())); @@ -1215,11 +1401,11 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, break; // Ok, we have a PHI of the form L op= R. Check for low // zero bits. - computeKnownBits(R, KnownZero2, KnownOne2, TD, Depth+1, Q); + computeKnownBits(R, KnownZero2, KnownOne2, DL, Depth + 1, Q); // We need to take the minimum number of known bits APInt KnownZero3(KnownZero), KnownOne3(KnownOne); - computeKnownBits(L, KnownZero3, KnownOne3, TD, Depth+1, Q); + computeKnownBits(L, KnownZero3, KnownOne3, DL, Depth + 1, Q); KnownZero = APInt::getLowBitsSet(BitWidth, std::min(KnownZero2.countTrailingOnes(), @@ -1250,8 +1436,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, KnownOne2 = APInt(BitWidth, 0); // Recurse, but cap the recursion to one level, because we don't // want to waste time spinning around in loops. - computeKnownBits(P->getIncomingValue(i), KnownZero2, KnownOne2, TD, - MaxDepth-1, Q); + computeKnownBits(P->getIncomingValue(i), KnownZero2, KnownOne2, DL, + MaxDepth - 1, Q); KnownZero &= KnownZero2; KnownOne &= KnownOne2; // If all bits have been ruled out, there's no need to check @@ -1303,19 +1489,19 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, case Intrinsic::sadd_with_overflow: computeKnownBitsAddSub(true, II->getArgOperand(0), II->getArgOperand(1), false, KnownZero, - KnownOne, KnownZero2, KnownOne2, TD, Depth, Q); + KnownOne, KnownZero2, KnownOne2, DL, Depth, Q); break; case Intrinsic::usub_with_overflow: case Intrinsic::ssub_with_overflow: computeKnownBitsAddSub(false, II->getArgOperand(0), II->getArgOperand(1), false, KnownZero, - KnownOne, KnownZero2, KnownOne2, TD, Depth, Q); + KnownOne, KnownZero2, KnownOne2, DL, Depth, Q); break; case Intrinsic::umul_with_overflow: case Intrinsic::smul_with_overflow: - computeKnownBitsMul(II->getArgOperand(0), II->getArgOperand(1), - false, KnownZero, KnownOne, - KnownZero2, KnownOne2, TD, Depth, Q); + computeKnownBitsMul(II->getArgOperand(0), II->getArgOperand(1), false, + KnownZero, KnownOne, KnownZero2, KnownOne2, DL, + Depth, Q); break; } } @@ -1328,9 +1514,8 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, /// Determine whether the sign bit is known to be zero or one. /// Convenience wrapper around computeKnownBits. void ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne, - const DataLayout *TD, unsigned Depth, - const Query &Q) { - unsigned BitWidth = getBitWidth(V->getType(), TD); + const DataLayout &DL, unsigned Depth, const Query &Q) { + unsigned BitWidth = getBitWidth(V->getType(), DL); if (!BitWidth) { KnownZero = false; KnownOne = false; @@ -1338,7 +1523,7 @@ void ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne, } APInt ZeroBits(BitWidth, 0); APInt OneBits(BitWidth, 0); - computeKnownBits(V, ZeroBits, OneBits, TD, Depth, Q); + computeKnownBits(V, ZeroBits, OneBits, DL, Depth, Q); KnownOne = OneBits[BitWidth - 1]; KnownZero = ZeroBits[BitWidth - 1]; } @@ -1348,7 +1533,7 @@ void ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne, /// be a power of two when defined. Supports values with integer or pointer /// types and vectors of integers. bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, - const Query &Q) { + const Query &Q, const DataLayout &DL) { if (Constant *C = dyn_cast(V)) { if (C->isNullValue()) return OrZero; @@ -1375,20 +1560,19 @@ bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, // A shift of a power of two is a power of two or zero. if (OrZero && (match(V, m_Shl(m_Value(X), m_Value())) || match(V, m_Shr(m_Value(X), m_Value())))) - return isKnownToBeAPowerOfTwo(X, /*OrZero*/true, Depth, Q); + return isKnownToBeAPowerOfTwo(X, /*OrZero*/ true, Depth, Q, DL); if (ZExtInst *ZI = dyn_cast(V)) - return isKnownToBeAPowerOfTwo(ZI->getOperand(0), OrZero, Depth, Q); + return isKnownToBeAPowerOfTwo(ZI->getOperand(0), OrZero, Depth, Q, DL); if (SelectInst *SI = dyn_cast(V)) - return - isKnownToBeAPowerOfTwo(SI->getTrueValue(), OrZero, Depth, Q) && - isKnownToBeAPowerOfTwo(SI->getFalseValue(), OrZero, Depth, Q); + return isKnownToBeAPowerOfTwo(SI->getTrueValue(), OrZero, Depth, Q, DL) && + isKnownToBeAPowerOfTwo(SI->getFalseValue(), OrZero, Depth, Q, DL); if (OrZero && match(V, m_And(m_Value(X), m_Value(Y)))) { // A power of two and'd with anything is a power of two or zero. - if (isKnownToBeAPowerOfTwo(X, /*OrZero*/true, Depth, Q) || - isKnownToBeAPowerOfTwo(Y, /*OrZero*/true, Depth, Q)) + if (isKnownToBeAPowerOfTwo(X, /*OrZero*/ true, Depth, Q, DL) || + isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, Depth, Q, DL)) return true; // X & (-X) is always a power of two or zero. if (match(X, m_Neg(m_Specific(Y))) || match(Y, m_Neg(m_Specific(X)))) @@ -1403,19 +1587,19 @@ bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, if (OrZero || VOBO->hasNoUnsignedWrap() || VOBO->hasNoSignedWrap()) { if (match(X, m_And(m_Specific(Y), m_Value())) || match(X, m_And(m_Value(), m_Specific(Y)))) - if (isKnownToBeAPowerOfTwo(Y, OrZero, Depth, Q)) + if (isKnownToBeAPowerOfTwo(Y, OrZero, Depth, Q, DL)) return true; if (match(Y, m_And(m_Specific(X), m_Value())) || match(Y, m_And(m_Value(), m_Specific(X)))) - if (isKnownToBeAPowerOfTwo(X, OrZero, Depth, Q)) + if (isKnownToBeAPowerOfTwo(X, OrZero, Depth, Q, DL)) return true; unsigned BitWidth = V->getType()->getScalarSizeInBits(); APInt LHSZeroBits(BitWidth, 0), LHSOneBits(BitWidth, 0); - computeKnownBits(X, LHSZeroBits, LHSOneBits, nullptr, Depth, Q); + computeKnownBits(X, LHSZeroBits, LHSOneBits, DL, Depth, Q); APInt RHSZeroBits(BitWidth, 0), RHSOneBits(BitWidth, 0); - computeKnownBits(Y, RHSZeroBits, RHSOneBits, nullptr, Depth, Q); + computeKnownBits(Y, RHSZeroBits, RHSOneBits, DL, Depth, Q); // If i8 V is a power of two or zero: // ZeroBits: 1 1 1 0 1 1 1 1 // ~ZeroBits: 0 0 0 1 0 0 0 0 @@ -1433,7 +1617,7 @@ bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, if (match(V, m_Exact(m_LShr(m_Value(), m_Value()))) || match(V, m_Exact(m_UDiv(m_Value(), m_Value())))) { return isKnownToBeAPowerOfTwo(cast(V)->getOperand(0), OrZero, - Depth, Q); + Depth, Q, DL); } return false; @@ -1445,7 +1629,7 @@ bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero, unsigned Depth, /// to be non-null. /// /// Currently this routine does not support vector GEPs. -static bool isGEPKnownNonNull(GEPOperator *GEP, const DataLayout *DL, +static bool isGEPKnownNonNull(GEPOperator *GEP, const DataLayout &DL, unsigned Depth, const Query &Q) { if (!GEP->isInBounds() || GEP->getPointerAddressSpace() != 0) return false; @@ -1458,10 +1642,6 @@ static bool isGEPKnownNonNull(GEPOperator *GEP, const DataLayout *DL, if (isKnownNonZero(GEP->getPointerOperand(), DL, Depth, Q)) return true; - // Past this, if we don't have DataLayout, we can't do much. - if (!DL) - return false; - // Walk the GEP operands and see if any operand introduces a non-zero offset. // If so, then the GEP cannot produce a null pointer, as doing so would // inherently violate the inbounds contract within address space zero. @@ -1471,7 +1651,7 @@ static bool isGEPKnownNonNull(GEPOperator *GEP, const DataLayout *DL, if (StructType *STy = dyn_cast(*GTI)) { ConstantInt *OpC = cast(GTI.getOperand()); unsigned ElementIdx = OpC->getZExtValue(); - const StructLayout *SL = DL->getStructLayout(STy); + const StructLayout *SL = DL.getStructLayout(STy); uint64_t ElementOffset = SL->getElementOffset(ElementIdx); if (ElementOffset > 0) return true; @@ -1479,7 +1659,7 @@ static bool isGEPKnownNonNull(GEPOperator *GEP, const DataLayout *DL, } // If we have a zero-sized type, the index doesn't matter. Keep looping. - if (DL->getTypeAllocSize(GTI.getIndexedType()) == 0) + if (DL.getTypeAllocSize(GTI.getIndexedType()) == 0) continue; // Fast path the constant operand case both for efficiency and so we don't @@ -1528,7 +1708,7 @@ static bool rangeMetadataExcludesValue(MDNode* Ranges, /// For vectors return true if every element is known to be non-zero when /// defined. Supports values with integer or pointer type and vectors of /// integers. -bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, +bool isKnownNonZero(Value *V, const DataLayout &DL, unsigned Depth, const Query &Q) { if (Constant *C = dyn_cast(V)) { if (C->isNullValue()) @@ -1561,21 +1741,20 @@ bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, if (isKnownNonNull(V)) return true; if (GEPOperator *GEP = dyn_cast(V)) - if (isGEPKnownNonNull(GEP, TD, Depth, Q)) + if (isGEPKnownNonNull(GEP, DL, Depth, Q)) return true; } - unsigned BitWidth = getBitWidth(V->getType()->getScalarType(), TD); + unsigned BitWidth = getBitWidth(V->getType()->getScalarType(), DL); // X | Y != 0 if X != 0 or Y != 0. Value *X = nullptr, *Y = nullptr; if (match(V, m_Or(m_Value(X), m_Value(Y)))) - return isKnownNonZero(X, TD, Depth, Q) || - isKnownNonZero(Y, TD, Depth, Q); + return isKnownNonZero(X, DL, Depth, Q) || isKnownNonZero(Y, DL, Depth, Q); // ext X != 0 if X != 0. if (isa(V) || isa(V)) - return isKnownNonZero(cast(V)->getOperand(0), TD, Depth, Q); + return isKnownNonZero(cast(V)->getOperand(0), DL, Depth, Q); // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined // if the lowest bit is shifted off the end. @@ -1583,11 +1762,11 @@ bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, // shl nuw can't remove any non-zero bits. OverflowingBinaryOperator *BO = cast(V); if (BO->hasNoUnsignedWrap()) - return isKnownNonZero(X, TD, Depth, Q); + return isKnownNonZero(X, DL, Depth, Q); APInt KnownZero(BitWidth, 0); APInt KnownOne(BitWidth, 0); - computeKnownBits(X, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBits(X, KnownZero, KnownOne, DL, Depth, Q); if (KnownOne[0]) return true; } @@ -1597,29 +1776,28 @@ bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, // shr exact can only shift out zero bits. PossiblyExactOperator *BO = cast(V); if (BO->isExact()) - return isKnownNonZero(X, TD, Depth, Q); + return isKnownNonZero(X, DL, Depth, Q); bool XKnownNonNegative, XKnownNegative; - ComputeSignBit(X, XKnownNonNegative, XKnownNegative, TD, Depth, Q); + ComputeSignBit(X, XKnownNonNegative, XKnownNegative, DL, Depth, Q); if (XKnownNegative) return true; } // div exact can only produce a zero if the dividend is zero. else if (match(V, m_Exact(m_IDiv(m_Value(X), m_Value())))) { - return isKnownNonZero(X, TD, Depth, Q); + return isKnownNonZero(X, DL, Depth, Q); } // X + Y. else if (match(V, m_Add(m_Value(X), m_Value(Y)))) { bool XKnownNonNegative, XKnownNegative; bool YKnownNonNegative, YKnownNegative; - ComputeSignBit(X, XKnownNonNegative, XKnownNegative, TD, Depth, Q); - ComputeSignBit(Y, YKnownNonNegative, YKnownNegative, TD, Depth, Q); + ComputeSignBit(X, XKnownNonNegative, XKnownNegative, DL, Depth, Q); + ComputeSignBit(Y, YKnownNonNegative, YKnownNegative, DL, Depth, Q); // If X and Y are both non-negative (as signed values) then their sum is not // zero unless both X and Y are zero. if (XKnownNonNegative && YKnownNonNegative) - if (isKnownNonZero(X, TD, Depth, Q) || - isKnownNonZero(Y, TD, Depth, Q)) + if (isKnownNonZero(X, DL, Depth, Q) || isKnownNonZero(Y, DL, Depth, Q)) return true; // If X and Y are both negative (as signed values) then their sum is not @@ -1630,22 +1808,22 @@ bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, APInt Mask = APInt::getSignedMaxValue(BitWidth); // The sign bit of X is set. If some other bit is set then X is not equal // to INT_MIN. - computeKnownBits(X, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBits(X, KnownZero, KnownOne, DL, Depth, Q); if ((KnownOne & Mask) != 0) return true; // The sign bit of Y is set. If some other bit is set then Y is not equal // to INT_MIN. - computeKnownBits(Y, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBits(Y, KnownZero, KnownOne, DL, Depth, Q); if ((KnownOne & Mask) != 0) return true; } // The sum of a non-negative number and a power of two is not zero. if (XKnownNonNegative && - isKnownToBeAPowerOfTwo(Y, /*OrZero*/false, Depth, Q)) + isKnownToBeAPowerOfTwo(Y, /*OrZero*/ false, Depth, Q, DL)) return true; if (YKnownNonNegative && - isKnownToBeAPowerOfTwo(X, /*OrZero*/false, Depth, Q)) + isKnownToBeAPowerOfTwo(X, /*OrZero*/ false, Depth, Q, DL)) return true; } // X * Y. @@ -1654,21 +1832,20 @@ bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, // If X and Y are non-zero then so is X * Y as long as the multiplication // does not overflow. if ((BO->hasNoSignedWrap() || BO->hasNoUnsignedWrap()) && - isKnownNonZero(X, TD, Depth, Q) && - isKnownNonZero(Y, TD, Depth, Q)) + isKnownNonZero(X, DL, Depth, Q) && isKnownNonZero(Y, DL, Depth, Q)) return true; } // (C ? X : Y) != 0 if X != 0 and Y != 0. else if (SelectInst *SI = dyn_cast(V)) { - if (isKnownNonZero(SI->getTrueValue(), TD, Depth, Q) && - isKnownNonZero(SI->getFalseValue(), TD, Depth, Q)) + if (isKnownNonZero(SI->getTrueValue(), DL, Depth, Q) && + isKnownNonZero(SI->getFalseValue(), DL, Depth, Q)) return true; } if (!BitWidth) return false; APInt KnownZero(BitWidth, 0); APInt KnownOne(BitWidth, 0); - computeKnownBits(V, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBits(V, KnownZero, KnownOne, DL, Depth, Q); return KnownOne != 0; } @@ -1677,15 +1854,14 @@ bool isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth, /// cannot have. /// /// This function is defined on values with integer type, values with pointer -/// type (but only if TD is non-null), and vectors of integers. In the case +/// type, and vectors of integers. In the case /// where V is a vector, the mask, known zero, and known one values are the /// same width as the vector element, and the bit is set only if it is true /// for all of the elements in the vector. -bool MaskedValueIsZero(Value *V, const APInt &Mask, - const DataLayout *TD, unsigned Depth, - const Query &Q) { +bool MaskedValueIsZero(Value *V, const APInt &Mask, const DataLayout &DL, + unsigned Depth, const Query &Q) { APInt KnownZero(Mask.getBitWidth(), 0), KnownOne(Mask.getBitWidth(), 0); - computeKnownBits(V, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBits(V, KnownZero, KnownOne, DL, Depth, Q); return (KnownZero & Mask) == Mask; } @@ -1699,14 +1875,9 @@ bool MaskedValueIsZero(Value *V, const APInt &Mask, /// /// 'Op' must have a scalar integer type. /// -unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, - unsigned Depth, const Query &Q) { - assert((TD || V->getType()->isIntOrIntVectorTy()) && - "ComputeNumSignBits requires a DataLayout object to operate " - "on non-integer values!"); - Type *Ty = V->getType(); - unsigned TyBits = TD ? TD->getTypeSizeInBits(V->getType()->getScalarType()) : - Ty->getScalarSizeInBits(); +unsigned ComputeNumSignBits(Value *V, const DataLayout &DL, unsigned Depth, + const Query &Q) { + unsigned TyBits = DL.getTypeSizeInBits(V->getType()->getScalarType()); unsigned Tmp, Tmp2; unsigned FirstAnswer = 1; @@ -1721,10 +1892,63 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, default: break; case Instruction::SExt: Tmp = TyBits - U->getOperand(0)->getType()->getScalarSizeInBits(); - return ComputeNumSignBits(U->getOperand(0), TD, Depth+1, Q) + Tmp; + return ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q) + Tmp; + + case Instruction::SDiv: { + const APInt *Denominator; + // sdiv X, C -> adds log(C) sign bits. + if (match(U->getOperand(1), m_APInt(Denominator))) { + + // Ignore non-positive denominator. + if (!Denominator->isStrictlyPositive()) + break; + + // Calculate the incoming numerator bits. + unsigned NumBits = ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); + + // Add floor(log(C)) bits to the numerator bits. + return std::min(TyBits, NumBits + Denominator->logBase2()); + } + break; + } + + case Instruction::SRem: { + const APInt *Denominator; + // srem X, C -> we know that the result is within [-C+1,C) when C is a + // positive constant. This let us put a lower bound on the number of sign + // bits. + if (match(U->getOperand(1), m_APInt(Denominator))) { + + // Ignore non-positive denominator. + if (!Denominator->isStrictlyPositive()) + break; + + // Calculate the incoming numerator bits. SRem by a positive constant + // can't lower the number of sign bits. + unsigned NumrBits = + ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); + + // Calculate the leading sign bit constraints by examining the + // denominator. Given that the denominator is positive, there are two + // cases: + // + // 1. the numerator is positive. The result range is [0,C) and [0,C) u< + // (1 << ceilLogBase2(C)). + // + // 2. the numerator is negative. Then the result range is (-C,0] and + // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)). + // + // Thus a lower bound on the number of sign bits is `TyBits - + // ceilLogBase2(C)`. + + unsigned ResBits = TyBits - Denominator->ceilLogBase2(); + return std::max(NumrBits, ResBits); + } + break; + } case Instruction::AShr: { - Tmp = ComputeNumSignBits(U->getOperand(0), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); // ashr X, C -> adds C sign bits. Vectors too. const APInt *ShAmt; if (match(U->getOperand(1), m_APInt(ShAmt))) { @@ -1737,7 +1961,7 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, const APInt *ShAmt; if (match(U->getOperand(1), m_APInt(ShAmt))) { // shl destroys sign bits. - Tmp = ComputeNumSignBits(U->getOperand(0), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); Tmp2 = ShAmt->getZExtValue(); if (Tmp2 >= TyBits || // Bad shift. Tmp2 >= Tmp) break; // Shifted all sign bits out. @@ -1749,9 +1973,9 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, case Instruction::Or: case Instruction::Xor: // NOT is handled here. // Logical binary ops preserve the number of sign bits at the worst. - Tmp = ComputeNumSignBits(U->getOperand(0), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); if (Tmp != 1) { - Tmp2 = ComputeNumSignBits(U->getOperand(1), TD, Depth+1, Q); + Tmp2 = ComputeNumSignBits(U->getOperand(1), DL, Depth + 1, Q); FirstAnswer = std::min(Tmp, Tmp2); // We computed what we know about the sign bits as our first // answer. Now proceed to the generic code that uses @@ -1760,22 +1984,23 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, break; case Instruction::Select: - Tmp = ComputeNumSignBits(U->getOperand(1), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(U->getOperand(1), DL, Depth + 1, Q); if (Tmp == 1) return 1; // Early out. - Tmp2 = ComputeNumSignBits(U->getOperand(2), TD, Depth+1, Q); + Tmp2 = ComputeNumSignBits(U->getOperand(2), DL, Depth + 1, Q); return std::min(Tmp, Tmp2); case Instruction::Add: // Add can have at most one carry bit. Thus we know that the output // is, at worst, one more bit than the inputs. - Tmp = ComputeNumSignBits(U->getOperand(0), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); if (Tmp == 1) return 1; // Early out. // Special case decrementing a value (ADD X, -1): if (const auto *CRHS = dyn_cast(U->getOperand(1))) if (CRHS->isAllOnesValue()) { APInt KnownZero(TyBits, 0), KnownOne(TyBits, 0); - computeKnownBits(U->getOperand(0), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(U->getOperand(0), KnownZero, KnownOne, DL, Depth + 1, + Q); // If the input is known to be 0 or 1, the output is 0/-1, which is all // sign bits set. @@ -1788,19 +2013,20 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, return Tmp; } - Tmp2 = ComputeNumSignBits(U->getOperand(1), TD, Depth+1, Q); + Tmp2 = ComputeNumSignBits(U->getOperand(1), DL, Depth + 1, Q); if (Tmp2 == 1) return 1; return std::min(Tmp, Tmp2)-1; case Instruction::Sub: - Tmp2 = ComputeNumSignBits(U->getOperand(1), TD, Depth+1, Q); + Tmp2 = ComputeNumSignBits(U->getOperand(1), DL, Depth + 1, Q); if (Tmp2 == 1) return 1; // Handle NEG. if (const auto *CLHS = dyn_cast(U->getOperand(0))) if (CLHS->isNullValue()) { APInt KnownZero(TyBits, 0), KnownOne(TyBits, 0); - computeKnownBits(U->getOperand(1), KnownZero, KnownOne, TD, Depth+1, Q); + computeKnownBits(U->getOperand(1), KnownZero, KnownOne, DL, Depth + 1, + Q); // If the input is known to be 0 or 1, the output is 0/-1, which is all // sign bits set. if ((KnownZero | APInt(TyBits, 1)).isAllOnesValue()) @@ -1816,7 +2042,7 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, // Sub can have at most one carry bit. Thus we know that the output // is, at worst, one more bit than the inputs. - Tmp = ComputeNumSignBits(U->getOperand(0), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(U->getOperand(0), DL, Depth + 1, Q); if (Tmp == 1) return 1; // Early out. return std::min(Tmp, Tmp2)-1; @@ -1830,12 +2056,11 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, // Take the minimum of all incoming values. This can't infinitely loop // because of our depth threshold. - Tmp = ComputeNumSignBits(PN->getIncomingValue(0), TD, Depth+1, Q); + Tmp = ComputeNumSignBits(PN->getIncomingValue(0), DL, Depth + 1, Q); for (unsigned i = 1, e = NumIncomingValues; i != e; ++i) { if (Tmp == 1) return Tmp; - Tmp = std::min(Tmp, - ComputeNumSignBits(PN->getIncomingValue(i), TD, - Depth+1, Q)); + Tmp = std::min( + Tmp, ComputeNumSignBits(PN->getIncomingValue(i), DL, Depth + 1, Q)); } return Tmp; } @@ -1850,7 +2075,7 @@ unsigned ComputeNumSignBits(Value *V, const DataLayout *TD, // use this information. APInt KnownZero(TyBits, 0), KnownOne(TyBits, 0); APInt Mask; - computeKnownBits(V, KnownZero, KnownOne, TD, Depth, Q); + computeKnownBits(V, KnownZero, KnownOne, DL, Depth, Q); if (KnownZero.isNegative()) { // sign bit is 0 Mask = KnownZero; @@ -2132,9 +2357,7 @@ Value *llvm::isBytewiseValue(Value *V) { if (CI->getBitWidth() % 8 == 0) { assert(CI->getBitWidth() > 8 && "8 bits should be handled above!"); - // We can check that all bytes of an integer are equal by making use of a - // little trick: rotate by 8 and check if it's still the same value. - if (CI->getValue() != CI->getValue().rotl(8)) + if (!CI->getValue().isSplat(8)) return nullptr; return ConstantInt::get(V->getContext(), CI->getValue().trunc(8)); } @@ -2335,23 +2558,19 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef idx_range, /// Analyze the specified pointer to see if it can be expressed as a base /// pointer plus a constant offset. Return the base and offset to the caller. Value *llvm::GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, - const DataLayout *DL) { - // Without DataLayout, conservatively assume 64-bit offsets, which is - // the widest we support. - unsigned BitWidth = DL ? DL->getPointerTypeSizeInBits(Ptr->getType()) : 64; + const DataLayout &DL) { + unsigned BitWidth = DL.getPointerTypeSizeInBits(Ptr->getType()); APInt ByteOffset(BitWidth, 0); while (1) { if (Ptr->getType()->isVectorTy()) break; if (GEPOperator *GEP = dyn_cast(Ptr)) { - if (DL) { - APInt GEPOffset(BitWidth, 0); - if (!GEP->accumulateConstantOffset(*DL, GEPOffset)) - break; + APInt GEPOffset(BitWidth, 0); + if (!GEP->accumulateConstantOffset(DL, GEPOffset)) + break; - ByteOffset += GEPOffset; - } + ByteOffset += GEPOffset; Ptr = GEP->getPointerOperand(); } else if (Operator::getOpcode(Ptr) == Instruction::BitCast || @@ -2380,7 +2599,7 @@ bool llvm::getConstantStringInfo(const Value *V, StringRef &Str, // Look through bitcast instructions and geps. V = V->stripPointerCasts(); - // If the value is a GEP instructionor constant expression, treat it as an + // If the value is a GEP instruction or constant expression, treat it as an // offset. if (const GEPOperator *GEP = dyn_cast(V)) { // Make sure the GEP has exactly three arguments. @@ -2407,7 +2626,8 @@ bool llvm::getConstantStringInfo(const Value *V, StringRef &Str, StartIdx = CI->getZExtValue(); else return false; - return getConstantStringInfo(GEP->getOperand(0), Str, StartIdx+Offset); + return getConstantStringInfo(GEP->getOperand(0), Str, StartIdx + Offset, + TrimAtNul); } // The GEP instruction, constant or instruction, must reference a global @@ -2517,8 +2737,8 @@ uint64_t llvm::GetStringLength(Value *V) { return Len == ~0ULL ? 1 : Len; } -Value * -llvm::GetUnderlyingObject(Value *V, const DataLayout *TD, unsigned MaxLookup) { +Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL, + unsigned MaxLookup) { if (!V->getType()->isPointerTy()) return V; for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) { @@ -2535,7 +2755,7 @@ llvm::GetUnderlyingObject(Value *V, const DataLayout *TD, unsigned MaxLookup) { // See if InstructionSimplify knows any relevant tricks. if (Instruction *I = dyn_cast(V)) // TODO: Acquire a DominatorTree and AssumptionCache and use them. - if (Value *Simplified = SimplifyInstruction(I, TD, nullptr)) { + if (Value *Simplified = SimplifyInstruction(I, DL, nullptr)) { V = Simplified; continue; } @@ -2547,17 +2767,14 @@ llvm::GetUnderlyingObject(Value *V, const DataLayout *TD, unsigned MaxLookup) { return V; } -void -llvm::GetUnderlyingObjects(Value *V, - SmallVectorImpl &Objects, - const DataLayout *TD, - unsigned MaxLookup) { +void llvm::GetUnderlyingObjects(Value *V, SmallVectorImpl &Objects, + const DataLayout &DL, unsigned MaxLookup) { SmallPtrSet Visited; SmallVector Worklist; Worklist.push_back(V); do { Value *P = Worklist.pop_back_val(); - P = GetUnderlyingObject(P, TD, MaxLookup); + P = GetUnderlyingObject(P, DL, MaxLookup); if (!Visited.insert(P).second) continue; @@ -2591,8 +2808,7 @@ bool llvm::onlyUsedByLifetimeMarkers(const Value *V) { return true; } -bool llvm::isSafeToSpeculativelyExecute(const Value *V, - const DataLayout *TD) { +bool llvm::isSafeToSpeculativelyExecute(const Value *V) { const Operator *Inst = dyn_cast(V); if (!Inst) return false; @@ -2638,7 +2854,8 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V, // Speculative load may create a race that did not exist in the source. LI->getParent()->getParent()->hasFnAttribute(Attribute::SanitizeThread)) return false; - return LI->getPointerOperand()->isDereferenceablePointer(TD); + const DataLayout &DL = LI->getModule()->getDataLayout(); + return LI->getPointerOperand()->isDereferenceablePointer(DL); } case Instruction::Call: { if (const IntrinsicInst *II = dyn_cast(Inst)) { @@ -2730,7 +2947,7 @@ bool llvm::isKnownNonNull(const Value *V, const TargetLibraryInfo *TLI) { } OverflowResult llvm::computeOverflowForUnsignedMul(Value *LHS, Value *RHS, - const DataLayout *DL, + const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { @@ -2780,7 +2997,7 @@ OverflowResult llvm::computeOverflowForUnsignedMul(Value *LHS, Value *RHS, } OverflowResult llvm::computeOverflowForUnsignedAdd(Value *LHS, Value *RHS, - const DataLayout *DL, + const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index 9e7354e..103c8c4 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -2270,13 +2270,13 @@ bool LLParser::PerFunctionState::SetInstName(int NameID, /// forward reference record if needed. BasicBlock *LLParser::PerFunctionState::GetBB(const std::string &Name, LocTy Loc) { - return cast_or_null(GetVal(Name, - Type::getLabelTy(F.getContext()), Loc)); + return dyn_cast_or_null(GetVal(Name, + Type::getLabelTy(F.getContext()), Loc)); } BasicBlock *LLParser::PerFunctionState::GetBB(unsigned ID, LocTy Loc) { - return cast_or_null(GetVal(ID, - Type::getLabelTy(F.getContext()), Loc)); + return dyn_cast_or_null(GetVal(ID, + Type::getLabelTy(F.getContext()), Loc)); } /// DefineBB - Define the specified basic block, which is either named or @@ -2512,7 +2512,12 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { if (!F) { // Make a global variable as a placeholder for this reference. - GlobalValue *&FwdRef = ForwardRefBlockAddresses[Fn][Label]; + GlobalValue *&FwdRef = + ForwardRefBlockAddresses.insert(std::make_pair( + std::move(Fn), + std::map())) + .first->second.insert(std::make_pair(std::move(Label), nullptr)) + .first->second; if (!FwdRef) FwdRef = new GlobalVariable(*M, Type::getInt8Ty(Context), false, GlobalValue::InternalLinkage, nullptr, ""); @@ -2772,11 +2777,23 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { unsigned Opc = Lex.getUIntVal(); SmallVector Elts; bool InBounds = false; + Type *Ty; Lex.Lex(); + if (Opc == Instruction::GetElementPtr) InBounds = EatIfPresent(lltok::kw_inbounds); - if (ParseToken(lltok::lparen, "expected '(' in constantexpr") || - ParseGlobalValueVector(Elts) || + + if (ParseToken(lltok::lparen, "expected '(' in constantexpr")) + return true; + + LocTy ExplicitTypeLoc = Lex.getLoc(); + if (Opc == Instruction::GetElementPtr) { + if (ParseType(Ty) || + ParseToken(lltok::comma, "expected comma after getelementptr's type")) + return true; + } + + if (ParseGlobalValueVector(Elts) || ParseToken(lltok::rparen, "expected ')' in constantexpr")) return true; @@ -2787,6 +2804,10 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { Type *BaseType = Elts[0]->getType(); auto *BasePointerType = cast(BaseType->getScalarType()); + if (Ty != BasePointerType->getElementType()) + return Error( + ExplicitTypeLoc, + "explicit pointee type doesn't match operand's pointee type"); ArrayRef Indices(Elts.begin() + 1, Elts.end()); for (Constant *Val : Indices) { @@ -2805,7 +2826,9 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { } } - if (!Indices.empty() && !BasePointerType->getElementType()->isSized()) + SmallPtrSet Visited; + if (!Indices.empty() && + !BasePointerType->getElementType()->isSized(&Visited)) return Error(ID.Loc, "base element of getelementptr must be sized"); if (!GetElementPtrInst::getIndexedType(Elts[0]->getType(), Indices)) @@ -2976,6 +2999,8 @@ struct ColumnField : public MDUnsignedField { }; struct DwarfTagField : public MDUnsignedField { DwarfTagField() : MDUnsignedField(0, dwarf::DW_TAG_hi_user) {} + DwarfTagField(dwarf::Tag DefaultTag) + : MDUnsignedField(DefaultTag, dwarf::DW_TAG_hi_user) {} }; struct DwarfAttEncodingField : public MDUnsignedField { DwarfAttEncodingField() : MDUnsignedField(0, dwarf::DW_ATE_hi_user) {} @@ -3323,8 +3348,8 @@ bool LLParser::ParseMDLocation(MDNode *&Result, bool IsDistinct) { PARSE_MD_FIELDS(); #undef VISIT_MD_FIELDS - auto get = (IsDistinct ? MDLocation::getDistinct : MDLocation::get); - Result = get(Context, line.Val, column.Val, scope.Val, inlinedAt.Val); + Result = GET_OR_DISTINCT( + MDLocation, (Context, line.Val, column.Val, scope.Val, inlinedAt.Val)); return false; } @@ -3373,7 +3398,7 @@ bool LLParser::ParseMDEnumerator(MDNode *&Result, bool IsDistinct) { /// ::= !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32) bool LLParser::ParseMDBasicType(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ - REQUIRED(tag, DwarfTagField, ); \ + OPTIONAL(tag, DwarfTagField, (dwarf::DW_TAG_base_type)); \ OPTIONAL(name, MDStringField, ); \ OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX)); \ OPTIONAL(align, MDUnsignedField, (0, UINT64_MAX)); \ @@ -3509,7 +3534,7 @@ bool LLParser::ParseMDCompileUnit(MDNode *&Result, bool IsDistinct) { bool LLParser::ParseMDSubprogram(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ OPTIONAL(scope, MDField, ); \ - REQUIRED(name, MDStringField, ); \ + OPTIONAL(name, MDStringField, ); \ OPTIONAL(linkageName, MDStringField, ); \ OPTIONAL(file, MDField, ); \ OPTIONAL(line, LineField, ); \ @@ -3604,9 +3629,9 @@ bool LLParser::ParseMDTemplateTypeParameter(MDNode *&Result, bool IsDistinct) { /// name: "V", type: !1, value: i32 7) bool LLParser::ParseMDTemplateValueParameter(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ - REQUIRED(tag, DwarfTagField, ); \ + OPTIONAL(tag, DwarfTagField, (dwarf::DW_TAG_template_value_parameter)); \ OPTIONAL(name, MDStringField, ); \ - REQUIRED(type, MDField, ); \ + OPTIONAL(type, MDField, ); \ REQUIRED(value, MDField, ); PARSE_MD_FIELDS(); #undef VISIT_MD_FIELDS @@ -3624,7 +3649,7 @@ bool LLParser::ParseMDTemplateValueParameter(MDNode *&Result, bool IsDistinct) { bool LLParser::ParseMDGlobalVariable(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ OPTIONAL(scope, MDField, ); \ - REQUIRED(name, MDStringField, ); \ + OPTIONAL(name, MDStringField, ); \ OPTIONAL(linkageName, MDStringField, ); \ OPTIONAL(file, MDField, ); \ OPTIONAL(line, LineField, ); \ @@ -3710,7 +3735,7 @@ bool LLParser::ParseMDExpression(MDNode *&Result, bool IsDistinct) { /// getter: "getFoo", attributes: 7, type: !2) bool LLParser::ParseMDObjCProperty(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ - REQUIRED(name, MDStringField, ); \ + OPTIONAL(name, MDStringField, ); \ OPTIONAL(file, MDField, ); \ OPTIONAL(line, LineField, ); \ OPTIONAL(setter, MDStringField, ); \ @@ -4297,7 +4322,9 @@ bool LLParser::ParseBasicBlock(PerFunctionState &PFS) { } BasicBlock *BB = PFS.DefineBB(Name, NameLoc); - if (!BB) return true; + if (!BB) + return Error(NameLoc, + "unable to create block named '" + Name + "'"); std::string NameStr; @@ -5032,7 +5059,7 @@ bool LLParser::ParseLandingPad(Instruction *&Inst, PerFunctionState &PFS) { ParseTypeAndValue(PersFn, PersFnLoc, PFS)) return true; - LandingPadInst *LP = LandingPadInst::Create(Ty, PersFn, 0); + std::unique_ptr LP(LandingPadInst::Create(Ty, PersFn, 0)); LP->setCleanup(EatIfPresent(lltok::kw_cleanup)); while (Lex.getKind() == lltok::kw_catch || Lex.getKind() == lltok::kw_filter){ @@ -5046,10 +5073,8 @@ bool LLParser::ParseLandingPad(Instruction *&Inst, PerFunctionState &PFS) { Value *V; LocTy VLoc; - if (ParseTypeAndValue(V, VLoc, PFS)) { - delete LP; + if (ParseTypeAndValue(V, VLoc, PFS)) return true; - } // A 'catch' type expects a non-array constant. A filter clause expects an // array constant. @@ -5061,10 +5086,13 @@ bool LLParser::ParseLandingPad(Instruction *&Inst, PerFunctionState &PFS) { Error(VLoc, "'filter' clause has an invalid type"); } - LP->addClause(cast(V)); + Constant *CV = dyn_cast(V); + if (!CV) + return Error(VLoc, "clause argument must be a constant"); + LP->addClause(CV); } - Inst = LP; + Inst = LP.release(); return false; } @@ -5241,7 +5269,11 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { Lex.Lex(); } - if (ParseTypeAndValue(Val, Loc, PFS) || + Type *Ty = nullptr; + LocTy ExplicitTypeLoc = Lex.getLoc(); + if (ParseType(Ty) || + ParseToken(lltok::comma, "expected comma after load's type") || + ParseTypeAndValue(Val, Loc, PFS) || ParseScopeAndOrdering(isAtomic, Scope, Ordering) || ParseOptionalCommaAlign(Alignment, AteExtraComma)) return true; @@ -5254,6 +5286,10 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { if (Ordering == Release || Ordering == AcquireRelease) return Error(Loc, "atomic load cannot use Release ordering"); + if (Ty != cast(Val->getType())->getElementType()) + return Error(ExplicitTypeLoc, + "explicit pointee type doesn't match operand's pointee type"); + Inst = new LoadInst(Val, "", isVolatile, Alignment, Ordering, Scope); return AteExtraComma ? InstExtraComma : InstNormal; } @@ -5440,13 +5476,22 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) { bool InBounds = EatIfPresent(lltok::kw_inbounds); - if (ParseTypeAndValue(Ptr, Loc, PFS)) return true; + Type *Ty = nullptr; + LocTy ExplicitTypeLoc = Lex.getLoc(); + if (ParseType(Ty) || + ParseToken(lltok::comma, "expected comma after getelementptr's type") || + ParseTypeAndValue(Ptr, Loc, PFS)) + return true; Type *BaseType = Ptr->getType(); PointerType *BasePointerType = dyn_cast(BaseType->getScalarType()); if (!BasePointerType) return Error(Loc, "base of getelementptr must be a pointer"); + if (Ty != BasePointerType->getElementType()) + return Error(ExplicitTypeLoc, + "explicit pointee type doesn't match operand's pointee type"); + SmallVector Indices; bool AteExtraComma = false; while (EatIfPresent(lltok::comma)) { @@ -5469,12 +5514,14 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) { Indices.push_back(Val); } - if (!Indices.empty() && !BasePointerType->getElementType()->isSized()) + SmallPtrSet Visited; + if (!Indices.empty() && + !BasePointerType->getElementType()->isSized(&Visited)) return Error(Loc, "base element of getelementptr must be sized"); if (!GetElementPtrInst::getIndexedType(BaseType, Indices)) return Error(Loc, "invalid getelementptr indices"); - Inst = GetElementPtrInst::Create(Ptr, Indices); + Inst = GetElementPtrInst::Create(Ty, Ptr, Indices); if (InBounds) cast(Inst)->setIsInBounds(true); return AteExtraComma ? InstExtraComma : InstNormal; diff --git a/lib/AsmParser/Parser.cpp b/lib/AsmParser/Parser.cpp index ed1a753..2e76c0e 100644 --- a/lib/AsmParser/Parser.cpp +++ b/lib/AsmParser/Parser.cpp @@ -13,6 +13,7 @@ #include "llvm/AsmParser/Parser.h" #include "LLParser.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/IR/Module.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 92af0f8..84753ff 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -8,34 +8,375 @@ //===----------------------------------------------------------------------===// #include "llvm/Bitcode/ReaderWriter.h" -#include "BitcodeReader.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" +#include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/OperandTraits.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/Support/DataStream.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" - +#include using namespace llvm; +namespace { enum { SWITCH_INST_MAGIC = 0x4B5 // May 2012 => 1205 => Hex }; +class BitcodeReaderValueList { + std::vector ValuePtrs; + + /// ResolveConstants - As we resolve forward-referenced constants, we add + /// information about them to this vector. This allows us to resolve them in + /// bulk instead of resolving each reference at a time. See the code in + /// ResolveConstantForwardRefs for more information about this. + /// + /// The key of this vector is the placeholder constant, the value is the slot + /// number that holds the resolved value. + typedef std::vector > ResolveConstantsTy; + ResolveConstantsTy ResolveConstants; + LLVMContext &Context; +public: + BitcodeReaderValueList(LLVMContext &C) : Context(C) {} + ~BitcodeReaderValueList() { + assert(ResolveConstants.empty() && "Constants not resolved?"); + } + + // vector compatibility methods + unsigned size() const { return ValuePtrs.size(); } + void resize(unsigned N) { ValuePtrs.resize(N); } + void push_back(Value *V) { + ValuePtrs.push_back(V); + } + + void clear() { + assert(ResolveConstants.empty() && "Constants not resolved?"); + ValuePtrs.clear(); + } + + Value *operator[](unsigned i) const { + assert(i < ValuePtrs.size()); + return ValuePtrs[i]; + } + + Value *back() const { return ValuePtrs.back(); } + void pop_back() { ValuePtrs.pop_back(); } + bool empty() const { return ValuePtrs.empty(); } + void shrinkTo(unsigned N) { + assert(N <= size() && "Invalid shrinkTo request!"); + ValuePtrs.resize(N); + } + + Constant *getConstantFwdRef(unsigned Idx, Type *Ty); + Value *getValueFwdRef(unsigned Idx, Type *Ty); + + void AssignValue(Value *V, unsigned Idx); + + /// ResolveConstantForwardRefs - Once all constants are read, this method bulk + /// resolves any forward references. + void ResolveConstantForwardRefs(); +}; + +class BitcodeReaderMDValueList { + unsigned NumFwdRefs; + bool AnyFwdRefs; + unsigned MinFwdRef; + unsigned MaxFwdRef; + std::vector MDValuePtrs; + + LLVMContext &Context; +public: + BitcodeReaderMDValueList(LLVMContext &C) + : NumFwdRefs(0), AnyFwdRefs(false), Context(C) {} + + // vector compatibility methods + unsigned size() const { return MDValuePtrs.size(); } + void resize(unsigned N) { MDValuePtrs.resize(N); } + void push_back(Metadata *MD) { MDValuePtrs.emplace_back(MD); } + void clear() { MDValuePtrs.clear(); } + Metadata *back() const { return MDValuePtrs.back(); } + void pop_back() { MDValuePtrs.pop_back(); } + bool empty() const { return MDValuePtrs.empty(); } + + Metadata *operator[](unsigned i) const { + assert(i < MDValuePtrs.size()); + return MDValuePtrs[i]; + } + + void shrinkTo(unsigned N) { + assert(N <= size() && "Invalid shrinkTo request!"); + MDValuePtrs.resize(N); + } + + Metadata *getValueFwdRef(unsigned Idx); + void AssignValue(Metadata *MD, unsigned Idx); + void tryToResolveCycles(); +}; + +class BitcodeReader : public GVMaterializer { + LLVMContext &Context; + DiagnosticHandlerFunction DiagnosticHandler; + Module *TheModule; + std::unique_ptr Buffer; + std::unique_ptr StreamFile; + BitstreamCursor Stream; + DataStreamer *LazyStreamer; + uint64_t NextUnreadBit; + bool SeenValueSymbolTable; + + std::vector TypeList; + BitcodeReaderValueList ValueList; + BitcodeReaderMDValueList MDValueList; + std::vector ComdatList; + SmallVector InstructionList; + + std::vector > GlobalInits; + std::vector > AliasInits; + std::vector > FunctionPrefixes; + std::vector > FunctionPrologues; + + SmallVector InstsWithTBAATag; + + /// MAttributes - The set of attributes by index. Index zero in the + /// file is for null, and is thus not represented here. As such all indices + /// are off by one. + std::vector MAttributes; + + /// \brief The set of attribute groups. + std::map MAttributeGroups; + + /// FunctionBBs - While parsing a function body, this is a list of the basic + /// blocks for the function. + std::vector FunctionBBs; + + // When reading the module header, this list is populated with functions that + // have bodies later in the file. + std::vector FunctionsWithBodies; + + // When intrinsic functions are encountered which require upgrading they are + // stored here with their replacement function. + typedef std::vector > UpgradedIntrinsicMap; + UpgradedIntrinsicMap UpgradedIntrinsics; + + // Map the bitcode's custom MDKind ID to the Module's MDKind ID. + DenseMap MDKindMap; + + // Several operations happen after the module header has been read, but + // before function bodies are processed. This keeps track of whether + // we've done this yet. + bool SeenFirstFunctionBody; + + /// DeferredFunctionInfo - When function bodies are initially scanned, this + /// map contains info about where to find deferred function body in the + /// stream. + DenseMap DeferredFunctionInfo; + + /// When Metadata block is initially scanned when parsing the module, we may + /// choose to defer parsing of the metadata. This vector contains info about + /// which Metadata blocks are deferred. + std::vector DeferredMetadataInfo; + + /// These are basic blocks forward-referenced by block addresses. They are + /// inserted lazily into functions when they're loaded. The basic block ID is + /// its index into the vector. + DenseMap> BasicBlockFwdRefs; + std::deque BasicBlockFwdRefQueue; + + /// UseRelativeIDs - Indicates that we are using a new encoding for + /// instruction operands where most operands in the current + /// FUNCTION_BLOCK are encoded relative to the instruction number, + /// for a more compact encoding. Some instruction operands are not + /// relative to the instruction ID: basic block numbers, and types. + /// Once the old style function blocks have been phased out, we would + /// not need this flag. + bool UseRelativeIDs; + + /// True if all functions will be materialized, negating the need to process + /// (e.g.) blockaddress forward references. + bool WillMaterializeAllForwardRefs; + + /// Functions that have block addresses taken. This is usually empty. + SmallPtrSet BlockAddressesTaken; + + /// True if any Metadata block has been materialized. + bool IsMetadataMaterialized; + +public: + std::error_code Error(BitcodeError E, const Twine &Message); + std::error_code Error(BitcodeError E); + std::error_code Error(const Twine &Message); + + explicit BitcodeReader(MemoryBuffer *buffer, LLVMContext &C, + DiagnosticHandlerFunction DiagnosticHandler); + explicit BitcodeReader(DataStreamer *streamer, LLVMContext &C, + DiagnosticHandlerFunction DiagnosticHandler); + ~BitcodeReader() { FreeState(); } + + std::error_code materializeForwardReferencedFunctions(); + + void FreeState(); + + void releaseBuffer(); + + bool isDematerializable(const GlobalValue *GV) const override; + std::error_code materialize(GlobalValue *GV) override; + std::error_code MaterializeModule(Module *M) override; + std::vector getIdentifiedStructTypes() const override; + void Dematerialize(GlobalValue *GV) override; + + /// @brief Main interface to parsing a bitcode buffer. + /// @returns true if an error occurred. + std::error_code ParseBitcodeInto(Module *M, + bool ShouldLazyLoadMetadata = false); + + /// @brief Cheap mechanism to just extract module triple + /// @returns true if an error occurred. + ErrorOr parseTriple(); + + static uint64_t decodeSignRotatedValue(uint64_t V); + + /// Materialize any deferred Metadata block. + std::error_code materializeMetadata() override; + +private: + std::vector IdentifiedStructTypes; + StructType *createIdentifiedStructType(LLVMContext &Context, StringRef Name); + StructType *createIdentifiedStructType(LLVMContext &Context); + + Type *getTypeByID(unsigned ID); + Value *getFnValueByID(unsigned ID, Type *Ty) { + if (Ty && Ty->isMetadataTy()) + return MetadataAsValue::get(Ty->getContext(), getFnMetadataByID(ID)); + return ValueList.getValueFwdRef(ID, Ty); + } + Metadata *getFnMetadataByID(unsigned ID) { + return MDValueList.getValueFwdRef(ID); + } + BasicBlock *getBasicBlock(unsigned ID) const { + if (ID >= FunctionBBs.size()) return nullptr; // Invalid ID + return FunctionBBs[ID]; + } + AttributeSet getAttributes(unsigned i) const { + if (i-1 < MAttributes.size()) + return MAttributes[i-1]; + return AttributeSet(); + } + + /// getValueTypePair - Read a value/type pair out of the specified record from + /// slot 'Slot'. Increment Slot past the number of slots used in the record. + /// Return true on failure. + bool getValueTypePair(SmallVectorImpl &Record, unsigned &Slot, + unsigned InstNum, Value *&ResVal) { + if (Slot == Record.size()) return true; + unsigned ValNo = (unsigned)Record[Slot++]; + // Adjust the ValNo, if it was encoded relative to the InstNum. + if (UseRelativeIDs) + ValNo = InstNum - ValNo; + if (ValNo < InstNum) { + // If this is not a forward reference, just return the value we already + // have. + ResVal = getFnValueByID(ValNo, nullptr); + return ResVal == nullptr; + } else if (Slot == Record.size()) { + return true; + } + + unsigned TypeNo = (unsigned)Record[Slot++]; + ResVal = getFnValueByID(ValNo, getTypeByID(TypeNo)); + return ResVal == nullptr; + } + + /// popValue - Read a value out of the specified record from slot 'Slot'. + /// Increment Slot past the number of slots used by the value in the record. + /// Return true if there is an error. + bool popValue(SmallVectorImpl &Record, unsigned &Slot, + unsigned InstNum, Type *Ty, Value *&ResVal) { + if (getValue(Record, Slot, InstNum, Ty, ResVal)) + return true; + // All values currently take a single record slot. + ++Slot; + return false; + } + + /// getValue -- Like popValue, but does not increment the Slot number. + bool getValue(SmallVectorImpl &Record, unsigned Slot, + unsigned InstNum, Type *Ty, Value *&ResVal) { + ResVal = getValue(Record, Slot, InstNum, Ty); + return ResVal == nullptr; + } + + /// getValue -- Version of getValue that returns ResVal directly, + /// or 0 if there is an error. + Value *getValue(SmallVectorImpl &Record, unsigned Slot, + unsigned InstNum, Type *Ty) { + if (Slot == Record.size()) return nullptr; + unsigned ValNo = (unsigned)Record[Slot]; + // Adjust the ValNo, if it was encoded relative to the InstNum. + if (UseRelativeIDs) + ValNo = InstNum - ValNo; + return getFnValueByID(ValNo, Ty); + } + + /// getValueSigned -- Like getValue, but decodes signed VBRs. + Value *getValueSigned(SmallVectorImpl &Record, unsigned Slot, + unsigned InstNum, Type *Ty) { + if (Slot == Record.size()) return nullptr; + unsigned ValNo = (unsigned)decodeSignRotatedValue(Record[Slot]); + // Adjust the ValNo, if it was encoded relative to the InstNum. + if (UseRelativeIDs) + ValNo = InstNum - ValNo; + return getFnValueByID(ValNo, Ty); + } + + /// Converts alignment exponent (i.e. power of two (or zero)) to the + /// corresponding alignment to use. If alignment is too large, returns + /// a corresponding error code. + std::error_code parseAlignmentValue(uint64_t Exponent, unsigned &Alignment); + std::error_code ParseAttrKind(uint64_t Code, Attribute::AttrKind *Kind); + std::error_code ParseModule(bool Resume, bool ShouldLazyLoadMetadata = false); + std::error_code ParseAttributeBlock(); + std::error_code ParseAttributeGroupBlock(); + std::error_code ParseTypeTable(); + std::error_code ParseTypeTableBody(); + + std::error_code ParseValueSymbolTable(); + std::error_code ParseConstants(); + std::error_code RememberAndSkipFunctionBody(); + /// Save the positions of the Metadata blocks and skip parsing the blocks. + std::error_code rememberAndSkipMetadata(); + std::error_code ParseFunctionBody(Function *F); + std::error_code GlobalCleanup(); + std::error_code ResolveGlobalAndAliasInits(); + std::error_code ParseMetadata(); + std::error_code ParseMetadataAttachment(); + ErrorOr parseModuleTriple(); + std::error_code ParseUseLists(); + std::error_code InitStream(); + std::error_code InitStreamFromBuffer(); + std::error_code InitLazyStream(); + std::error_code FindFunctionInStream( + Function *F, + DenseMap::iterator DeferredFunctionInfoIterator); +}; +} // namespace + BitcodeDiagnosticInfo::BitcodeDiagnosticInfo(std::error_code EC, DiagnosticSeverity Severity, const Twine &Msg) @@ -81,7 +422,7 @@ BitcodeReader::BitcodeReader(MemoryBuffer *buffer, LLVMContext &C, TheModule(nullptr), Buffer(buffer), LazyStreamer(nullptr), NextUnreadBit(0), SeenValueSymbolTable(false), ValueList(C), MDValueList(C), SeenFirstFunctionBody(false), UseRelativeIDs(false), - WillMaterializeAllForwardRefs(false) {} + WillMaterializeAllForwardRefs(false), IsMetadataMaterialized(false) {} BitcodeReader::BitcodeReader(DataStreamer *streamer, LLVMContext &C, DiagnosticHandlerFunction DiagnosticHandler) @@ -89,7 +430,7 @@ BitcodeReader::BitcodeReader(DataStreamer *streamer, LLVMContext &C, TheModule(nullptr), Buffer(nullptr), LazyStreamer(streamer), NextUnreadBit(0), SeenValueSymbolTable(false), ValueList(C), MDValueList(C), SeenFirstFunctionBody(false), UseRelativeIDs(false), - WillMaterializeAllForwardRefs(false) {} + WillMaterializeAllForwardRefs(false), IsMetadataMaterialized(false) {} std::error_code BitcodeReader::materializeForwardReferencedFunctions() { if (WillMaterializeAllForwardRefs) @@ -135,6 +476,7 @@ void BitcodeReader::FreeState() { std::vector().swap(FunctionBBs); std::vector().swap(FunctionsWithBodies); DeferredFunctionInfo.clear(); + DeferredMetadataInfo.clear(); MDKindMap.clear(); assert(BasicBlockFwdRefs.empty() && "Unresolved blockaddress fwd references"); @@ -1198,6 +1540,7 @@ std::error_code BitcodeReader::ParseValueSymbolTable() { static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; } std::error_code BitcodeReader::ParseMetadata() { + IsMetadataMaterialized = true; unsigned NextMDValueNo = MDValueList.size(); if (Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID)) @@ -1348,14 +1691,15 @@ std::error_code BitcodeReader::ParseMetadata() { if (Record.size() != 5) return Error("Invalid record"); - auto get = Record[0] ? MDLocation::getDistinct : MDLocation::get; unsigned Line = Record[1]; unsigned Column = Record[2]; MDNode *Scope = cast(MDValueList.getValueFwdRef(Record[3])); Metadata *InlinedAt = Record[4] ? MDValueList.getValueFwdRef(Record[4] - 1) : nullptr; - MDValueList.AssignValue(get(Context, Line, Column, Scope, InlinedAt), - NextMDValueNo++); + MDValueList.AssignValue( + GET_OR_DISTINCT(MDLocation, Record[0], + (Context, Line, Column, Scope, InlinedAt)), + NextMDValueNo++); break; } case bitc::METADATA_GENERIC_DEBUG: { @@ -1952,19 +2296,26 @@ std::error_code BitcodeReader::ParseConstants() { } case bitc::CST_CODE_CE_INBOUNDS_GEP: case bitc::CST_CODE_CE_GEP: { // CE_GEP: [n x operands] - if (Record.size() & 1) - return Error("Invalid record"); + unsigned OpNum = 0; + Type *PointeeType = nullptr; + if (Record.size() % 2) + PointeeType = getTypeByID(Record[OpNum++]); SmallVector Elts; - for (unsigned i = 0, e = Record.size(); i != e; i += 2) { - Type *ElTy = getTypeByID(Record[i]); + while (OpNum != Record.size()) { + Type *ElTy = getTypeByID(Record[OpNum++]); if (!ElTy) return Error("Invalid record"); - Elts.push_back(ValueList.getConstantFwdRef(Record[i+1], ElTy)); + Elts.push_back(ValueList.getConstantFwdRef(Record[OpNum++], ElTy)); } + ArrayRef Indices(Elts.begin() + 1, Elts.end()); V = ConstantExpr::getGetElementPtr(Elts[0], Indices, BitCode == bitc::CST_CODE_CE_INBOUNDS_GEP); + if (PointeeType && + PointeeType != cast(V)->getSourceElementType()) + return Error("Explicit gep operator type does not match pointee type " + "of pointer operand"); break; } case bitc::CST_CODE_CE_SELECT: { // CE_SELECT: [opval#, opval#, opval#] @@ -2234,6 +2585,30 @@ std::error_code BitcodeReader::ParseUseLists() { } } +/// When we see the block for metadata, remember where it is and then skip it. +/// This lets us lazily deserialize the metadata. +std::error_code BitcodeReader::rememberAndSkipMetadata() { + // Save the current stream state. + uint64_t CurBit = Stream.GetCurrentBitNo(); + DeferredMetadataInfo.push_back(CurBit); + + // Skip over the block for now. + if (Stream.SkipBlock()) + return Error("Invalid record"); + return std::error_code(); +} + +std::error_code BitcodeReader::materializeMetadata() { + for (uint64_t BitPos : DeferredMetadataInfo) { + // Move the bit stream to the saved position. + Stream.JumpToBit(BitPos); + if (std::error_code EC = ParseMetadata()) + return EC; + } + DeferredMetadataInfo.clear(); + return std::error_code(); +} + /// RememberAndSkipFunctionBody - When we see the block for a function body, /// remember where it is and then skip it. This lets us lazily deserialize the /// functions. @@ -2284,7 +2659,8 @@ std::error_code BitcodeReader::GlobalCleanup() { return std::error_code(); } -std::error_code BitcodeReader::ParseModule(bool Resume) { +std::error_code BitcodeReader::ParseModule(bool Resume, + bool ShouldLazyLoadMetadata) { if (Resume) Stream.JumpToBit(NextUnreadBit); else if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) @@ -2338,6 +2714,12 @@ std::error_code BitcodeReader::ParseModule(bool Resume) { return EC; break; case bitc::METADATA_BLOCK_ID: + if (ShouldLazyLoadMetadata && !IsMetadataMaterialized) { + if (std::error_code EC = rememberAndSkipMetadata()) + return EC; + break; + } + assert(DeferredMetadataInfo.empty() && "Unexpected deferred metadata"); if (std::error_code EC = ParseMetadata()) return EC; break; @@ -2652,7 +3034,8 @@ std::error_code BitcodeReader::ParseModule(bool Resume) { } } -std::error_code BitcodeReader::ParseBitcodeInto(Module *M) { +std::error_code BitcodeReader::ParseBitcodeInto(Module *M, + bool ShouldLazyLoadMetadata) { TheModule = nullptr; if (std::error_code EC = InitStream()) @@ -2693,7 +3076,7 @@ std::error_code BitcodeReader::ParseBitcodeInto(Module *M) { if (TheModule) return Error("Invalid multiple blocks"); TheModule = M; - if (std::error_code EC = ParseModule(false)) + if (std::error_code EC = ParseModule(false, ShouldLazyLoadMetadata)) return EC; if (LazyStreamer) return std::error_code(); @@ -3082,6 +3465,13 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { if (getValueTypePair(Record, OpNum, NextValueNo, BasePtr)) return Error("Invalid record"); + if (Ty && + Ty != + cast(BasePtr->getType()->getScalarType()) + ->getElementType()) + return Error( + "Explicit gep type does not match pointee type of pointer operand"); + SmallVector GEPIdx; while (OpNum != Record.size()) { Value *Op; @@ -3090,8 +3480,8 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { GEPIdx.push_back(Op); } - I = GetElementPtrInst::Create(BasePtr, GEPIdx); - assert(!Ty || Ty == cast(I)->getSourceElementType()); + I = GetElementPtrInst::Create(Ty, BasePtr, GEPIdx); + InstructionList.push_back(I); if (InBounds) cast(I)->setIsInBounds(true); @@ -3600,8 +3990,9 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { return EC; I = new LoadInst(Op, "", Record[OpNum+1], Align); - assert((!Ty || Ty == I->getType()) && - "Explicit type doesn't match pointee type of the first operand"); + if (Ty && Ty != I->getType()) + return Error("Explicit load type does not match pointee type of " + "pointer operand"); InstructionList.push_back(I); break; @@ -3631,6 +4022,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { return EC; I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SynchScope); + (void)Ty; assert((!Ty || Ty == I->getType()) && "Explicit type doesn't match pointee type of the first operand"); @@ -3890,6 +4282,9 @@ std::error_code BitcodeReader::FindFunctionInStream( void BitcodeReader::releaseBuffer() { Buffer.release(); } std::error_code BitcodeReader::materialize(GlobalValue *GV) { + if (std::error_code EC = materializeMetadata()) + return EC; + Function *F = dyn_cast(GV); // If it's not a function or is already material, ignore the request. if (!F || !F->isMaterializable()) @@ -3957,6 +4352,9 @@ std::error_code BitcodeReader::MaterializeModule(Module *M) { assert(M == TheModule && "Can only Materialize the Module this BitcodeReader is attached to."); + if (std::error_code EC = materializeMetadata()) + return EC; + // Promise to materialize all forward references. WillMaterializeAllForwardRefs = true; @@ -4097,7 +4495,8 @@ const std::error_category &llvm::BitcodeErrorCategory() { static ErrorOr getLazyBitcodeModuleImpl(std::unique_ptr &&Buffer, LLVMContext &Context, bool WillMaterializeAll, - DiagnosticHandlerFunction DiagnosticHandler) { + DiagnosticHandlerFunction DiagnosticHandler, + bool ShouldLazyLoadMetadata = false) { Module *M = new Module(Buffer->getBufferIdentifier(), Context); BitcodeReader *R = new BitcodeReader(Buffer.get(), Context, DiagnosticHandler); @@ -4109,7 +4508,8 @@ getLazyBitcodeModuleImpl(std::unique_ptr &&Buffer, return EC; }; - if (std::error_code EC = R->ParseBitcodeInto(M)) + // Delay parsing Metadata if ShouldLazyLoadMetadata is true. + if (std::error_code EC = R->ParseBitcodeInto(M, ShouldLazyLoadMetadata)) return cleanupOnError(EC); if (!WillMaterializeAll) @@ -4124,9 +4524,10 @@ getLazyBitcodeModuleImpl(std::unique_ptr &&Buffer, ErrorOr llvm::getLazyBitcodeModule(std::unique_ptr &&Buffer, LLVMContext &Context, - DiagnosticHandlerFunction DiagnosticHandler) { + DiagnosticHandlerFunction DiagnosticHandler, + bool ShouldLazyLoadMetadata) { return getLazyBitcodeModuleImpl(std::move(Buffer), Context, false, - DiagnosticHandler); + DiagnosticHandler, ShouldLazyLoadMetadata); } ErrorOr> diff --git a/lib/Bitcode/Reader/BitcodeReader.h b/lib/Bitcode/Reader/BitcodeReader.h deleted file mode 100644 index 9803e78..0000000 --- a/lib/Bitcode/Reader/BitcodeReader.h +++ /dev/null @@ -1,369 +0,0 @@ -//===- BitcodeReader.h - Internal BitcodeReader impl ------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This header defines the BitcodeReader class. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_BITCODE_READER_BITCODEREADER_H -#define LLVM_LIB_BITCODE_READER_BITCODEREADER_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/Bitcode/BitstreamReader.h" -#include "llvm/Bitcode/LLVMBitCodes.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/GVMaterializer.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/OperandTraits.h" -#include "llvm/IR/TrackingMDRef.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/ValueHandle.h" -#include -#include -#include - -namespace llvm { - class Comdat; - class MemoryBuffer; - class LLVMContext; - -//===----------------------------------------------------------------------===// -// BitcodeReaderValueList Class -//===----------------------------------------------------------------------===// - -class BitcodeReaderValueList { - std::vector ValuePtrs; - - /// ResolveConstants - As we resolve forward-referenced constants, we add - /// information about them to this vector. This allows us to resolve them in - /// bulk instead of resolving each reference at a time. See the code in - /// ResolveConstantForwardRefs for more information about this. - /// - /// The key of this vector is the placeholder constant, the value is the slot - /// number that holds the resolved value. - typedef std::vector > ResolveConstantsTy; - ResolveConstantsTy ResolveConstants; - LLVMContext &Context; -public: - BitcodeReaderValueList(LLVMContext &C) : Context(C) {} - ~BitcodeReaderValueList() { - assert(ResolveConstants.empty() && "Constants not resolved?"); - } - - // vector compatibility methods - unsigned size() const { return ValuePtrs.size(); } - void resize(unsigned N) { ValuePtrs.resize(N); } - void push_back(Value *V) { - ValuePtrs.push_back(V); - } - - void clear() { - assert(ResolveConstants.empty() && "Constants not resolved?"); - ValuePtrs.clear(); - } - - Value *operator[](unsigned i) const { - assert(i < ValuePtrs.size()); - return ValuePtrs[i]; - } - - Value *back() const { return ValuePtrs.back(); } - void pop_back() { ValuePtrs.pop_back(); } - bool empty() const { return ValuePtrs.empty(); } - void shrinkTo(unsigned N) { - assert(N <= size() && "Invalid shrinkTo request!"); - ValuePtrs.resize(N); - } - - Constant *getConstantFwdRef(unsigned Idx, Type *Ty); - Value *getValueFwdRef(unsigned Idx, Type *Ty); - - void AssignValue(Value *V, unsigned Idx); - - /// ResolveConstantForwardRefs - Once all constants are read, this method bulk - /// resolves any forward references. - void ResolveConstantForwardRefs(); -}; - - -//===----------------------------------------------------------------------===// -// BitcodeReaderMDValueList Class -//===----------------------------------------------------------------------===// - -class BitcodeReaderMDValueList { - unsigned NumFwdRefs; - bool AnyFwdRefs; - unsigned MinFwdRef; - unsigned MaxFwdRef; - std::vector MDValuePtrs; - - LLVMContext &Context; -public: - BitcodeReaderMDValueList(LLVMContext &C) - : NumFwdRefs(0), AnyFwdRefs(false), Context(C) {} - - // vector compatibility methods - unsigned size() const { return MDValuePtrs.size(); } - void resize(unsigned N) { MDValuePtrs.resize(N); } - void push_back(Metadata *MD) { MDValuePtrs.emplace_back(MD); } - void clear() { MDValuePtrs.clear(); } - Metadata *back() const { return MDValuePtrs.back(); } - void pop_back() { MDValuePtrs.pop_back(); } - bool empty() const { return MDValuePtrs.empty(); } - - Metadata *operator[](unsigned i) const { - assert(i < MDValuePtrs.size()); - return MDValuePtrs[i]; - } - - void shrinkTo(unsigned N) { - assert(N <= size() && "Invalid shrinkTo request!"); - MDValuePtrs.resize(N); - } - - Metadata *getValueFwdRef(unsigned Idx); - void AssignValue(Metadata *MD, unsigned Idx); - void tryToResolveCycles(); -}; - -class BitcodeReader : public GVMaterializer { - LLVMContext &Context; - DiagnosticHandlerFunction DiagnosticHandler; - Module *TheModule; - std::unique_ptr Buffer; - std::unique_ptr StreamFile; - BitstreamCursor Stream; - DataStreamer *LazyStreamer; - uint64_t NextUnreadBit; - bool SeenValueSymbolTable; - - std::vector TypeList; - BitcodeReaderValueList ValueList; - BitcodeReaderMDValueList MDValueList; - std::vector ComdatList; - SmallVector InstructionList; - - std::vector > GlobalInits; - std::vector > AliasInits; - std::vector > FunctionPrefixes; - std::vector > FunctionPrologues; - - SmallVector InstsWithTBAATag; - - /// MAttributes - The set of attributes by index. Index zero in the - /// file is for null, and is thus not represented here. As such all indices - /// are off by one. - std::vector MAttributes; - - /// \brief The set of attribute groups. - std::map MAttributeGroups; - - /// FunctionBBs - While parsing a function body, this is a list of the basic - /// blocks for the function. - std::vector FunctionBBs; - - // When reading the module header, this list is populated with functions that - // have bodies later in the file. - std::vector FunctionsWithBodies; - - // When intrinsic functions are encountered which require upgrading they are - // stored here with their replacement function. - typedef std::vector > UpgradedIntrinsicMap; - UpgradedIntrinsicMap UpgradedIntrinsics; - - // Map the bitcode's custom MDKind ID to the Module's MDKind ID. - DenseMap MDKindMap; - - // Several operations happen after the module header has been read, but - // before function bodies are processed. This keeps track of whether - // we've done this yet. - bool SeenFirstFunctionBody; - - /// DeferredFunctionInfo - When function bodies are initially scanned, this - /// map contains info about where to find deferred function body in the - /// stream. - DenseMap DeferredFunctionInfo; - - /// These are basic blocks forward-referenced by block addresses. They are - /// inserted lazily into functions when they're loaded. The basic block ID is - /// its index into the vector. - DenseMap> BasicBlockFwdRefs; - std::deque BasicBlockFwdRefQueue; - - /// UseRelativeIDs - Indicates that we are using a new encoding for - /// instruction operands where most operands in the current - /// FUNCTION_BLOCK are encoded relative to the instruction number, - /// for a more compact encoding. Some instruction operands are not - /// relative to the instruction ID: basic block numbers, and types. - /// Once the old style function blocks have been phased out, we would - /// not need this flag. - bool UseRelativeIDs; - - /// True if all functions will be materialized, negating the need to process - /// (e.g.) blockaddress forward references. - bool WillMaterializeAllForwardRefs; - - /// Functions that have block addresses taken. This is usually empty. - SmallPtrSet BlockAddressesTaken; - -public: - std::error_code Error(BitcodeError E, const Twine &Message); - std::error_code Error(BitcodeError E); - std::error_code Error(const Twine &Message); - - explicit BitcodeReader(MemoryBuffer *buffer, LLVMContext &C, - DiagnosticHandlerFunction DiagnosticHandler); - explicit BitcodeReader(DataStreamer *streamer, LLVMContext &C, - DiagnosticHandlerFunction DiagnosticHandler); - ~BitcodeReader() { FreeState(); } - - std::error_code materializeForwardReferencedFunctions(); - - void FreeState(); - - void releaseBuffer(); - - bool isDematerializable(const GlobalValue *GV) const override; - std::error_code materialize(GlobalValue *GV) override; - std::error_code MaterializeModule(Module *M) override; - std::vector getIdentifiedStructTypes() const override; - void Dematerialize(GlobalValue *GV) override; - - /// @brief Main interface to parsing a bitcode buffer. - /// @returns true if an error occurred. - std::error_code ParseBitcodeInto(Module *M); - - /// @brief Cheap mechanism to just extract module triple - /// @returns true if an error occurred. - ErrorOr parseTriple(); - - static uint64_t decodeSignRotatedValue(uint64_t V); - -private: - std::vector IdentifiedStructTypes; - StructType *createIdentifiedStructType(LLVMContext &Context, StringRef Name); - StructType *createIdentifiedStructType(LLVMContext &Context); - - Type *getTypeByID(unsigned ID); - Value *getFnValueByID(unsigned ID, Type *Ty) { - if (Ty && Ty->isMetadataTy()) - return MetadataAsValue::get(Ty->getContext(), getFnMetadataByID(ID)); - return ValueList.getValueFwdRef(ID, Ty); - } - Metadata *getFnMetadataByID(unsigned ID) { - return MDValueList.getValueFwdRef(ID); - } - BasicBlock *getBasicBlock(unsigned ID) const { - if (ID >= FunctionBBs.size()) return nullptr; // Invalid ID - return FunctionBBs[ID]; - } - AttributeSet getAttributes(unsigned i) const { - if (i-1 < MAttributes.size()) - return MAttributes[i-1]; - return AttributeSet(); - } - - /// getValueTypePair - Read a value/type pair out of the specified record from - /// slot 'Slot'. Increment Slot past the number of slots used in the record. - /// Return true on failure. - bool getValueTypePair(SmallVectorImpl &Record, unsigned &Slot, - unsigned InstNum, Value *&ResVal) { - if (Slot == Record.size()) return true; - unsigned ValNo = (unsigned)Record[Slot++]; - // Adjust the ValNo, if it was encoded relative to the InstNum. - if (UseRelativeIDs) - ValNo = InstNum - ValNo; - if (ValNo < InstNum) { - // If this is not a forward reference, just return the value we already - // have. - ResVal = getFnValueByID(ValNo, nullptr); - return ResVal == nullptr; - } else if (Slot == Record.size()) { - return true; - } - - unsigned TypeNo = (unsigned)Record[Slot++]; - ResVal = getFnValueByID(ValNo, getTypeByID(TypeNo)); - return ResVal == nullptr; - } - - /// popValue - Read a value out of the specified record from slot 'Slot'. - /// Increment Slot past the number of slots used by the value in the record. - /// Return true if there is an error. - bool popValue(SmallVectorImpl &Record, unsigned &Slot, - unsigned InstNum, Type *Ty, Value *&ResVal) { - if (getValue(Record, Slot, InstNum, Ty, ResVal)) - return true; - // All values currently take a single record slot. - ++Slot; - return false; - } - - /// getValue -- Like popValue, but does not increment the Slot number. - bool getValue(SmallVectorImpl &Record, unsigned Slot, - unsigned InstNum, Type *Ty, Value *&ResVal) { - ResVal = getValue(Record, Slot, InstNum, Ty); - return ResVal == nullptr; - } - - /// getValue -- Version of getValue that returns ResVal directly, - /// or 0 if there is an error. - Value *getValue(SmallVectorImpl &Record, unsigned Slot, - unsigned InstNum, Type *Ty) { - if (Slot == Record.size()) return nullptr; - unsigned ValNo = (unsigned)Record[Slot]; - // Adjust the ValNo, if it was encoded relative to the InstNum. - if (UseRelativeIDs) - ValNo = InstNum - ValNo; - return getFnValueByID(ValNo, Ty); - } - - /// getValueSigned -- Like getValue, but decodes signed VBRs. - Value *getValueSigned(SmallVectorImpl &Record, unsigned Slot, - unsigned InstNum, Type *Ty) { - if (Slot == Record.size()) return nullptr; - unsigned ValNo = (unsigned)decodeSignRotatedValue(Record[Slot]); - // Adjust the ValNo, if it was encoded relative to the InstNum. - if (UseRelativeIDs) - ValNo = InstNum - ValNo; - return getFnValueByID(ValNo, Ty); - } - - /// Converts alignment exponent (i.e. power of two (or zero)) to the - /// corresponding alignment to use. If alignment is too large, returns - /// a corresponding error code. - std::error_code parseAlignmentValue(uint64_t Exponent, unsigned &Alignment); - std::error_code ParseAttrKind(uint64_t Code, Attribute::AttrKind *Kind); - std::error_code ParseModule(bool Resume); - std::error_code ParseAttributeBlock(); - std::error_code ParseAttributeGroupBlock(); - std::error_code ParseTypeTable(); - std::error_code ParseTypeTableBody(); - - std::error_code ParseValueSymbolTable(); - std::error_code ParseConstants(); - std::error_code RememberAndSkipFunctionBody(); - std::error_code ParseFunctionBody(Function *F); - std::error_code GlobalCleanup(); - std::error_code ResolveGlobalAndAliasInits(); - std::error_code ParseMetadata(); - std::error_code ParseMetadataAttachment(); - ErrorOr parseModuleTriple(); - std::error_code ParseUseLists(); - std::error_code InitStream(); - std::error_code InitStreamFromBuffer(); - std::error_code InitLazyStream(); - std::error_code FindFunctionInStream( - Function *F, - DenseMap::iterator DeferredFunctionInfoIterator); -}; - -} // End llvm namespace - -#endif diff --git a/lib/Bitcode/Reader/BitstreamReader.cpp b/lib/Bitcode/Reader/BitstreamReader.cpp index ca68257..beaaf7a 100644 --- a/lib/Bitcode/Reader/BitstreamReader.cpp +++ b/lib/Bitcode/Reader/BitstreamReader.cpp @@ -245,7 +245,7 @@ void BitstreamCursor::ReadAbbrevRecord() { BitCodeAbbrev *Abbv = new BitCodeAbbrev(); unsigned NumOpInfo = ReadVBR(5); for (unsigned i = 0; i != NumOpInfo; ++i) { - bool IsLiteral = Read(1) ? true : false; + bool IsLiteral = Read(1); if (IsLiteral) { Abbv->Add(BitCodeAbbrevOp(ReadVBR64(8))); continue; diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index ecb6f7c..0123fb2 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -1205,6 +1205,8 @@ static void WriteModuleMetadata(const Module *M, SmallVector Record; for (const Metadata *MD : MDs) { if (const MDNode *N = dyn_cast(MD)) { + assert(N->isResolved() && "Expected forward references to be resolved"); + switch (N->getMetadataID()) { default: llvm_unreachable("Invalid MDNode subclass"); @@ -1522,15 +1524,18 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, Record.push_back(Flags); } break; - case Instruction::GetElementPtr: + case Instruction::GetElementPtr: { Code = bitc::CST_CODE_CE_GEP; - if (cast(C)->isInBounds()) + const auto *GO = cast(C); + if (GO->isInBounds()) Code = bitc::CST_CODE_CE_INBOUNDS_GEP; + Record.push_back(VE.getTypeID(GO->getSourceElementType())); for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) { Record.push_back(VE.getTypeID(C->getOperand(i)->getType())); Record.push_back(VE.getValueID(C->getOperand(i))); } break; + } case Instruction::Select: Code = bitc::CST_CODE_CE_SELECT; Record.push_back(VE.getValueID(C->getOperand(0))); diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 8ab2d6e..ce10998 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -18,3 +18,4 @@ add_subdirectory(AsmParser) add_subdirectory(LineEditor) add_subdirectory(ProfileData) add_subdirectory(Fuzzer) +add_subdirectory(Passes) diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp index e50b846..8e11fe1 100644 --- a/lib/CodeGen/Analysis.cpp +++ b/lib/CodeGen/Analysis.cpp @@ -312,8 +312,7 @@ static const Value *getNoopInput(const Value *V, // previous aggregate. Combine the two paths to obtain the true address of // our element. ArrayRef ExtractLoc = EVI->getIndices(); - std::copy(ExtractLoc.rbegin(), ExtractLoc.rend(), - std::back_inserter(ValLoc)); + ValLoc.append(ExtractLoc.rbegin(), ExtractLoc.rend()); NoopInput = Op; } // Terminate if we couldn't find anything to look through. @@ -601,10 +600,8 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F, // The manipulations performed when we're looking through an insertvalue or // an extractvalue would happen at the front of the RetPath list, so since // we have to copy it anyway it's more efficient to create a reversed copy. - using std::copy; - SmallVector TmpRetPath, TmpCallPath; - copy(RetPath.rbegin(), RetPath.rend(), std::back_inserter(TmpRetPath)); - copy(CallPath.rbegin(), CallPath.rend(), std::back_inserter(TmpCallPath)); + SmallVector TmpRetPath(RetPath.rbegin(), RetPath.rend()); + SmallVector TmpCallPath(CallPath.rbegin(), CallPath.rend()); // Finally, we can check whether the value produced by the tail call at this // index is compatible with the value we return. diff --git a/lib/CodeGen/Android.mk b/lib/CodeGen/Android.mk index ec3cd77..2827d73 100644 --- a/lib/CodeGen/Android.mk +++ b/lib/CodeGen/Android.mk @@ -21,7 +21,6 @@ codegen_SRC_FILES := \ ExecutionDepsFix.cpp \ ExpandISelPseudos.cpp \ ExpandPostRAPseudos.cpp \ - ForwardControlFlowIntegrity.cpp \ GCMetadata.cpp \ GCMetadataPrinter.cpp \ GCRootLowering.cpp \ @@ -31,7 +30,6 @@ codegen_SRC_FILES := \ InlineSpiller.cpp \ InterferenceCache.cpp \ IntrinsicLowering.cpp \ - JumpInstrTables.cpp \ LatencyPriorityQueue.cpp \ LexicalScopes.cpp \ LiveDebugVariables.cpp \ @@ -53,6 +51,7 @@ codegen_SRC_FILES := \ MachineCombiner.cpp \ MachineCopyPropagation.cpp \ MachineCSE.cpp \ + MachineDominanceFrontier.cpp \ MachineDominators.cpp \ MachineFunctionAnalysis.cpp \ MachineFunction.cpp \ @@ -66,6 +65,7 @@ codegen_SRC_FILES := \ MachineModuleInfoImpls.cpp \ MachinePassRegistry.cpp \ MachinePostDominators.cpp \ + MachineRegionInfo.cpp \ MachineRegisterInfo.cpp \ MachineScheduler.cpp \ MachineSink.cpp \ diff --git a/lib/CodeGen/AsmPrinter/ARMException.cpp b/lib/CodeGen/AsmPrinter/ARMException.cpp index 6fe75ad..9a16e15 100644 --- a/lib/CodeGen/AsmPrinter/ARMException.cpp +++ b/lib/CodeGen/AsmPrinter/ARMException.cpp @@ -36,8 +36,7 @@ #include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; -ARMException::ARMException(AsmPrinter *A) - : EHStreamer(A), shouldEmitCFI(false) {} +ARMException::ARMException(AsmPrinter *A) : DwarfCFIExceptionBase(A) {} ARMException::~ARMException() {} @@ -53,13 +52,9 @@ void ARMException::endModule() { Asm->OutStreamer.EmitCFISections(false, true); } -/// beginFunction - Gather pre-function exception information. Assumes it's -/// being emitted immediately after the function entry point. void ARMException::beginFunction(const MachineFunction *MF) { if (Asm->MAI->getExceptionHandlingType() == ExceptionHandling::ARM) getTargetStreamer().emitFnStart(); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_begin", - Asm->getFunctionNumber())); // See if we need call frame info. AsmPrinter::CFIMoveType MoveType = Asm->needsCFIMoves(); assert(MoveType != AsmPrinter::CFI_M_EH && @@ -72,20 +67,12 @@ void ARMException::beginFunction(const MachineFunction *MF) { /// endFunction - Gather and emit post-function exception information. /// -void ARMException::endFunction(const MachineFunction *) { - if (shouldEmitCFI) - Asm->OutStreamer.EmitCFIEndProc(); - - // Map all labels and get rid of any dead landing pads. - MMI->TidyLandingPads(); - +void ARMException::endFunction(const MachineFunction *MF) { ARMTargetStreamer &ATS = getTargetStreamer(); if (!Asm->MF->getFunction()->needsUnwindTableEntry() && MMI->getLandingPads().empty()) ATS.emitCantUnwind(); else { - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_end", - Asm->getFunctionNumber())); if (!MMI->getLandingPads().empty()) { // Emit references to personality. if (const Function *Personality = MMI->getPersonality()) { diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 988381d..07d6731 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -77,11 +77,11 @@ static gcp_map_type &getGCMap(void *&P) { /// getGVAlignmentLog2 - Return the alignment to use for the specified global /// value in log2 form. This rounds up to the preferred alignment if possible /// and legal. -static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &TD, +static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &DL, unsigned InBits = 0) { unsigned NumBits = 0; if (const GlobalVariable *GVar = dyn_cast(GV)) - NumBits = TD.getPreferredAlignmentLog(GVar); + NumBits = DL.getPreferredAlignmentLog(GVar); // If InBits is specified, round it to it. if (InBits > NumBits) @@ -103,12 +103,14 @@ static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &TD, AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer) : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()), OutContext(Streamer->getContext()), OutStreamer(*Streamer.release()), - LastMI(nullptr), LastFn(0), Counter(~0U), SetCounter(0) { + LastMI(nullptr), LastFn(0), Counter(~0U) { DD = nullptr; MMI = nullptr; LI = nullptr; MF = nullptr; - CurrentFnSym = CurrentFnSymForSize = nullptr; + CurExceptionSym = CurrentFnSym = CurrentFnSymForSize = nullptr; + CurrentFnBegin = nullptr; + CurrentFnEnd = nullptr; GCMetadataPrinters = nullptr; VerboseAsm = OutStreamer.isVerboseAsm(); } @@ -219,9 +221,13 @@ bool AsmPrinter::doInitialization(Module &M) { // Emit module-level inline asm if it exists. if (!M.getModuleInlineAsm().empty()) { + // We're at the module level. Construct MCSubtarget from the default CPU + // and target triple. + std::unique_ptr STI(TM.getTarget().createMCSubtargetInfo( + TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString())); OutStreamer.AddComment("Start of file scope inline assembly"); OutStreamer.AddBlankLine(); - EmitInlineAsm(M.getModuleInlineAsm()+"\n"); + EmitInlineAsm(M.getModuleInlineAsm()+"\n", *STI); OutStreamer.AddComment("End of file scope inline assembly"); OutStreamer.AddBlankLine(); } @@ -525,7 +531,8 @@ void AsmPrinter::EmitFunctionHeader() { EmitVisibility(CurrentFnSym, F->getVisibility()); EmitLinkage(F, CurrentFnSym); - EmitAlignment(MF->getAlignment(), F); + if (MAI->hasFunctionAlignment()) + EmitAlignment(MF->getAlignment(), F); if (MAI->hasDotTypeDotSizeDirective()) OutStreamer.EmitSymbolAttribute(CurrentFnSym, MCSA_ELF_TypeFunction); @@ -554,6 +561,17 @@ void AsmPrinter::EmitFunctionHeader() { OutStreamer.EmitLabel(DeadBlockSyms[i]); } + if (CurrentFnBegin) { + if (MAI->useAssignmentForEHBegin()) { + MCSymbol *CurPos = OutContext.CreateTempSymbol(); + OutStreamer.EmitLabel(CurPos); + OutStreamer.EmitAssignment(CurrentFnBegin, + MCSymbolRefExpr::Create(CurPos, OutContext)); + } else { + OutStreamer.EmitLabel(CurrentFnBegin); + } + } + // Emit pre-function debug and/or EH information. for (const HandlerInfo &HI : Handlers) { NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); @@ -764,6 +782,8 @@ void AsmPrinter::emitFrameAlloc(const MachineInstr &MI) { /// EmitFunctionBody - This method emits the body and trailer for a /// function. void AsmPrinter::EmitFunctionBody() { + EmitFunctionHeader(); + // Emit target-specific gunk before the function body. EmitFunctionBodyStart(); @@ -867,32 +887,41 @@ void AsmPrinter::EmitFunctionBody() { // Emit target-specific gunk after the function body. EmitFunctionBodyEnd(); + if (!MMI->getLandingPads().empty() || MMI->hasDebugInfo() || + MAI->hasDotTypeDotSizeDirective()) { + // Create a symbol for the end of function. + CurrentFnEnd = createTempSymbol("func_end"); + OutStreamer.EmitLabel(CurrentFnEnd); + } + // If the target wants a .size directive for the size of the function, emit // it. if (MAI->hasDotTypeDotSizeDirective()) { - // Create a symbol for the end of function, so we can get the size as - // difference between the function label and the temp label. - MCSymbol *FnEndLabel = OutContext.CreateTempSymbol(); - OutStreamer.EmitLabel(FnEndLabel); - + // We can get the size as difference between the function label and the + // temp label. const MCExpr *SizeExp = - MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(FnEndLabel, OutContext), + MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(CurrentFnEnd, OutContext), MCSymbolRefExpr::Create(CurrentFnSymForSize, OutContext), OutContext); OutStreamer.EmitELFSize(CurrentFnSym, SizeExp); } - // Emit post-function debug and/or EH information. for (const HandlerInfo &HI : Handlers) { NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); - HI.Handler->endFunction(MF); + HI.Handler->markFunctionEnd(); } - MMI->EndFunction(); // Print out jump tables referenced by the function. EmitJumpTableInfo(); + // Emit post-function debug and/or EH information. + for (const HandlerInfo &HI : Handlers) { + NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); + HI.Handler->endFunction(MF); + } + MMI->EndFunction(); + OutStreamer.AddBlankLine(); } @@ -928,7 +957,7 @@ static bool isGOTEquivalentCandidate(const GlobalVariable *GV, // To be a got equivalent, at least one of its users need to be a constant // expression used by another global variable. for (auto *U : GV->users()) - NumGOTEquivUsers += getNumGlobalVariableUses(cast(U)); + NumGOTEquivUsers += getNumGlobalVariableUses(dyn_cast(U)); return NumGOTEquivUsers > 0; } @@ -961,17 +990,25 @@ void AsmPrinter::emitGlobalGOTEquivs() { if (!getObjFileLowering().supportIndirectSymViaGOTPCRel()) return; - while (!GlobalGOTEquivs.empty()) { - DenseMap::iterator I = - GlobalGOTEquivs.begin(); - const MCSymbol *S = I->first; - const GlobalVariable *GV = I->second.first; - GlobalGOTEquivs.erase(S); - EmitGlobalVariable(GV); + SmallVector FailedCandidates; + for (auto &I : GlobalGOTEquivs) { + const GlobalVariable *GV = I.second.first; + unsigned Cnt = I.second.second; + if (Cnt) + FailedCandidates.push_back(GV); } + GlobalGOTEquivs.clear(); + + for (auto *GV : FailedCandidates) + EmitGlobalVariable(GV); } bool AsmPrinter::doFinalization(Module &M) { + // Set the MachineFunction to nullptr so that we can catch attempted + // accesses to MF specific features at the module level and so that + // we can conditionalize accesses based on whether or not it is nullptr. + MF = nullptr; + // Gather all GOT equivalent globals in the module. We really need two // passes over the globals: one to compute and another to avoid its emission // in EmitGlobalVariable, otherwise we would not be able to handle cases @@ -997,59 +1034,6 @@ bool AsmPrinter::doFinalization(Module &M) { EmitVisibility(Name, V, false); } - // Get information about jump-instruction tables to print. - JumpInstrTableInfo *JITI = getAnalysisIfAvailable(); - - if (JITI && !JITI->getTables().empty()) { - // Since we're at the module level we can't use a function specific - // MCSubtargetInfo - instead create one with the module defaults. - std::unique_ptr STI(TM.getTarget().createMCSubtargetInfo( - TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString())); - unsigned Arch = Triple(getTargetTriple()).getArch(); - bool IsThumb = (Arch == Triple::thumb || Arch == Triple::thumbeb); - const TargetInstrInfo *TII = TM.getSubtargetImpl()->getInstrInfo(); - MCInst TrapInst; - TII->getTrap(TrapInst); - unsigned LogAlignment = llvm::Log2_64(JITI->entryByteAlignment()); - - // Emit the right section for these functions. - OutStreamer.SwitchSection(OutContext.getObjectFileInfo()->getTextSection()); - for (const auto &KV : JITI->getTables()) { - uint64_t Count = 0; - for (const auto &FunPair : KV.second) { - // Emit the function labels to make this be a function entry point. - MCSymbol *FunSym = - OutContext.GetOrCreateSymbol(FunPair.second->getName()); - EmitAlignment(LogAlignment); - if (IsThumb) - OutStreamer.EmitThumbFunc(FunSym); - if (MAI->hasDotTypeDotSizeDirective()) - OutStreamer.EmitSymbolAttribute(FunSym, MCSA_ELF_TypeFunction); - OutStreamer.EmitLabel(FunSym); - - // Emit the jump instruction to transfer control to the original - // function. - MCInst JumpToFun; - MCSymbol *TargetSymbol = - OutContext.GetOrCreateSymbol(FunPair.first->getName()); - const MCSymbolRefExpr *TargetSymRef = - MCSymbolRefExpr::Create(TargetSymbol, MCSymbolRefExpr::VK_PLT, - OutContext); - TII->getUnconditionalBranch(JumpToFun, TargetSymRef); - OutStreamer.EmitInstruction(JumpToFun, *STI); - ++Count; - } - - // Emit enough padding instructions to fill up to the next power of two. - uint64_t Remaining = NextPowerOf2(Count) - Count; - for (uint64_t C = 0; C < Remaining; ++C) { - EmitAlignment(LogAlignment); - OutStreamer.EmitInstruction(TrapInst, *STI); - } - - } - } - // Emit module flags. SmallVector ModuleFlags; M.getModuleFlagsMetadata(ModuleFlags); @@ -1152,11 +1136,26 @@ bool AsmPrinter::doFinalization(Module &M) { return false; } +MCSymbol *AsmPrinter::getCurExceptionSym() { + if (!CurExceptionSym) + CurExceptionSym = createTempSymbol("exception"); + return CurExceptionSym; +} + void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { this->MF = &MF; // Get the function symbol. CurrentFnSym = getSymbol(MF.getFunction()); CurrentFnSymForSize = CurrentFnSym; + CurrentFnBegin = nullptr; + CurExceptionSym = nullptr; + bool NeedsLocalForSize = MAI->needsLocalForSize(); + if (!MMI->getLandingPads().empty() || MMI->hasDebugInfo() || + NeedsLocalForSize) { + CurrentFnBegin = createTempSymbol("func_begin"); + if (NeedsLocalForSize) + CurrentFnSymForSize = CurrentFnBegin; + } if (isVerbose()) LI = &getAnalysis(); @@ -1273,10 +1272,8 @@ void AsmPrinter::EmitJumpTableInfo() { bool JTInDiffSection = !TLOF.shouldPutJumpTableInFunctionSection( MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32, *F); - if (!JTInDiffSection) { - OutStreamer.SwitchSection(TLOF.SectionForGlobal(F, *Mang, TM)); - } else { - // Otherwise, drop it in the readonly section. + if (JTInDiffSection) { + // Drop it in the readonly section. const MCSection *ReadOnlySection = TLOF.getSectionForJumpTable(*F, *Mang, TM); OutStreamer.SwitchSection(ReadOnlySection); @@ -1585,7 +1582,7 @@ void AsmPrinter::EmitLabelDifference(const MCSymbol *Hi, const MCSymbol *Lo, } // Otherwise, emit with .set (aka assignment). - MCSymbol *SetLabel = GetTempSymbol("set", SetCounter++); + MCSymbol *SetLabel = createTempSymbol("set"); OutStreamer.EmitAssignment(SetLabel, Diff); OutStreamer.EmitSymbolValue(SetLabel, Size); } @@ -1667,8 +1664,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) { // If the code isn't optimized, there may be outstanding folding // opportunities. Attempt to fold the expression using DataLayout as a // last resort before giving up. - if (Constant *C = ConstantFoldConstantExpression( - CE, TM.getDataLayout())) + if (Constant *C = ConstantFoldConstantExpression(CE, *TM.getDataLayout())) if (C != CE) return lowerConstant(C); @@ -2112,9 +2108,15 @@ static void handleIndirectSymViaGOTPCRel(AsmPrinter &AP, const MCExpr **ME, // // gotpcrelcst := + // + // If gotpcrelcst is positive it means that we can safely fold the pc rel + // displacement into the GOTPCREL. We can also can have an extra offset + // if the target knows how to encode it. + // int64_t GOTPCRelCst = Offset + MV.getConstant(); if (GOTPCRelCst < 0) return; + if (!AP.getObjFileLowering().supportGOTPCRelWithOffset() && GOTPCRelCst != 0) + return; // Emit the GOT PC relative to replace the got equivalent global, i.e.: // @@ -2134,18 +2136,16 @@ static void handleIndirectSymViaGOTPCRel(AsmPrinter &AP, const MCExpr **ME, // AsmPrinter::GOTEquivUsePair Result = AP.GlobalGOTEquivs[GOTEquivSym]; const GlobalVariable *GV = Result.first; - unsigned NumUses = Result.second; + int NumUses = (int)Result.second; const GlobalValue *FinalGV = dyn_cast(GV->getOperand(0)); const MCSymbol *FinalSym = AP.getSymbol(FinalGV); - *ME = AP.getObjFileLowering().getIndirectSymViaGOTPCRel(FinalSym, - GOTPCRelCst); + *ME = AP.getObjFileLowering().getIndirectSymViaGOTPCRel( + FinalSym, MV, Offset, AP.MMI, AP.OutStreamer); // Update GOT equivalent usage information --NumUses; - if (NumUses) + if (NumUses >= 0) AP.GlobalGOTEquivs[GOTEquivSym] = std::make_pair(GV, NumUses); - else - AP.GlobalGOTEquivs.erase(GOTEquivSym); } static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP, @@ -2206,7 +2206,7 @@ static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP, // If the constant expression's size is greater than 64-bits, then we have // to emit the value in chunks. Try to constant fold the value and emit it // that way. - Constant *New = ConstantFoldConstantExpression(CE, DL); + Constant *New = ConstantFoldConstantExpression(CE, *DL); if (New && New != CE) return emitGlobalConstantImpl(New, AP); } @@ -2257,23 +2257,10 @@ void AsmPrinter::printOffset(int64_t Offset, raw_ostream &OS) const { // Symbol Lowering Routines. //===----------------------------------------------------------------------===// -/// GetTempSymbol - Return the MCSymbol corresponding to the assembler -/// temporary label with the specified stem and unique ID. -MCSymbol *AsmPrinter::GetTempSymbol(const Twine &Name, unsigned ID) const { - const DataLayout *DL = TM.getDataLayout(); - return OutContext.GetOrCreateSymbol(Twine(DL->getPrivateGlobalPrefix()) + - Name + Twine(ID)); -} - -/// GetTempSymbol - Return an assembler temporary label with the specified -/// stem. -MCSymbol *AsmPrinter::GetTempSymbol(const Twine &Name) const { - const DataLayout *DL = TM.getDataLayout(); - return OutContext.GetOrCreateSymbol(Twine(DL->getPrivateGlobalPrefix())+ - Name); +MCSymbol *AsmPrinter::createTempSymbol(const Twine &Name) const { + return OutContext.createTempSymbol(Name, true); } - MCSymbol *AsmPrinter::GetBlockAddressSymbol(const BlockAddress *BA) const { return MMI->getAddrLabelSymbol(BA->getBasicBlock()); } @@ -2523,3 +2510,5 @@ GCMetadataPrinter *AsmPrinter::GetOrCreateGCPrinter(GCStrategy &S) { /// Pin vtable to this file. AsmPrinterHandler::~AsmPrinterHandler() {} + +void AsmPrinterHandler::markFunctionEnd() {} diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp index d0958c1..9de36da 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp @@ -12,9 +12,12 @@ //===----------------------------------------------------------------------===// #include "ByteStreamer.h" +#include "DwarfDebug.h" #include "DwarfExpression.h" #include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/DIE.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/MC/MCAsmInfo.h" @@ -27,29 +30,11 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; #define DEBUG_TYPE "asm-printer" -void DebugLocDwarfExpression::EmitOp(uint8_t Op, const char *Comment) { - BS.EmitInt8( - Op, Comment ? Twine(Comment) + " " + dwarf::OperationEncodingString(Op) - : dwarf::OperationEncodingString(Op)); -} - -void DebugLocDwarfExpression::EmitSigned(int Value) { - BS.EmitSLEB128(Value, Twine(Value)); -} - -void DebugLocDwarfExpression::EmitUnsigned(unsigned Value) { - BS.EmitULEB128(Value, Twine(Value)); -} - -bool DebugLocDwarfExpression::isFrameRegister(unsigned MachineReg) { - // This information is not available while emitting .debug_loc entries. - return false; -} - //===----------------------------------------------------------------------===// // Dwarf Emission Helper Routines //===----------------------------------------------------------------------===// @@ -178,57 +163,28 @@ void AsmPrinter::EmitTTypeReference(const GlobalValue *GV, /// /// SectionLabel is a temporary label emitted at the start of the section that /// Label lives in. -void AsmPrinter::EmitSectionOffset(const MCSymbol *Label, - const MCSymbol *SectionLabel) const { +void AsmPrinter::emitSectionOffset(const MCSymbol *Label) const { // On COFF targets, we have to emit the special .secrel32 directive. if (MAI->needsDwarfSectionOffsetDirective()) { OutStreamer.EmitCOFFSecRel32(Label); return; } - // Get the section that we're referring to, based on SectionLabel. - const MCSection &Section = SectionLabel->getSection(); - - // If Label has already been emitted, verify that it is in the same section as - // section label for sanity. - assert((!Label->isInSection() || &Label->getSection() == &Section) && - "Section offset using wrong section base for label"); - - // If the section in question will end up with an address of 0 anyway, we can - // just emit an absolute reference to save a relocation. - if (Section.isBaseAddressKnownZero()) { + // If the format uses relocations with dwarf, refer to the symbol directly. + if (MAI->doesDwarfUseRelocationsAcrossSections()) { OutStreamer.EmitSymbolValue(Label, 4); return; } // Otherwise, emit it as a label difference from the start of the section. - EmitLabelDifference(Label, SectionLabel, 4); -} - -// Some targets do not provide a DWARF register number for every -// register. This function attempts to emit a DWARF register by -// emitting a piece of a super-register or by piecing together -// multiple subregisters that alias the register. -void AsmPrinter::EmitDwarfRegOpPiece(ByteStreamer &Streamer, - const MachineLocation &MLoc, - unsigned PieceSizeInBits, - unsigned PieceOffsetInBits) const { - assert(MLoc.isReg() && "MLoc must be a register"); - DebugLocDwarfExpression Expr(*this, Streamer); - Expr.AddMachineRegPiece(MLoc.getReg(), PieceSizeInBits, PieceOffsetInBits); -} - -void AsmPrinter::EmitDwarfOpPiece(ByteStreamer &Streamer, - unsigned PieceSizeInBits, - unsigned PieceOffsetInBits) const { - DebugLocDwarfExpression Expr(*this, Streamer); - Expr.AddOpPiece(PieceSizeInBits, PieceOffsetInBits); + EmitLabelDifference(Label, Label->getSection().getBeginSymbol(), 4); } /// EmitDwarfRegOp - Emit dwarf register operation. void AsmPrinter::EmitDwarfRegOp(ByteStreamer &Streamer, const MachineLocation &MLoc) const { - DebugLocDwarfExpression Expr(*this, Streamer); + DebugLocDwarfExpression Expr(*MF->getSubtarget().getRegisterInfo(), + getDwarfDebug()->getDwarfVersion(), Streamer); const MCRegisterInfo *MRI = MMI->getContext().getRegisterInfo(); int Reg = MRI->getDwarfRegNum(MLoc.getReg(), false); if (Reg < 0) { @@ -285,3 +241,60 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const { break; } } + +void AsmPrinter::emitDwarfDIE(const DIE &Die) const { + // Get the abbreviation for this DIE. + const DIEAbbrev &Abbrev = Die.getAbbrev(); + + // Emit the code (index) for the abbreviation. + if (isVerbose()) + OutStreamer.AddComment("Abbrev [" + Twine(Abbrev.getNumber()) + + "] 0x" + Twine::utohexstr(Die.getOffset()) + + ":0x" + Twine::utohexstr(Die.getSize()) + " " + + dwarf::TagString(Abbrev.getTag())); + EmitULEB128(Abbrev.getNumber()); + + const SmallVectorImpl &Values = Die.getValues(); + const SmallVectorImpl &AbbrevData = Abbrev.getData(); + + // Emit the DIE attribute values. + for (unsigned i = 0, N = Values.size(); i < N; ++i) { + dwarf::Attribute Attr = AbbrevData[i].getAttribute(); + dwarf::Form Form = AbbrevData[i].getForm(); + assert(Form && "Too many attributes for DIE (check abbreviation)"); + + if (isVerbose()) { + OutStreamer.AddComment(dwarf::AttributeString(Attr)); + if (Attr == dwarf::DW_AT_accessibility) + OutStreamer.AddComment(dwarf::AccessibilityString( + cast(Values[i])->getValue())); + } + + // Emit an attribute using the defined form. + Values[i]->EmitValue(this, Form); + } + + // Emit the DIE children if any. + if (Abbrev.hasChildren()) { + for (auto &Child : Die.getChildren()) + emitDwarfDIE(*Child); + + OutStreamer.AddComment("End Of Children Mark"); + EmitInt8(0); + } +} + +void +AsmPrinter::emitDwarfAbbrevs(const std::vector& Abbrevs) const { + // For each abbrevation. + for (const DIEAbbrev *Abbrev : Abbrevs) { + // Emit the abbrevations code (base 1 index.) + EmitULEB128(Abbrev->getNumber(), "Abbreviation Code"); + + // Emit the abbreviations data. + Abbrev->Emit(this); + } + + // Mark end of abbreviations. + EmitULEB128(0, "EOM(3)"); +} diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h b/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h index 31867dd..f1efe9d 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h +++ b/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h @@ -41,6 +41,10 @@ public: /// call. virtual void beginFunction(const MachineFunction *MF) = 0; + // \brief Emit any of function marker (like .cfi_endproc). This is called + // before endFunction and cannot switch sections. + virtual void markFunctionEnd(); + /// \brief Gather post-function debug information. /// Please note that some AsmPrinter implementations may not call /// beginFunction at all. diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index e6e7c97..bf63b1b 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -73,7 +73,8 @@ static void srcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) { } /// EmitInlineAsm - Emit a blob of inline asm to the output streamer. -void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode, +void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, + const MDNode *LocMDNode, InlineAsm::AsmDialect Dialect) const { assert(!Str.empty() && "Can't emit empty inline asm block"); @@ -93,17 +94,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode, !OutStreamer.isIntegratedAssemblerRequired()) { emitInlineAsmStart(); OutStreamer.EmitRawText(Str); - // If we have a machine function then grab the MCSubtarget off of that, - // otherwise we're at the module level and want to construct one from - // the default CPU and target triple. - if (MF) { - emitInlineAsmEnd(MF->getSubtarget(), nullptr); - } else { - std::unique_ptr STI(TM.getTarget().createMCSubtargetInfo( - TM.getTargetTriple(), TM.getTargetCPU(), - TM.getTargetFeatureString())); - emitInlineAsmEnd(*STI, nullptr); - } + emitInlineAsmEnd(STI, nullptr); return; } @@ -135,19 +126,11 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode, std::unique_ptr Parser( createMCAsmParser(SrcMgr, OutContext, OutStreamer, *MAI)); - // Initialize the parser with a fresh subtarget info. It is better to use a - // new STI here because the parser may modify it and we do not want those - // modifications to persist after parsing the inlineasm. The modifications - // made by the parser will be seen by the code emitters because it passes - // the current STI down to the EncodeInstruction() method. - std::unique_ptr STI(TM.getTarget().createMCSubtargetInfo( - TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString())); - - // Preserve a copy of the original STI because the parser may modify it. For - // example, when switching between arm and thumb mode. If the target needs to - // emit code to return to the original state it can do so in + // Create a temporary copy of the original STI because the parser may modify + // it. For example, when switching between arm and thumb mode. If the target + // needs to emit code to return to the original state it can do so in // emitInlineAsmEnd(). - MCSubtargetInfo STIOrig = *STI; + MCSubtargetInfo TmpSTI = STI; // We create a new MCInstrInfo here since we might be at the module level // and not have a MachineFunction to initialize the TargetInstrInfo from and @@ -155,7 +138,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode, // because it's not subtarget dependent. std::unique_ptr MII(TM.getTarget().createMCInstrInfo()); std::unique_ptr TAP(TM.getTarget().createMCAsmParser( - *STI, *Parser, *MII, TM.Options.MCOptions)); + TmpSTI, *Parser, *MII, TM.Options.MCOptions)); if (!TAP) report_fatal_error("Inline asm not supported by this streamer because" " we don't have an asm parser for this target\n"); @@ -170,7 +153,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode, // Don't implicitly switch to the text section before the asm. int Res = Parser->Run(/*NoInitialTextSection*/ true, /*NoFinalize*/ true); - emitInlineAsmEnd(STIOrig, STI.get()); + emitInlineAsmEnd(STI, &TmpSTI); if (Res && !HasDiagHandler) report_fatal_error("Error parsing inline asm\n"); } @@ -505,7 +488,7 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const { else EmitMSInlineAsmStr(AsmStr, MI, MMI, InlineAsmVariant, AP, LocCookie, OS); - EmitInlineAsm(OS.str(), LocMD, MI->getInlineAsmDialect()); + EmitInlineAsm(OS.str(), getSubtargetInfo(), LocMD, MI->getInlineAsmDialect()); // Emit the #NOAPP end marker. This has to happen even if verbose-asm isn't // enabled, so we use emitRawComment. diff --git a/lib/CodeGen/AsmPrinter/ByteStreamer.h b/lib/CodeGen/AsmPrinter/ByteStreamer.h index 42be114..179a4d4 100644 --- a/lib/CodeGen/AsmPrinter/ByteStreamer.h +++ b/lib/CodeGen/AsmPrinter/ByteStreamer.h @@ -19,6 +19,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/MC/MCStreamer.h" +#include "llvm/Support/LEB128.h" +#include namespace llvm { class ByteStreamer { @@ -66,6 +68,33 @@ class HashingByteStreamer : public ByteStreamer { Hash.addULEB128(DWord); } }; + +class BufferByteStreamer : public ByteStreamer { +private: + SmallVectorImpl &Buffer; + // FIXME: This is actually only needed for textual asm output. + SmallVectorImpl &Comments; + +public: + BufferByteStreamer(SmallVectorImpl &Buffer, + SmallVectorImpl &Comments) + : Buffer(Buffer), Comments(Comments) {} + void EmitInt8(uint8_t Byte, const Twine &Comment) override { + Buffer.push_back(Byte); + Comments.push_back(Comment.str()); + } + void EmitSLEB128(uint64_t DWord, const Twine &Comment) override { + raw_svector_ostream OSE(Buffer); + encodeSLEB128(DWord, OSE); + Comments.push_back(Comment.str()); + } + void EmitULEB128(uint64_t DWord, const Twine &Comment) override { + raw_svector_ostream OSE(Buffer); + encodeULEB128(DWord, OSE); + Comments.push_back(Comment.str()); + } +}; + } #endif diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp index 64ba56b..1a706f7 100644 --- a/lib/CodeGen/AsmPrinter/DIE.cpp +++ b/lib/CodeGen/AsmPrinter/DIE.cpp @@ -28,6 +28,7 @@ #include "llvm/Support/FormattedStream.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/MD5.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -60,7 +61,7 @@ void DIEAbbrev::Profile(FoldingSetNodeID &ID) const { /// Emit - Print the abbreviation using the specified asm printer. /// -void DIEAbbrev::Emit(AsmPrinter *AP) const { +void DIEAbbrev::Emit(const AsmPrinter *AP) const { // Emit its Dwarf tag type. AP->EmitULEB128(Tag, dwarf::TagString(Tag)); @@ -204,7 +205,7 @@ void DIEValue::dump() const { /// EmitValue - Emit integer of appropriate size. /// -void DIEInteger::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { +void DIEInteger::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const { unsigned Size = ~0U; switch (Form) { case dwarf::DW_FORM_flag_present: @@ -218,6 +219,7 @@ void DIEInteger::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { case dwarf::DW_FORM_ref2: // Fall thru case dwarf::DW_FORM_data2: Size = 2; break; case dwarf::DW_FORM_sec_offset: // Fall thru + case dwarf::DW_FORM_strp: // Fall thru case dwarf::DW_FORM_ref4: // Fall thru case dwarf::DW_FORM_data4: Size = 4; break; case dwarf::DW_FORM_ref8: // Fall thru @@ -229,6 +231,9 @@ void DIEInteger::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { case dwarf::DW_FORM_sdata: Asm->EmitSLEB128(Integer); return; case dwarf::DW_FORM_addr: Size = Asm->getDataLayout().getPointerSize(); break; + case dwarf::DW_FORM_ref_addr: + Size = SizeOf(Asm, dwarf::DW_FORM_ref_addr); + break; default: llvm_unreachable("DIE Value form not supported yet"); } Asm->OutStreamer.EmitIntValue(Integer, Size); @@ -236,7 +241,7 @@ void DIEInteger::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { /// SizeOf - Determine size of integer value in bytes. /// -unsigned DIEInteger::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIEInteger::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { switch (Form) { case dwarf::DW_FORM_flag_present: return 0; case dwarf::DW_FORM_flag: // Fall thru @@ -245,6 +250,7 @@ unsigned DIEInteger::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { case dwarf::DW_FORM_ref2: // Fall thru case dwarf::DW_FORM_data2: return sizeof(int16_t); case dwarf::DW_FORM_sec_offset: // Fall thru + case dwarf::DW_FORM_strp: // Fall thru case dwarf::DW_FORM_ref4: // Fall thru case dwarf::DW_FORM_data4: return sizeof(int32_t); case dwarf::DW_FORM_ref8: // Fall thru @@ -255,6 +261,10 @@ unsigned DIEInteger::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { case dwarf::DW_FORM_udata: return getULEB128Size(Integer); case dwarf::DW_FORM_sdata: return getSLEB128Size(Integer); case dwarf::DW_FORM_addr: return AP->getDataLayout().getPointerSize(); + case dwarf::DW_FORM_ref_addr: + if (AP->OutStreamer.getContext().getDwarfVersion() == 2) + return AP->getDataLayout().getPointerSize(); + return sizeof(int32_t); default: llvm_unreachable("DIE Value form not supported yet"); } } @@ -272,13 +282,13 @@ void DIEInteger::print(raw_ostream &O) const { /// EmitValue - Emit expression value. /// -void DIEExpr::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { +void DIEExpr::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { AP->OutStreamer.EmitValue(Expr, SizeOf(AP, Form)); } /// SizeOf - Determine size of expression value in bytes. /// -unsigned DIEExpr::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIEExpr::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { if (Form == dwarf::DW_FORM_data4) return 4; if (Form == dwarf::DW_FORM_sec_offset) return 4; if (Form == dwarf::DW_FORM_strp) return 4; @@ -298,7 +308,7 @@ void DIEExpr::print(raw_ostream &O) const { /// EmitValue - Emit label value. /// -void DIELabel::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { +void DIELabel::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { AP->EmitLabelReference(Label, SizeOf(AP, Form), Form == dwarf::DW_FORM_strp || Form == dwarf::DW_FORM_sec_offset || @@ -307,7 +317,7 @@ void DIELabel::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { /// SizeOf - Determine size of label value in bytes. /// -unsigned DIELabel::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIELabel::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { if (Form == dwarf::DW_FORM_data4) return 4; if (Form == dwarf::DW_FORM_sec_offset) return 4; if (Form == dwarf::DW_FORM_strp) return 4; @@ -326,13 +336,13 @@ void DIELabel::print(raw_ostream &O) const { /// EmitValue - Emit delta value. /// -void DIEDelta::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { +void DIEDelta::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { AP->EmitLabelDifference(LabelHi, LabelLo, SizeOf(AP, Form)); } /// SizeOf - Determine size of delta value in bytes. /// -unsigned DIEDelta::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIEDelta::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { if (Form == dwarf::DW_FORM_data4) return 4; if (Form == dwarf::DW_FORM_sec_offset) return 4; if (Form == dwarf::DW_FORM_strp) return 4; @@ -351,13 +361,13 @@ void DIEDelta::print(raw_ostream &O) const { /// EmitValue - Emit string value. /// -void DIEString::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { +void DIEString::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { Access->EmitValue(AP, Form); } /// SizeOf - Determine size of delta value in bytes. /// -unsigned DIEString::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIEString::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { return Access->SizeOf(AP, Form); } @@ -372,32 +382,9 @@ void DIEString::print(raw_ostream &O) const { // DIEEntry Implementation //===----------------------------------------------------------------------===// -/// Emit something like ".long Hi+Offset-Lo" where the size in bytes of the -/// directive is specified by Size and Hi/Lo specify the labels. -static void emitLabelOffsetDifference(MCStreamer &Streamer, const MCSymbol *Hi, - uint64_t Offset, const MCSymbol *Lo, - unsigned Size) { - MCContext &Context = Streamer.getContext(); - - // Emit Hi+Offset - Lo - // Get the Hi+Offset expression. - const MCExpr *Plus = - MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(Hi, Context), - MCConstantExpr::Create(Offset, Context), Context); - - // Get the Hi+Offset-Lo expression. - const MCExpr *Diff = MCBinaryExpr::CreateSub( - Plus, MCSymbolRefExpr::Create(Lo, Context), Context); - - // Otherwise, emit with .set (aka assignment). - MCSymbol *SetLabel = Context.CreateTempSymbol(); - Streamer.EmitAssignment(SetLabel, Diff); - Streamer.EmitSymbolValue(SetLabel, Size); -} - /// EmitValue - Emit debug information entry offset. /// -void DIEEntry::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { +void DIEEntry::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { if (Form == dwarf::DW_FORM_ref_addr) { const DwarfDebug *DD = AP->getDwarfDebug(); @@ -413,14 +400,12 @@ void DIEEntry::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { AP->EmitLabelPlusOffset(CU->getSectionSym(), Addr, DIEEntry::getRefAddrSize(AP)); else - emitLabelOffsetDifference(AP->OutStreamer, CU->getSectionSym(), Addr, - CU->getSectionSym(), - DIEEntry::getRefAddrSize(AP)); + AP->OutStreamer.EmitIntValue(Addr, DIEEntry::getRefAddrSize(AP)); } else AP->EmitInt32(Entry.getOffset()); } -unsigned DIEEntry::getRefAddrSize(AsmPrinter *AP) { +unsigned DIEEntry::getRefAddrSize(const AsmPrinter *AP) { // DWARF4: References that use the attribute form DW_FORM_ref_addr are // specified to be four bytes in the DWARF 32-bit format and eight bytes // in the DWARF 64-bit format, while DWARF Version 2 specifies that such @@ -441,7 +426,7 @@ void DIEEntry::print(raw_ostream &O) const { //===----------------------------------------------------------------------===// // DIETypeSignature Implementation //===----------------------------------------------------------------------===// -void DIETypeSignature::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { +void DIETypeSignature::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const { assert(Form == dwarf::DW_FORM_ref_sig8); Asm->OutStreamer.EmitIntValue(Unit.getTypeSignature(), 8); } @@ -460,7 +445,7 @@ void DIETypeSignature::dump() const { print(dbgs()); } /// ComputeSize - calculate the size of the location expression. /// -unsigned DIELoc::ComputeSize(AsmPrinter *AP) const { +unsigned DIELoc::ComputeSize(const AsmPrinter *AP) const { if (!Size) { const SmallVectorImpl &AbbrevData = Abbrev.getData(); for (unsigned i = 0, N = Values.size(); i < N; ++i) @@ -472,7 +457,7 @@ unsigned DIELoc::ComputeSize(AsmPrinter *AP) const { /// EmitValue - Emit location data. /// -void DIELoc::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { +void DIELoc::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const { switch (Form) { default: llvm_unreachable("Improper form for block"); case dwarf::DW_FORM_block1: Asm->EmitInt8(Size); break; @@ -490,7 +475,7 @@ void DIELoc::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { /// SizeOf - Determine size of location data in bytes. /// -unsigned DIELoc::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIELoc::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { switch (Form) { case dwarf::DW_FORM_block1: return Size + sizeof(int8_t); case dwarf::DW_FORM_block2: return Size + sizeof(int16_t); @@ -515,7 +500,7 @@ void DIELoc::print(raw_ostream &O) const { /// ComputeSize - calculate the size of the block. /// -unsigned DIEBlock::ComputeSize(AsmPrinter *AP) const { +unsigned DIEBlock::ComputeSize(const AsmPrinter *AP) const { if (!Size) { const SmallVectorImpl &AbbrevData = Abbrev.getData(); for (unsigned i = 0, N = Values.size(); i < N; ++i) @@ -527,7 +512,7 @@ unsigned DIEBlock::ComputeSize(AsmPrinter *AP) const { /// EmitValue - Emit block data. /// -void DIEBlock::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { +void DIEBlock::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const { switch (Form) { default: llvm_unreachable("Improper form for block"); case dwarf::DW_FORM_block1: Asm->EmitInt8(Size); break; @@ -543,7 +528,7 @@ void DIEBlock::EmitValue(AsmPrinter *Asm, dwarf::Form Form) const { /// SizeOf - Determine size of block data in bytes. /// -unsigned DIEBlock::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIEBlock::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { switch (Form) { case dwarf::DW_FORM_block1: return Size + sizeof(int8_t); case dwarf::DW_FORM_block2: return Size + sizeof(int16_t); @@ -564,7 +549,7 @@ void DIEBlock::print(raw_ostream &O) const { // DIELocList Implementation //===----------------------------------------------------------------------===// -unsigned DIELocList::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { +unsigned DIELocList::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { if (Form == dwarf::DW_FORM_data4) return 4; if (Form == dwarf::DW_FORM_sec_offset) @@ -574,14 +559,14 @@ unsigned DIELocList::SizeOf(AsmPrinter *AP, dwarf::Form Form) const { /// EmitValue - Emit label value. /// -void DIELocList::EmitValue(AsmPrinter *AP, dwarf::Form Form) const { +void DIELocList::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { DwarfDebug *DD = AP->getDwarfDebug(); MCSymbol *Label = DD->getDebugLocEntries()[Index].Label; if (AP->MAI->doesDwarfUseRelocationsAcrossSections() && !DD->useSplitDwarf()) - AP->EmitSectionOffset(Label, DD->getDebugLocSym()); + AP->emitSectionOffset(Label); else - AP->EmitLabelDifference(Label, DD->getDebugLocSym(), 4); + AP->EmitLabelDifference(Label, Label->getSection().getBeginSymbol(), 4); } #ifndef NDEBUG diff --git a/lib/CodeGen/AsmPrinter/DIEHash.cpp b/lib/CodeGen/AsmPrinter/DIEHash.cpp index 1e2ba2c..da7252a 100644 --- a/lib/CodeGen/AsmPrinter/DIEHash.cpp +++ b/lib/CodeGen/AsmPrinter/DIEHash.cpp @@ -510,7 +510,7 @@ uint64_t DIEHash::computeDIEODRSignature(const DIE &Die) { // ... take the least significant 8 bytes and return those. Our MD5 // implementation always returns its results in little endian, swap bytes // appropriately. - return *reinterpret_cast(Result + 8); + return support::endian::read64le(Result + 8); } /// This is based on the type signature computation given in section 7.27 of the @@ -531,7 +531,7 @@ uint64_t DIEHash::computeCUSignature(const DIE &Die) { // ... take the least significant 8 bytes and return those. Our MD5 // implementation always returns its results in little endian, swap bytes // appropriately. - return *reinterpret_cast(Result + 8); + return support::endian::read64le(Result + 8); } /// This is based on the type signature computation given in section 7.27 of the @@ -555,5 +555,5 @@ uint64_t DIEHash::computeTypeSignature(const DIE &Die) { // ... take the least significant 8 bytes and return those. Our MD5 // implementation always returns its results in little endian, swap bytes // appropriately. - return *reinterpret_cast(Result + 8); + return support::endian::read64le(Result + 8); } diff --git a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp index 0c2a5e5..bbdf237 100644 --- a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp +++ b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp @@ -14,6 +14,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/DebugInfo.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetRegisterInfo.h" #include #include diff --git a/lib/CodeGen/AsmPrinter/DebugLocEntry.h b/lib/CodeGen/AsmPrinter/DebugLocEntry.h index 6d55c03..6914bbe 100644 --- a/lib/CodeGen/AsmPrinter/DebugLocEntry.h +++ b/lib/CodeGen/AsmPrinter/DebugLocEntry.h @@ -9,22 +9,24 @@ #ifndef LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGLOCENTRY_H #define LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGLOCENTRY_H +#include "llvm/ADT/SmallString.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MachineLocation.h" namespace llvm { +class AsmPrinter; class MDNode; /// \brief This struct describes location entries emitted in the .debug_loc /// section. class DebugLocEntry { - // Begin and end symbols for the address range that this location is valid. + /// Begin and end symbols for the address range that this location is valid. const MCSymbol *Begin; const MCSymbol *End; public: - /// A single location or constant. + /// \brief A single location or constant. struct Value { Value(const MDNode *Var, const MDNode *Expr, int64_t i) : Variable(Var), Expression(Expr), EntryKind(E_Integer) { @@ -41,20 +43,20 @@ public: Value(const MDNode *Var, const MDNode *Expr, MachineLocation Loc) : Variable(Var), Expression(Expr), EntryKind(E_Location), Loc(Loc) { assert(DIVariable(Var).Verify()); - assert(DIExpression(Expr).Verify()); + assert(DIExpression(Expr)->isValid()); } - // The variable to which this location entry corresponds. + /// The variable to which this location entry corresponds. const MDNode *Variable; - // Any complex address location expression for this Value. + /// Any complex address location expression for this Value. const MDNode *Expression; - // Type of entry that this represents. + /// Type of entry that this represents. enum EntryType { E_Location, E_Integer, E_ConstantFP, E_ConstantInt }; enum EntryType EntryKind; - // Either a constant, + /// Either a constant, union { int64_t Int; const ConstantFP *CFP; @@ -84,6 +86,8 @@ private: /// A nonempty list of locations/constants belonging to this entry, /// sorted by offset. SmallVector Values; + SmallString<8> DWARFBytes; + SmallVector Comments; public: DebugLocEntry(const MCSymbol *B, const MCSymbol *E, Value Val) @@ -92,9 +96,9 @@ public: } /// \brief If this and Next are describing different pieces of the same - // variable, merge them by appending Next's values to the current - // list of values. - // Return true if the merge was successful. + /// variable, merge them by appending Next's values to the current + /// list of values. + /// Return true if the merge was successful. bool MergeValues(const DebugLocEntry &Next) { if (Begin == Next.Begin) { DIExpression Expr(Values[0].Expression); @@ -135,7 +139,7 @@ public: }) && "value must be a piece"); } - // Sort the pieces by offset. + // \brief Sort the pieces by offset. // Remove any duplicate entries by dropping all but the first. void sortUniqueValues() { std::sort(Values.begin(), Values.end()); @@ -146,9 +150,18 @@ public: }), Values.end()); } + + /// \brief Lower this entry into a DWARF expression. + void finalize(const AsmPrinter &AP, + const DITypeIdentifierMap &TypeIdentifierMap); + + /// \brief Return the lowered DWARF expression. + StringRef getDWARFBytes() const { return DWARFBytes; } + /// \brief Return the assembler comments for the lowered DWARF expression. + const SmallVectorImpl &getComments() const { return Comments; } }; -/// Compare two Values for equality. +/// \brief Compare two Values for equality. inline bool operator==(const DebugLocEntry::Value &A, const DebugLocEntry::Value &B) { if (A.EntryKind != B.EntryKind) @@ -173,7 +186,7 @@ inline bool operator==(const DebugLocEntry::Value &A, llvm_unreachable("unhandled EntryKind"); } -/// Compare two pieces based on their offset. +/// \brief Compare two pieces based on their offset. inline bool operator<(const DebugLocEntry::Value &A, const DebugLocEntry::Value &B) { return A.getExpression().getBitPieceOffset() < diff --git a/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp b/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp index a71f35e..f64338e 100644 --- a/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp @@ -54,7 +54,7 @@ void DwarfAccelTable::ComputeBucketCount(void) { // Then compute the bucket size, minimum of 1 bucket. if (num > 1024) Header.bucket_count = num / 4; - if (num > 16) + else if (num > 16) Header.bucket_count = num / 2; else Header.bucket_count = num > 0 ? num : 1; @@ -70,6 +70,7 @@ static bool compareDIEs(const DwarfAccelTable::HashDataContents *A, void DwarfAccelTable::FinalizeTable(AsmPrinter *Asm, StringRef Prefix) { // Create the individual hash data outputs. + Data.reserve(Entries.size()); for (StringMap::iterator EI = Entries.begin(), EE = Entries.end(); EI != EE; ++EI) { @@ -95,8 +96,17 @@ void DwarfAccelTable::FinalizeTable(AsmPrinter *Asm, StringRef Prefix) { for (size_t i = 0, e = Data.size(); i < e; ++i) { uint32_t bucket = Data[i]->HashValue % Header.bucket_count; Buckets[bucket].push_back(Data[i]); - Data[i]->Sym = Asm->GetTempSymbol(Prefix, i); + Data[i]->Sym = Asm->createTempSymbol(Prefix); } + + // Sort the contents of the buckets by hash value so that hash + // collisions end up together. Stable sort makes testing easier and + // doesn't cost much more. + for (size_t i = 0; i < Buckets.size(); ++i) + std::stable_sort(Buckets[i].begin(), Buckets[i].end(), + [] (HashData *LHS, HashData *RHS) { + return LHS->HashValue < RHS->HashValue; + }); } // Emits the header for the table via the AsmPrinter. @@ -136,19 +146,32 @@ void DwarfAccelTable::EmitBuckets(AsmPrinter *Asm) { Asm->EmitInt32(index); else Asm->EmitInt32(UINT32_MAX); - index += Buckets[i].size(); + // Buckets point in the list of hashes, not to the data. Do not + // increment the index multiple times in case of hash collisions. + uint64_t PrevHash = UINT64_MAX; + for (auto *HD : Buckets[i]) { + uint32_t HashValue = HD->HashValue; + if (PrevHash != HashValue) + ++index; + PrevHash = HashValue; + } } } // Walk through the buckets and emit the individual hashes for each // bucket. void DwarfAccelTable::EmitHashes(AsmPrinter *Asm) { + uint64_t PrevHash = UINT64_MAX; for (size_t i = 0, e = Buckets.size(); i < e; ++i) { for (HashList::const_iterator HI = Buckets[i].begin(), HE = Buckets[i].end(); HI != HE; ++HI) { + uint32_t HashValue = (*HI)->HashValue; + if (PrevHash == HashValue) + continue; Asm->OutStreamer.AddComment("Hash in Bucket " + Twine(i)); - Asm->EmitInt32((*HI)->HashValue); + Asm->EmitInt32(HashValue); + PrevHash = HashValue; } } } @@ -157,11 +180,16 @@ void DwarfAccelTable::EmitHashes(AsmPrinter *Asm) { // element in each bucket. This is done via a symbol subtraction from the // beginning of the section. The non-section symbol will be output later // when we emit the actual data. -void DwarfAccelTable::EmitOffsets(AsmPrinter *Asm, MCSymbol *SecBegin) { +void DwarfAccelTable::emitOffsets(AsmPrinter *Asm, const MCSymbol *SecBegin) { + uint64_t PrevHash = UINT64_MAX; for (size_t i = 0, e = Buckets.size(); i < e; ++i) { for (HashList::const_iterator HI = Buckets[i].begin(), HE = Buckets[i].end(); HI != HE; ++HI) { + uint32_t HashValue = (*HI)->HashValue; + if (PrevHash == HashValue) + continue; + PrevHash = HashValue; Asm->OutStreamer.AddComment("Offset in Bucket " + Twine(i)); MCContext &Context = Asm->OutStreamer.getContext(); const MCExpr *Sub = MCBinaryExpr::CreateSub( @@ -175,17 +203,20 @@ void DwarfAccelTable::EmitOffsets(AsmPrinter *Asm, MCSymbol *SecBegin) { // Walk through the buckets and emit the full data for each element in // the bucket. For the string case emit the dies and the various offsets. // Terminate each HashData bucket with 0. -void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D, - MCSymbol *StrSym) { - uint64_t PrevHash = UINT64_MAX; +void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D) { for (size_t i = 0, e = Buckets.size(); i < e; ++i) { + uint64_t PrevHash = UINT64_MAX; for (HashList::const_iterator HI = Buckets[i].begin(), HE = Buckets[i].end(); HI != HE; ++HI) { + // Terminate the previous entry if there is no hash collision + // with the current one. + if (PrevHash != UINT64_MAX && PrevHash != (*HI)->HashValue) + Asm->EmitInt32(0); // Remember to emit the label for our offset. Asm->OutStreamer.EmitLabel((*HI)->Sym); Asm->OutStreamer.AddComment((*HI)->Str); - Asm->EmitSectionOffset((*HI)->Data.StrSym, StrSym); + Asm->emitSectionOffset((*HI)->Data.StrSym); Asm->OutStreamer.AddComment("Num DIEs"); Asm->EmitInt32((*HI)->Data.Values.size()); for (HashDataContents *HD : (*HI)->Data.Values) { @@ -200,17 +231,17 @@ void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D, Asm->EmitInt8(HD->Flags); } } - // Emit a 0 to terminate the data unless we have a hash collision. - if (PrevHash != (*HI)->HashValue) - Asm->EmitInt32(0); PrevHash = (*HI)->HashValue; } + // Emit the final end marker for the bucket. + if (!Buckets[i].empty()) + Asm->EmitInt32(0); } } // Emit the entire data structure to the output file. -void DwarfAccelTable::Emit(AsmPrinter *Asm, MCSymbol *SecBegin, DwarfDebug *D, - MCSymbol *StrSym) { +void DwarfAccelTable::emit(AsmPrinter *Asm, const MCSymbol *SecBegin, + DwarfDebug *D) { // Emit the header. EmitHeader(Asm); @@ -221,10 +252,10 @@ void DwarfAccelTable::Emit(AsmPrinter *Asm, MCSymbol *SecBegin, DwarfDebug *D, EmitHashes(Asm); // Emit the offsets. - EmitOffsets(Asm, SecBegin); + emitOffsets(Asm, SecBegin); // Emit the hash data. - EmitData(Asm, D, StrSym); + EmitData(Asm, D); } #ifndef NDEBUG diff --git a/lib/CodeGen/AsmPrinter/DwarfAccelTable.h b/lib/CodeGen/AsmPrinter/DwarfAccelTable.h index 74963da..e6fdf08 100644 --- a/lib/CodeGen/AsmPrinter/DwarfAccelTable.h +++ b/lib/CodeGen/AsmPrinter/DwarfAccelTable.h @@ -222,8 +222,8 @@ private: void EmitHeader(AsmPrinter *); void EmitBuckets(AsmPrinter *); void EmitHashes(AsmPrinter *); - void EmitOffsets(AsmPrinter *, MCSymbol *); - void EmitData(AsmPrinter *, DwarfDebug *D, MCSymbol *StrSym); + void emitOffsets(AsmPrinter *, const MCSymbol *); + void EmitData(AsmPrinter *, DwarfDebug *D); // Allocator for HashData and HashDataContents. BumpPtrAllocator Allocator; @@ -248,7 +248,7 @@ public: void AddName(StringRef Name, MCSymbol *StrSym, const DIE *Die, char Flags = 0); void FinalizeTable(AsmPrinter *, StringRef); - void Emit(AsmPrinter *, MCSymbol *, DwarfDebug *, MCSymbol *StrSym); + void emit(AsmPrinter *, const MCSymbol *, DwarfDebug *); #ifndef NDEBUG void print(raw_ostream &O); void dump() { print(dbgs()); } diff --git a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp index f45b24c..1bee367 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -39,9 +39,24 @@ #include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; +DwarfCFIExceptionBase::DwarfCFIExceptionBase(AsmPrinter *A) + : EHStreamer(A), shouldEmitCFI(false) {} + +void DwarfCFIExceptionBase::markFunctionEnd() { + if (shouldEmitCFI) + Asm->OutStreamer.EmitCFIEndProc(); + + if (MMI->getLandingPads().empty()) + return; + + // Map all labels and get rid of any dead landing pads. + MMI->TidyLandingPads(); +} + DwarfCFIException::DwarfCFIException(AsmPrinter *A) - : EHStreamer(A), shouldEmitPersonality(false), shouldEmitLSDA(false), - shouldEmitMoves(false), moveTypeModule(AsmPrinter::CFI_M_None) {} + : DwarfCFIExceptionBase(A), shouldEmitPersonality(false), + shouldEmitLSDA(false), shouldEmitMoves(false), + moveTypeModule(AsmPrinter::CFI_M_None) {} DwarfCFIException::~DwarfCFIException() {} @@ -72,8 +87,6 @@ void DwarfCFIException::endModule() { } } -/// beginFunction - Gather pre-function exception information. Assumes it's -/// being emitted immediately after the function entry point. void DwarfCFIException::beginFunction(const MachineFunction *MF) { shouldEmitMoves = shouldEmitPersonality = shouldEmitLSDA = false; @@ -100,7 +113,8 @@ void DwarfCFIException::beginFunction(const MachineFunction *MF) { shouldEmitLSDA = shouldEmitPersonality && LSDAEncoding != dwarf::DW_EH_PE_omit; - if (!shouldEmitPersonality && !shouldEmitMoves) + shouldEmitCFI = shouldEmitPersonality || shouldEmitMoves; + if (!shouldEmitCFI) return; Asm->OutStreamer.EmitCFIStartProc(/*IsSimple=*/false); @@ -113,43 +127,18 @@ void DwarfCFIException::beginFunction(const MachineFunction *MF) { TLOF.getCFIPersonalitySymbol(Per, *Asm->Mang, Asm->TM, MMI); Asm->OutStreamer.EmitCFIPersonality(Sym, PerEncoding); - MCSymbol *EHBegin = - Asm->GetTempSymbol("eh_func_begin", Asm->getFunctionNumber()); - if (Asm->MAI->useAssignmentForEHBegin()) { - MCContext &Ctx = Asm->OutContext; - MCSymbol *CurPos = Ctx.CreateTempSymbol(); - Asm->OutStreamer.EmitLabel(CurPos); - Asm->OutStreamer.EmitAssignment(EHBegin, - MCSymbolRefExpr::Create(CurPos, Ctx)); - } else { - Asm->OutStreamer.EmitLabel(EHBegin); - } - // Provide LSDA information. if (!shouldEmitLSDA) return; - Asm->OutStreamer.EmitCFILsda(Asm->GetTempSymbol("exception", - Asm->getFunctionNumber()), - LSDAEncoding); + Asm->OutStreamer.EmitCFILsda(Asm->getCurExceptionSym(), LSDAEncoding); } /// endFunction - Gather and emit post-function exception information. /// void DwarfCFIException::endFunction(const MachineFunction *) { - if (!shouldEmitPersonality && !shouldEmitMoves) - return; - - Asm->OutStreamer.EmitCFIEndProc(); - if (!shouldEmitPersonality) return; - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_end", - Asm->getFunctionNumber())); - - // Map all labels and get rid of any dead landing pads. - MMI->TidyLandingPads(); - emitExceptionTable(); } diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index dcc5fe4..eee5fc5 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -19,7 +19,7 @@ DwarfCompileUnit::DwarfCompileUnit(unsigned UID, DICompileUnit Node, AsmPrinter *A, DwarfDebug *DW, DwarfFile *DWU) : DwarfUnit(UID, dwarf::DW_TAG_compile_unit, Node, A, DW, DWU), - Skeleton(nullptr), LabelBegin(nullptr), BaseAddress(nullptr) { + Skeleton(nullptr), BaseAddress(nullptr) { insertDIE(Node, &getUnitDie()); } @@ -164,24 +164,17 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) { addUInt(*Loc, dwarf::DW_FORM_udata, DD->getAddressPool().getIndex(Sym, /* TLS */ true)); } - // 3) followed by a custom OP to make the debugger do a TLS lookup. - addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_GNU_push_tls_address); + // 3) followed by an OP to make the debugger do a TLS lookup. + addUInt(*Loc, dwarf::DW_FORM_data1, + DD->useGNUTLSOpcode() ? dwarf::DW_OP_GNU_push_tls_address + : dwarf::DW_OP_form_tls_address); } else { DD->addArangeLabel(SymbolCU(this, Sym)); addOpAddress(*Loc, Sym); } addBlock(*VariableDIE, dwarf::DW_AT_location, Loc); - // Add the linkage name. - StringRef LinkageName = GV.getLinkageName(); - if (!LinkageName.empty()) - // From DWARF4: DIEs to which DW_AT_linkage_name may apply include: - // TAG_common_block, TAG_constant, TAG_entry_point, TAG_subprogram and - // TAG_variable. - addString(*VariableDIE, - DD->getDwarfVersion() >= 4 ? dwarf::DW_AT_linkage_name - : dwarf::DW_AT_MIPS_linkage_name, - GlobalValue::getRealLinkageName(LinkageName)); + addLinkageName(*VariableDIE, GV.getLinkageName()); } else if (const ConstantInt *CI = dyn_cast_or_null(GV.getConstant())) { addConstantValue(*VariableDIE, CI, GTy); @@ -243,7 +236,7 @@ void DwarfCompileUnit::addSectionLabel(DIE &Die, dwarf::Attribute Attribute, addSectionDelta(Die, Attribute, Label, Sec); } -void DwarfCompileUnit::initStmtList(MCSymbol *DwarfLineSectionSym) { +void DwarfCompileUnit::initStmtList() { // Define start line table label for each Compile Unit. MCSymbol *LineTableStartSym = Asm->OutStreamer.getDwarfLineTableSymbol(getUniqueID()); @@ -255,8 +248,9 @@ void DwarfCompileUnit::initStmtList(MCSymbol *DwarfLineSectionSym) { // left in the skeleton CU and so not included. // The line table entries are not always emitted in assembly, so it // is not okay to use line_table_start here. + const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); addSectionLabel(UnitDie, dwarf::DW_AT_stmt_list, LineTableStartSym, - DwarfLineSectionSym); + TLOF.getDwarfLineSection()->getBeginSymbol()); } void DwarfCompileUnit::applyStmtList(DIE &D) { @@ -285,7 +279,7 @@ void DwarfCompileUnit::attachLowHighPC(DIE &D, const MCSymbol *Begin, DIE &DwarfCompileUnit::updateSubprogramScopeDIE(DISubprogram SP) { DIE *SPDie = getOrCreateSubprogramDIE(SP, includeMinimalInlineScopes()); - attachLowHighPC(*SPDie, DD->getFunctionBeginSym(), DD->getFunctionEndSym()); + attachLowHighPC(*SPDie, Asm->getFunctionBegin(), Asm->getFunctionEnd()); if (!DD->getCurrentFunction()->getTarget().Options.DisableFramePointerElim( *DD->getCurrentFunction())) addFlag(*SPDie, dwarf::DW_AT_APPLE_omit_frame_ptr); @@ -378,13 +372,14 @@ void DwarfCompileUnit::addSectionDelta(DIE &Die, dwarf::Attribute Attribute, void DwarfCompileUnit::addScopeRangeList(DIE &ScopeDIE, SmallVector Range) { + const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); + // Emit offset in .debug_range as a relocatable label. emitDIE will handle // emitting it appropriately. - auto *RangeSectionSym = DD->getRangeSectionSym(); + const MCSymbol *RangeSectionSym = + TLOF.getDwarfRangesSection()->getBeginSymbol(); - RangeSpanList List( - Asm->GetTempSymbol("debug_ranges", DD->getNextRangeNumber()), - std::move(Range)); + RangeSpanList List(Asm->createTempSymbol("debug_ranges"), std::move(Range)); // Under fission, ranges are specified by constant offsets relative to the // CU's DW_AT_GNU_ranges_base. @@ -709,12 +704,14 @@ void DwarfCompileUnit::collectDeadVariables(DISubprogram SP) { } } -void DwarfCompileUnit::emitHeader(const MCSymbol *ASectionSym) const { +void DwarfCompileUnit::emitHeader(bool UseOffsets) { // Don't bother labeling the .dwo unit, as its offset isn't used. - if (!Skeleton) + if (!Skeleton) { + LabelBegin = Asm->createTempSymbol("cu_begin"); Asm->OutStreamer.EmitLabel(LabelBegin); + } - DwarfUnit::emitHeader(ASectionSym); + DwarfUnit::emitHeader(UseOffsets); } /// addGlobalName - Add a new global name to the compile unit. diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h index c66af65..9484bb6 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h @@ -36,9 +36,6 @@ class DwarfCompileUnit : public DwarfUnit { /// Skeleton unit associated with this unit. DwarfCompileUnit *Skeleton; - /// A label at the start of the non-dwo section related to this unit. - MCSymbol *SectionSym; - /// The start of the unit within its section. MCSymbol *LabelBegin; @@ -76,7 +73,7 @@ public: return Skeleton; } - void initStmtList(MCSymbol *DwarfLineSectionSym); + void initStmtList(); /// Apply the DW_AT_stmt_list from this compile unit to the specified DIE. void applyStmtList(DIE &D); @@ -168,22 +165,9 @@ public: /// Set the skeleton unit associated with this unit. void setSkeleton(DwarfCompileUnit &Skel) { Skeleton = &Skel; } - MCSymbol *getSectionSym() const { + const MCSymbol *getSectionSym() const { assert(Section); - return SectionSym; - } - - /// Pass in the SectionSym even though we could recreate it in every compile - /// unit (type units will have actually distinct symbols once they're in - /// comdat sections). - void initSection(const MCSection *Section, MCSymbol *SectionSym) { - DwarfUnit::initSection(Section); - this->SectionSym = SectionSym; - - // Don't bother labeling the .dwo unit, as its offset isn't used. - if (!Skeleton) - LabelBegin = - Asm->GetTempSymbol(Section->getLabelBeginName(), getUniqueID()); + return Section->getBeginSymbol(); } unsigned getLength() { @@ -191,7 +175,7 @@ public: getHeaderSize() + UnitDie.getSize(); } - void emitHeader(const MCSymbol *ASectionSym) const override; + void emitHeader(bool UseOffsets) override; MCSymbol *getLabelBegin() const { assert(Section); diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index aa1f79f..e9ebd97 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -45,6 +45,7 @@ #include "llvm/Support/MD5.h" #include "llvm/Support/Path.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" @@ -105,6 +106,25 @@ DwarfPubSections("generate-dwarf-pub-sections", cl::Hidden, static const char *const DWARFGroupName = "DWARF Emission"; static const char *const DbgTimerName = "DWARF Debug Writer"; +void DebugLocDwarfExpression::EmitOp(uint8_t Op, const char *Comment) { + BS.EmitInt8( + Op, Comment ? Twine(Comment) + " " + dwarf::OperationEncodingString(Op) + : dwarf::OperationEncodingString(Op)); +} + +void DebugLocDwarfExpression::EmitSigned(int64_t Value) { + BS.EmitSLEB128(Value, Twine(Value)); +} + +void DebugLocDwarfExpression::EmitUnsigned(uint64_t Value) { + BS.EmitULEB128(Value, Twine(Value)); +} + +bool DebugLocDwarfExpression::isFrameRegister(unsigned MachineReg) { + // This information is not available while emitting .debug_loc entries. + return false; +} + //===----------------------------------------------------------------------===// /// resolve - Look in the DwarfDebug map for the MDNode that @@ -169,11 +189,12 @@ static LLVM_CONSTEXPR DwarfAccelTable::Atom TypeAtoms[] = { DwarfAccelTable::Atom(dwarf::DW_ATOM_type_flags, dwarf::DW_FORM_data1)}; DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) - : Asm(A), MMI(Asm->MMI), PrevLabel(nullptr), GlobalRangeCount(0), - InfoHolder(A, *this, "info_string", DIEValueAllocator), + : Asm(A), MMI(Asm->MMI), PrevLabel(nullptr), + InfoHolder(A, "info_string", DIEValueAllocator), UsedNonDefaultText(false), - SkeletonHolder(A, *this, "skel_string", DIEValueAllocator), + SkeletonHolder(A, "skel_string", DIEValueAllocator), IsDarwin(Triple(A->getTargetTriple()).isOSDarwin()), + IsPS4(Triple(A->getTargetTriple()).isPS4()), AccelNames(DwarfAccelTable::Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)), AccelObjC(DwarfAccelTable::Atom(dwarf::DW_ATOM_die_offset, @@ -182,17 +203,11 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) dwarf::DW_FORM_data4)), AccelTypes(TypeAtoms) { - DwarfInfoSectionSym = DwarfAbbrevSectionSym = DwarfStrSectionSym = nullptr; - DwarfDebugRangeSectionSym = DwarfDebugLocSectionSym = nullptr; - DwarfLineSectionSym = nullptr; - DwarfAddrSectionSym = nullptr; - DwarfAbbrevDWOSectionSym = DwarfStrDWOSectionSym = nullptr; - FunctionBeginSym = FunctionEndSym = nullptr; CurFn = nullptr; CurMI = nullptr; // Turn on accelerator tables for Darwin by default, pubnames by - // default for non-Darwin, and handle split dwarf. + // default for non-Darwin/PS4, and handle split dwarf. if (DwarfAccelTables == Default) HasDwarfAccelTables = IsDarwin; else @@ -204,7 +219,7 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) HasSplitDwarf = SplitDwarf == Enable; if (DwarfPubSections == Default) - HasDwarfPubSections = !IsDarwin; + HasDwarfPubSections = !IsDarwin && !IsPS4; else HasDwarfPubSections = DwarfPubSections == Enable; @@ -212,6 +227,10 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) DwarfVersion = DwarfVersionNumber ? DwarfVersionNumber : MMI->getModule()->getDwarfVersion(); + // Darwin and PS4 use the standard TLS opcode (defined in DWARF 3). + // Everybody else uses GNU's. + UseGNUTLSOpcode = !(IsDarwin || IsPS4) || DwarfVersion < 3; + Asm->OutStreamer.getContext().setDwarfVersion(DwarfVersion); { @@ -223,19 +242,6 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) // Define out of line so we don't have to include DwarfUnit.h in DwarfDebug.h. DwarfDebug::~DwarfDebug() { } -// Switch to the specified MCSection and emit an assembler -// temporary label to it if SymbolStem is specified. -static MCSymbol *emitSectionSym(AsmPrinter *Asm, const MCSection *Section, - const char *SymbolStem = nullptr) { - Asm->OutStreamer.SwitchSection(Section); - if (!SymbolStem) - return nullptr; - - MCSymbol *TmpSym = Asm->GetTempSymbol(SymbolStem); - Asm->OutStreamer.EmitLabel(TmpSym); - return TmpSym; -} - static bool isObjCClass(StringRef Name) { return Name.startswith("+") || Name.startswith("-"); } @@ -264,13 +270,6 @@ static StringRef getObjCMethodName(StringRef In) { return In.slice(In.find(' ') + 1, In.find(']')); } -// Helper for sorting sections into a stable output order. -static bool SectionSort(const MCSection *A, const MCSection *B) { - std::string LA = (A ? A->getLabelBeginName() : ""); - std::string LB = (B ? B->getLabelBeginName() : ""); - return LA < LB; -} - // Add the various names to the Dwarf accelerator table names. // TODO: Determine whether or not we should add names for programs // that do not have a DW_AT_name or DW_AT_linkage_name field - this @@ -388,7 +387,7 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) { NewCU.addString(Die, dwarf::DW_AT_name, FN); if (!useSplitDwarf()) { - NewCU.initStmtList(DwarfLineSectionSym); + NewCU.initStmtList(); // If we're using split dwarf the compilation dir is going to be in the // skeleton CU and so we don't need to duplicate it here. @@ -410,11 +409,9 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) { dwarf::DW_FORM_data1, RVer); if (useSplitDwarf()) - NewCU.initSection(Asm->getObjFileLowering().getDwarfInfoDWOSection(), - DwarfInfoDWOSectionSym); + NewCU.initSection(Asm->getObjFileLowering().getDwarfInfoDWOSection()); else - NewCU.initSection(Asm->getObjFileLowering().getDwarfInfoSection(), - DwarfInfoSectionSym); + NewCU.initSection(Asm->getObjFileLowering().getDwarfInfoSection()); CUMap.insert(std::make_pair(DIUnit, &NewCU)); CUDieMap.insert(std::make_pair(&Die, &NewCU)); @@ -445,9 +442,6 @@ void DwarfDebug::beginModule() { return; TypeIdentifierMap = generateDITypeIdentifierMap(CU_Nodes); - // Emit initial sections so we can reference labels later. - emitSectionLabels(); - SingleCU = CU_Nodes->getNumOperands() == 1; for (MDNode *N : CU_Nodes->operands()) { @@ -458,8 +452,11 @@ void DwarfDebug::beginModule() { ScopesWithImportedEntities.push_back(std::make_pair( DIImportedEntity(ImportedEntities.getElement(i)).getContext(), ImportedEntities.getElement(i))); - std::sort(ScopesWithImportedEntities.begin(), - ScopesWithImportedEntities.end(), less_first()); + // Stable sort to preserve the order of appearance of imported entities. + // This is to avoid out-of-order processing of interdependent declarations + // within the same scope, e.g. { namespace A = base; namespace B = A; } + std::stable_sort(ScopesWithImportedEntities.begin(), + ScopesWithImportedEntities.end(), less_first()); DIArray GVs = CUNode.getGlobalVariables(); for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i) CU.getOrCreateGlobalVariableDIE(DIGlobalVariable(GVs.getElement(i))); @@ -541,6 +538,8 @@ void DwarfDebug::collectDeadVariables() { } void DwarfDebug::finalizeModuleInfo() { + const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); + finishSubprogramDefinitions(); finishVariableDefinitions(); @@ -570,13 +569,16 @@ void DwarfDebug::finalizeModuleInfo() { // We don't keep track of which addresses are used in which CU so this // is a bit pessimistic under LTO. - if (!AddrPool.isEmpty()) + if (!AddrPool.isEmpty()) { + const MCSymbol *Sym = TLOF.getDwarfAddrSection()->getBeginSymbol(); SkCU->addSectionLabel(SkCU->getUnitDie(), dwarf::DW_AT_GNU_addr_base, - DwarfAddrSectionSym, DwarfAddrSectionSym); - if (!SkCU->getRangeLists().empty()) + Sym, Sym); + } + if (!SkCU->getRangeLists().empty()) { + const MCSymbol *Sym = TLOF.getDwarfRangesSection()->getBeginSymbol(); SkCU->addSectionLabel(SkCU->getUnitDie(), dwarf::DW_AT_GNU_ranges_base, - DwarfDebugRangeSectionSym, - DwarfDebugRangeSectionSym); + Sym, Sym); + } } // If we have code split among multiple sections or non-contiguous @@ -613,7 +615,7 @@ void DwarfDebug::endModule() { // If we aren't actually generating debug info (check beginModule - // conditionalized on !DisableDebugInfoPrinting and the presence of the // llvm.dbg.cu metadata node) - if (!DwarfInfoSectionSym) + if (!MMI->hasDebugInfo()) return; // Finalize the debug info for the module. @@ -621,12 +623,18 @@ void DwarfDebug::endModule() { emitDebugStr(); - // Emit all the DIEs into a debug info section. - emitDebugInfo(); + if (useSplitDwarf()) + emitDebugLocDWO(); + else + // Emit info into a debug loc section. + emitDebugLoc(); // Corresponding abbreviations into a abbrev section. emitAbbreviations(); + // Emit all the DIEs into a debug info section. + emitDebugInfo(); + // Emit info into a debug aranges section. if (GenerateARangeSection) emitDebugARanges(); @@ -639,12 +647,9 @@ void DwarfDebug::endModule() { emitDebugInfoDWO(); emitDebugAbbrevDWO(); emitDebugLineDWO(); - emitDebugLocDWO(); // Emit DWO addresses. AddrPool.emit(*Asm, Asm->getObjFileLowering().getDwarfAddrSection()); - } else - // Emit info into a debug loc section. - emitDebugLoc(); + } // Emit info into the dwarf accelerator table sections. if (useDwarfAccelTables()) { @@ -828,7 +833,7 @@ DwarfDebug::buildLocationList(SmallVectorImpl &DebugLoc, if (End != nullptr) EndLabel = getLabelAfterInsn(End); else if (std::next(I) == Ranges.end()) - EndLabel = FunctionEndSym; + EndLabel = Asm->getFunctionEnd(); else EndLabel = getLabelBeforeInsn(std::next(I)->first); assert(EndLabel && "Forgot label after instruction ending a range!"); @@ -922,11 +927,13 @@ DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP, DotDebugLocEntries.resize(DotDebugLocEntries.size() + 1); DebugLocList &LocList = DotDebugLocEntries.back(); LocList.CU = &TheCU; - LocList.Label = - Asm->GetTempSymbol("debug_loc", DotDebugLocEntries.size() - 1); + LocList.Label = Asm->createTempSymbol("debug_loc"); // Build the location list for this variable. buildLocationList(LocList.List, Ranges); + // Finalize the entry by lowering it into a DWARF bytestream. + for (auto &Entry : LocList.List) + Entry.finalize(*Asm, TypeIdentifierMap); } // Collect info for variables that were optimized out. @@ -964,23 +971,25 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { // Check if source location changes, but ignore DBG_VALUE locations. if (!MI->isDebugValue()) { DebugLoc DL = MI->getDebugLoc(); - if (DL != PrevInstLoc && (!DL.isUnknown() || UnknownLocations)) { - unsigned Flags = 0; - PrevInstLoc = DL; - if (DL == PrologEndLoc) { - Flags |= DWARF2_FLAG_PROLOGUE_END; - PrologEndLoc = DebugLoc(); - Flags |= DWARF2_FLAG_IS_STMT; - } - if (DL.getLine() != - Asm->OutStreamer.getContext().getCurrentDwarfLoc().getLine()) - Flags |= DWARF2_FLAG_IS_STMT; - + if (DL != PrevInstLoc) { if (!DL.isUnknown()) { + unsigned Flags = 0; + PrevInstLoc = DL; + if (DL == PrologEndLoc) { + Flags |= DWARF2_FLAG_PROLOGUE_END; + PrologEndLoc = DebugLoc(); + Flags |= DWARF2_FLAG_IS_STMT; + } + if (DL.getLine() != + Asm->OutStreamer.getContext().getCurrentDwarfLoc().getLine()) + Flags |= DWARF2_FLAG_IS_STMT; + const MDNode *Scope = DL.getScope(Asm->MF->getFunction()->getContext()); recordSourceLine(DL.getLine(), DL.getCol(), Scope, Flags); - } else + } else if (UnknownLocations) { + PrevInstLoc = DL; recordSourceLine(0, 0, nullptr, 0); + } } } @@ -1116,11 +1125,6 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { else Asm->OutStreamer.getContext().setDwarfCompileUnitID(TheCU->getUniqueID()); - // Emit a label for the function so that we have a beginning address. - FunctionBeginSym = Asm->GetTempSymbol("func_begin", Asm->getFunctionNumber()); - // Assumes in correct section after the entry point. - Asm->OutStreamer.EmitLabel(FunctionBeginSym); - // Calculate history for local variables. calculateDbgValueHistory(MF, Asm->MF->getSubtarget().getRegisterInfo(), DbgValues); @@ -1131,12 +1135,12 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { if (Ranges.empty()) continue; - // The first mention of a function argument gets the FunctionBeginSym + // The first mention of a function argument gets the CurrentFnBegin // label, so arguments are visible when breaking at function entry. DIVariable DIVar(Ranges.front().first->getDebugVariable()); if (DIVar.isVariable() && DIVar.getTag() == dwarf::DW_TAG_arg_variable && getDISubprogram(DIVar.getContext()).describes(MF->getFunction())) { - LabelsBeforeInsn[Ranges.front().first] = FunctionBeginSym; + LabelsBeforeInsn[Ranges.front().first] = Asm->getFunctionBegin(); if (Ranges.front().first->getDebugExpression().isBitPiece()) { // Mark all non-overlapping initial pieces. for (auto I = Ranges.begin(); I != Ranges.end(); ++I) { @@ -1145,7 +1149,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { [&](DbgValueHistoryMap::InstrRange Pred) { return !piecesOverlap(Piece, Pred.first->getDebugExpression()); })) - LabelsBeforeInsn[I->first] = FunctionBeginSym; + LabelsBeforeInsn[I->first] = Asm->getFunctionBegin(); else break; } @@ -1160,7 +1164,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { } PrevInstLoc = DebugLoc(); - PrevLabel = FunctionBeginSym; + PrevLabel = Asm->getFunctionBegin(); // Record beginning of function. PrologEndLoc = findPrologueEndLoc(MF); @@ -1191,11 +1195,6 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { return; } - // Define end label for subprogram. - FunctionEndSym = Asm->GetTempSymbol("func_end", Asm->getFunctionNumber()); - // Assumes in correct section after the entry point. - Asm->OutStreamer.EmitLabel(FunctionEndSym); - // Set DwarfDwarfCompileUnitID in MCContext to default value. Asm->OutStreamer.getContext().setDwarfCompileUnitID(0); @@ -1207,7 +1206,7 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { collectVariableInfo(TheCU, SP, ProcessedVars); // Add the range of this function to the list of ranges for the CU. - TheCU.addRange(RangeSpan(FunctionBeginSym, FunctionEndSym)); + TheCU.addRange(RangeSpan(Asm->getFunctionBegin(), Asm->getFunctionEnd())); // Under -gmlt, skip building the subprogram if there are no inlined // subroutines inside it. @@ -1290,103 +1289,10 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, // Emit Methods //===----------------------------------------------------------------------===// -// Emit initial Dwarf sections with a label at the start of each one. -void DwarfDebug::emitSectionLabels() { - const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); - - // Dwarf sections base addresses. - DwarfInfoSectionSym = - emitSectionSym(Asm, TLOF.getDwarfInfoSection(), "section_info"); - if (useSplitDwarf()) { - DwarfInfoDWOSectionSym = - emitSectionSym(Asm, TLOF.getDwarfInfoDWOSection(), "section_info_dwo"); - DwarfTypesDWOSectionSym = emitSectionSym( - Asm, TLOF.getDwarfTypesDWOSection(), "section_types_dwo"); - } - DwarfAbbrevSectionSym = - emitSectionSym(Asm, TLOF.getDwarfAbbrevSection(), "section_abbrev"); - if (useSplitDwarf()) - DwarfAbbrevDWOSectionSym = emitSectionSym( - Asm, TLOF.getDwarfAbbrevDWOSection(), "section_abbrev_dwo"); - if (GenerateARangeSection) - emitSectionSym(Asm, TLOF.getDwarfARangesSection()); - - DwarfLineSectionSym = - emitSectionSym(Asm, TLOF.getDwarfLineSection(), "section_line"); - if (GenerateGnuPubSections) { - DwarfGnuPubNamesSectionSym = - emitSectionSym(Asm, TLOF.getDwarfGnuPubNamesSection()); - DwarfGnuPubTypesSectionSym = - emitSectionSym(Asm, TLOF.getDwarfGnuPubTypesSection()); - } else if (HasDwarfPubSections) { - emitSectionSym(Asm, TLOF.getDwarfPubNamesSection()); - emitSectionSym(Asm, TLOF.getDwarfPubTypesSection()); - } - - DwarfStrSectionSym = - emitSectionSym(Asm, TLOF.getDwarfStrSection(), "info_string"); - if (useSplitDwarf()) { - DwarfStrDWOSectionSym = - emitSectionSym(Asm, TLOF.getDwarfStrDWOSection(), "skel_string"); - DwarfAddrSectionSym = - emitSectionSym(Asm, TLOF.getDwarfAddrSection(), "addr_sec"); - DwarfDebugLocSectionSym = - emitSectionSym(Asm, TLOF.getDwarfLocDWOSection(), "skel_loc"); - } else - DwarfDebugLocSectionSym = - emitSectionSym(Asm, TLOF.getDwarfLocSection(), "section_debug_loc"); - DwarfDebugRangeSectionSym = - emitSectionSym(Asm, TLOF.getDwarfRangesSection(), "debug_range"); -} - -// Recursively emits a debug information entry. -void DwarfDebug::emitDIE(DIE &Die) { - // Get the abbreviation for this DIE. - const DIEAbbrev &Abbrev = Die.getAbbrev(); - - // Emit the code (index) for the abbreviation. - if (Asm->isVerbose()) - Asm->OutStreamer.AddComment("Abbrev [" + Twine(Abbrev.getNumber()) + - "] 0x" + Twine::utohexstr(Die.getOffset()) + - ":0x" + Twine::utohexstr(Die.getSize()) + " " + - dwarf::TagString(Abbrev.getTag())); - Asm->EmitULEB128(Abbrev.getNumber()); - - const SmallVectorImpl &Values = Die.getValues(); - const SmallVectorImpl &AbbrevData = Abbrev.getData(); - - // Emit the DIE attribute values. - for (unsigned i = 0, N = Values.size(); i < N; ++i) { - dwarf::Attribute Attr = AbbrevData[i].getAttribute(); - dwarf::Form Form = AbbrevData[i].getForm(); - assert(Form && "Too many attributes for DIE (check abbreviation)"); - - if (Asm->isVerbose()) { - Asm->OutStreamer.AddComment(dwarf::AttributeString(Attr)); - if (Attr == dwarf::DW_AT_accessibility) - Asm->OutStreamer.AddComment(dwarf::AccessibilityString( - cast(Values[i])->getValue())); - } - - // Emit an attribute using the defined form. - Values[i]->EmitValue(Asm, Form); - } - - // Emit the DIE children if any. - if (Abbrev.hasChildren()) { - for (auto &Child : Die.getChildren()) - emitDIE(*Child); - - Asm->OutStreamer.AddComment("End Of Children Mark"); - Asm->EmitInt8(0); - } -} - // Emit the debug info section. void DwarfDebug::emitDebugInfo() { DwarfFile &Holder = useSplitDwarf() ? SkeletonHolder : InfoHolder; - - Holder.emitUnits(DwarfAbbrevSectionSym); + Holder.emitUnits(/* UseOffsets */ false); } // Emit the abbreviation section. @@ -1396,65 +1302,39 @@ void DwarfDebug::emitAbbreviations() { Holder.emitAbbrevs(Asm->getObjFileLowering().getDwarfAbbrevSection()); } -// Emit the last address of the section and the end of the line matrix. -void DwarfDebug::emitEndOfLineMatrix(unsigned SectionEnd) { - // Define last address of section. - Asm->OutStreamer.AddComment("Extended Op"); - Asm->EmitInt8(0); - - Asm->OutStreamer.AddComment("Op size"); - Asm->EmitInt8(Asm->getDataLayout().getPointerSize() + 1); - Asm->OutStreamer.AddComment("DW_LNE_set_address"); - Asm->EmitInt8(dwarf::DW_LNE_set_address); - - Asm->OutStreamer.AddComment("Section end label"); - - Asm->OutStreamer.EmitSymbolValue( - Asm->GetTempSymbol("section_end", SectionEnd), - Asm->getDataLayout().getPointerSize()); - - // Mark end of matrix. - Asm->OutStreamer.AddComment("DW_LNE_end_sequence"); - Asm->EmitInt8(0); - Asm->EmitInt8(1); - Asm->EmitInt8(1); -} - void DwarfDebug::emitAccel(DwarfAccelTable &Accel, const MCSection *Section, - StringRef TableName, StringRef SymName) { + StringRef TableName) { Accel.FinalizeTable(Asm, TableName); Asm->OutStreamer.SwitchSection(Section); - auto *SectionBegin = Asm->GetTempSymbol(SymName); - Asm->OutStreamer.EmitLabel(SectionBegin); // Emit the full data. - Accel.Emit(Asm, SectionBegin, this, DwarfStrSectionSym); + Accel.emit(Asm, Section->getBeginSymbol(), this); } // Emit visible names into a hashed accelerator table section. void DwarfDebug::emitAccelNames() { emitAccel(AccelNames, Asm->getObjFileLowering().getDwarfAccelNamesSection(), - "Names", "names_begin"); + "Names"); } // Emit objective C classes and categories into a hashed accelerator table // section. void DwarfDebug::emitAccelObjC() { emitAccel(AccelObjC, Asm->getObjFileLowering().getDwarfAccelObjCSection(), - "ObjC", "objc_begin"); + "ObjC"); } // Emit namespace dies into a hashed accelerator table. void DwarfDebug::emitAccelNamespaces() { emitAccel(AccelNamespace, Asm->getObjFileLowering().getDwarfAccelNamespaceSection(), - "namespac", "namespac_begin"); + "namespac"); } // Emit type dies into a hashed accelerator table. void DwarfDebug::emitAccelTypes() { emitAccel(AccelTypes, Asm->getObjFileLowering().getDwarfAccelTypesSection(), - "types", "types_begin"); + "types"); } // Public name handling. @@ -1537,15 +1417,14 @@ void DwarfDebug::emitDebugPubSection( if (auto *Skeleton = TheU->getSkeleton()) TheU = Skeleton; - unsigned ID = TheU->getUniqueID(); // Start the dwarf pubnames section. Asm->OutStreamer.SwitchSection(PSec); // Emit the header. Asm->OutStreamer.AddComment("Length of Public " + Name + " Info"); - MCSymbol *BeginLabel = Asm->GetTempSymbol("pub" + Name + "_begin", ID); - MCSymbol *EndLabel = Asm->GetTempSymbol("pub" + Name + "_end", ID); + MCSymbol *BeginLabel = Asm->createTempSymbol("pub" + Name + "_begin"); + MCSymbol *EndLabel = Asm->createTempSymbol("pub" + Name + "_end"); Asm->EmitLabelDifference(EndLabel, BeginLabel, 4); Asm->OutStreamer.EmitLabel(BeginLabel); @@ -1554,7 +1433,7 @@ void DwarfDebug::emitDebugPubSection( Asm->EmitInt16(dwarf::DW_PUBNAMES_VERSION); Asm->OutStreamer.AddComment("Offset of Compilation Unit Info"); - Asm->EmitSectionOffset(TheU->getLabelBegin(), TheU->getSectionSym()); + Asm->emitSectionOffset(TheU->getLabelBegin()); Asm->OutStreamer.AddComment("Compilation Unit Length"); Asm->EmitInt32(TheU->getLength()); @@ -1600,62 +1479,27 @@ void DwarfDebug::emitDebugStr() { Holder.emitStrings(Asm->getObjFileLowering().getDwarfStrSection()); } -/// Emits an optimal (=sorted) sequence of DW_OP_pieces. -void DwarfDebug::emitLocPieces(ByteStreamer &Streamer, - const DITypeIdentifierMap &Map, - ArrayRef Values) { - assert(std::all_of(Values.begin(), Values.end(), [](DebugLocEntry::Value P) { - return P.isBitPiece(); - }) && "all values are expected to be pieces"); - assert(std::is_sorted(Values.begin(), Values.end()) && - "pieces are expected to be sorted"); - - unsigned Offset = 0; - for (auto Piece : Values) { - DIExpression Expr = Piece.getExpression(); - unsigned PieceOffset = Expr.getBitPieceOffset(); - unsigned PieceSize = Expr.getBitPieceSize(); - assert(Offset <= PieceOffset && "overlapping or duplicate pieces"); - if (Offset < PieceOffset) { - // The DWARF spec seriously mandates pieces with no locations for gaps. - Asm->EmitDwarfOpPiece(Streamer, PieceOffset-Offset); - Offset += PieceOffset-Offset; - } - Offset += PieceSize; - -#ifndef NDEBUG - DIVariable Var = Piece.getVariable(); - unsigned VarSize = Var.getSizeInBits(Map); - assert(PieceSize+PieceOffset <= VarSize - && "piece is larger than or outside of variable"); - assert(PieceSize != VarSize - && "piece covers entire variable"); -#endif - emitDebugLocValue(Streamer, Piece, PieceOffset); - } -} - void DwarfDebug::emitDebugLocEntry(ByteStreamer &Streamer, const DebugLocEntry &Entry) { - const DebugLocEntry::Value Value = Entry.getValues()[0]; - if (Value.isBitPiece()) - // Emit all pieces that belong to the same variable and range. - return emitLocPieces(Streamer, TypeIdentifierMap, Entry.getValues()); - - assert(Entry.getValues().size() == 1 && "only pieces may have >1 value"); - emitDebugLocValue(Streamer, Value); + auto Comment = Entry.getComments().begin(); + auto End = Entry.getComments().end(); + for (uint8_t Byte : Entry.getDWARFBytes()) + Streamer.EmitInt8(Byte, Comment != End ? *(Comment++) : ""); } -void DwarfDebug::emitDebugLocValue(ByteStreamer &Streamer, - const DebugLocEntry::Value &Value, - unsigned PieceOffsetInBits) { +static void emitDebugLocValue(const AsmPrinter &AP, + const DITypeIdentifierMap &TypeIdentifierMap, + ByteStreamer &Streamer, + const DebugLocEntry::Value &Value, + unsigned PieceOffsetInBits) { DIVariable DV = Value.getVariable(); - DebugLocDwarfExpression DwarfExpr(*Asm, Streamer); - + DebugLocDwarfExpression DwarfExpr(*AP.MF->getSubtarget().getRegisterInfo(), + AP.getDwarfDebug()->getDwarfVersion(), + Streamer); // Regular entry. if (Value.isInt()) { - DIBasicType BTy(resolve(DV.getType())); + DIBasicType BTy(DV.getType().resolve(TypeIdentifierMap)); if (BTy.Verify() && (BTy.getEncoding() == dwarf::DW_ATE_signed || BTy.getEncoding() == dwarf::DW_ATE_signed_char)) DwarfExpr.AddSignedConstant(Value.getInt()); @@ -1666,7 +1510,7 @@ void DwarfDebug::emitDebugLocValue(ByteStreamer &Streamer, DIExpression Expr = Value.getExpression(); if (!Expr || (Expr.getNumElements() == 0)) // Regular entry. - Asm->EmitDwarfRegOp(Streamer, Loc); + AP.EmitDwarfRegOp(Streamer, Loc); else { // Complex address entry. if (Loc.getOffset()) { @@ -1682,6 +1526,52 @@ void DwarfDebug::emitDebugLocValue(ByteStreamer &Streamer, // FIXME: ^ } + +void DebugLocEntry::finalize(const AsmPrinter &AP, + const DITypeIdentifierMap &TypeIdentifierMap) { + BufferByteStreamer Streamer(DWARFBytes, Comments); + const DebugLocEntry::Value Value = Values[0]; + if (Value.isBitPiece()) { + // Emit all pieces that belong to the same variable and range. + assert(std::all_of(Values.begin(), Values.end(), [](DebugLocEntry::Value P) { + return P.isBitPiece(); + }) && "all values are expected to be pieces"); + assert(std::is_sorted(Values.begin(), Values.end()) && + "pieces are expected to be sorted"); + + unsigned Offset = 0; + for (auto Piece : Values) { + DIExpression Expr = Piece.getExpression(); + unsigned PieceOffset = Expr.getBitPieceOffset(); + unsigned PieceSize = Expr.getBitPieceSize(); + assert(Offset <= PieceOffset && "overlapping or duplicate pieces"); + if (Offset < PieceOffset) { + // The DWARF spec seriously mandates pieces with no locations for gaps. + DebugLocDwarfExpression Expr(*AP.MF->getSubtarget().getRegisterInfo(), + AP.getDwarfDebug()->getDwarfVersion(), + Streamer); + Expr.AddOpPiece(PieceOffset-Offset, 0); + Offset += PieceOffset-Offset; + } + Offset += PieceSize; + +#ifndef NDEBUG + DIVariable Var = Piece.getVariable(); + unsigned VarSize = Var.getSizeInBits(TypeIdentifierMap); + assert(PieceSize+PieceOffset <= VarSize + && "piece is larger than or outside of variable"); + assert(PieceSize != VarSize + && "piece covers entire variable"); +#endif + emitDebugLocValue(AP, TypeIdentifierMap, Streamer, Piece, PieceOffset); + } + } else { + assert(Values.size() == 1 && "only pieces may have >1 value"); + emitDebugLocValue(AP, TypeIdentifierMap, Streamer, Value, 0); + } +} + + void DwarfDebug::emitDebugLocEntryLocation(const DebugLocEntry &Entry) { Asm->OutStreamer.AddComment("Loc expr size"); MCSymbol *begin = Asm->OutStreamer.getContext().CreateTempSymbol(); @@ -1752,10 +1642,7 @@ struct ArangeSpan { // address we can tie back to a CU. void DwarfDebug::emitDebugARanges() { // Provides a unique id per text section. - DenseMap> SectionMap; - - // Prime section data. - SectionMap[Asm->getObjFileLowering().getTextSection()]; + MapVector> SectionMap; // Filter labels by section. for (const SymbolCU &SCU : ArangeLabels) { @@ -1772,31 +1659,13 @@ void DwarfDebug::emitDebugARanges() { } } - // Build a list of sections used. - std::vector Sections; - for (const auto &it : SectionMap) { - const MCSection *Section = it.first; - Sections.push_back(Section); - } - - // Sort the sections into order. - // This is only done to ensure consistent output order across different runs. - std::sort(Sections.begin(), Sections.end(), SectionSort); - // Add terminating symbols for each section. - for (unsigned ID = 0, E = Sections.size(); ID != E; ID++) { - const MCSection *Section = Sections[ID]; + for (const auto &I : SectionMap) { + const MCSection *Section = I.first; MCSymbol *Sym = nullptr; - if (Section) { - // We can't call MCSection::getLabelEndName, as it's only safe to do so - // if we know the section name up-front. For user-created sections, the - // resulting label may not be valid to use as a label. (section names can - // use a greater set of characters on some systems) - Sym = Asm->GetTempSymbol("debug_end", ID); - Asm->OutStreamer.SwitchSection(Section); - Asm->OutStreamer.EmitLabel(Sym); - } + if (Section) + Sym = Asm->OutStreamer.endSection(Section); // Insert a final terminator. SectionMap[Section].push_back(SymbolCU(nullptr, Sym)); @@ -1804,8 +1673,9 @@ void DwarfDebug::emitDebugARanges() { DenseMap> Spans; - for (const MCSection *Section : Sections) { - SmallVector &List = SectionMap[Section]; + for (auto &I : SectionMap) { + const MCSection *Section = I.first; + SmallVector &List = I.second; if (List.size() < 2) continue; @@ -1902,7 +1772,7 @@ void DwarfDebug::emitDebugARanges() { Asm->OutStreamer.AddComment("DWARF Arange version number"); Asm->EmitInt16(dwarf::DW_ARANGES_VERSION); Asm->OutStreamer.AddComment("Offset Into Debug Info Section"); - Asm->EmitSectionOffset(CU->getLabelBegin(), CU->getSectionSym()); + Asm->emitSectionOffset(CU->getLabelBegin()); Asm->OutStreamer.AddComment("Address Size (in bytes)"); Asm->EmitInt8(PtrSize); Asm->OutStreamer.AddComment("Segment Size (in bytes)"); @@ -1998,10 +1868,9 @@ DwarfCompileUnit &DwarfDebug::constructSkeletonCU(const DwarfCompileUnit &CU) { auto OwnedUnit = make_unique( CU.getUniqueID(), CU.getCUNode(), Asm, this, &SkeletonHolder); DwarfCompileUnit &NewCU = *OwnedUnit; - NewCU.initSection(Asm->getObjFileLowering().getDwarfInfoSection(), - DwarfInfoSectionSym); + NewCU.initSection(Asm->getObjFileLowering().getDwarfInfoSection()); - NewCU.initStmtList(DwarfLineSectionSym); + NewCU.initStmtList(); initSkeletonUnit(CU, NewCU.getUnitDie(), std::move(OwnedUnit)); @@ -2012,9 +1881,8 @@ DwarfCompileUnit &DwarfDebug::constructSkeletonCU(const DwarfCompileUnit &CU) { // compile units that would normally be in debug_info. void DwarfDebug::emitDebugInfoDWO() { assert(useSplitDwarf() && "No split dwarf debug info?"); - // Don't pass an abbrev symbol, using a constant zero instead so as not to - // emit relocations into the dwo file. - InfoHolder.emitUnits(/* AbbrevSymbol */ nullptr); + // Don't emit relocations into the dwo file. + InfoHolder.emitUnits(/* UseOffsets */ true); } // Emit the .debug_abbrev.dwo section for separated dwarf. This contains the @@ -2058,7 +1926,7 @@ static uint64_t makeTypeSignature(StringRef Identifier) { // appropriately. MD5::MD5Result Result; Hash.final(Result); - return *reinterpret_cast(Result + 8); + return support::endian::read64le(Result + 8); } void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU, diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h index 1c0e163..74db3ef 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -88,7 +88,8 @@ public: : Var(V), Expr(1, E), TheDIE(nullptr), DotDebugLocOffset(~0U), MInsn(nullptr), DD(DD) { FrameIndex.push_back(FI); - assert(Var.Verify() && E.Verify()); + assert(Var.Verify()); + assert(!E || E->isValid()); } /// Construct a DbgVariable from a DEBUG_VALUE. @@ -243,25 +244,10 @@ class DwarfDebug : public AsmPrinterHandler { // If nonnull, stores the CU in which the previous subprogram was contained. const DwarfCompileUnit *PrevCU; - // Section Symbols: these are assembler temporary labels that are emitted at - // the beginning of each supported dwarf section. These are used to form - // section offsets and are created by EmitSectionLabels. - MCSymbol *DwarfInfoSectionSym, *DwarfAbbrevSectionSym; - MCSymbol *DwarfStrSectionSym, *TextSectionSym, *DwarfDebugRangeSectionSym; - MCSymbol *DwarfDebugLocSectionSym, *DwarfLineSectionSym, *DwarfAddrSectionSym; - MCSymbol *FunctionBeginSym, *FunctionEndSym; - MCSymbol *DwarfInfoDWOSectionSym, *DwarfAbbrevDWOSectionSym; - MCSymbol *DwarfTypesDWOSectionSym; - MCSymbol *DwarfStrDWOSectionSym; - MCSymbol *DwarfGnuPubNamesSectionSym, *DwarfGnuPubTypesSectionSym; - // As an optimization, there is no need to emit an entry in the directory // table for the same directory as DW_AT_comp_dir. StringRef CompilationDir; - // Counter for assigning globally unique IDs for ranges. - unsigned GlobalRangeCount; - // Holder for the file specific debug information. DwarfFile InfoHolder; @@ -290,6 +276,9 @@ class DwarfDebug : public AsmPrinterHandler { // text. bool UsedNonDefaultText; + // Whether to use the GNU TLS opcode (instead of the standard opcode). + bool UseGNUTLSOpcode; + // Version of dwarf we're emitting. unsigned DwarfVersion; @@ -318,6 +307,7 @@ class DwarfDebug : public AsmPrinterHandler { // True iff there are multiple CUs in this module. bool SingleCU; bool IsDarwin; + bool IsPS4; AddressPool AddrPool; @@ -347,9 +337,6 @@ class DwarfDebug : public AsmPrinterHandler { /// \brief Construct a DIE for this abstract scope. void constructAbstractSubprogramScopeDIE(LexicalScope *Scope); - /// \brief Emit initial Dwarf sections with a label at the start of each one. - void emitSectionLabels(); - /// \brief Compute the size and offset of a DIE given an incoming Offset. unsigned computeSizeAndOffset(DIE *Die, unsigned Offset); @@ -373,13 +360,9 @@ class DwarfDebug : public AsmPrinterHandler { /// \brief Emit the abbreviation section. void emitAbbreviations(); - /// \brief Emit the last address of the section and the end of - /// the line matrix. - void emitEndOfLineMatrix(unsigned SectionEnd); - /// \brief Emit a specified accelerator table. void emitAccel(DwarfAccelTable &Accel, const MCSection *Section, - StringRef TableName, StringRef SymName); + StringRef TableName); /// \brief Emit visible names into a hashed accelerator table section. void emitAccelNames(); @@ -540,8 +523,9 @@ public: SymSize[Sym] = Size; } - /// \brief Recursively Emits a debug information entry. - void emitDIE(DIE &Die); + /// \brief Returns whether to use DW_OP_GNU_push_tls_address, instead of the + /// standard DW_OP_form_tls_address opcode + bool useGNUTLSOpcode() const { return UseGNUTLSOpcode; } // Experimental DWARF5 features. @@ -556,15 +540,6 @@ public: /// Returns the Dwarf Version. unsigned getDwarfVersion() const { return DwarfVersion; } - /// Returns the section symbol for the .debug_loc section. - MCSymbol *getDebugLocSym() const { return DwarfDebugLocSectionSym; } - - /// Returns the section symbol for the .debug_str section. - MCSymbol *getDebugStrSym() const { return DwarfStrSectionSym; } - - /// Returns the section symbol for the .debug_ranges section. - MCSymbol *getRangeSectionSym() const { return DwarfDebugRangeSectionSym; } - /// Returns the previous CU that was being updated const DwarfCompileUnit *getPrevCU() const { return PrevCU; } void setPrevCU(const DwarfCompileUnit *PrevCU) { this->PrevCU = PrevCU; } @@ -577,7 +552,8 @@ public: /// \brief Emit an entry for the debug loc section. This can be used to /// handle an entry that's going to be emitted into the debug loc section. - void emitDebugLocEntry(ByteStreamer &Streamer, const DebugLocEntry &Entry); + void emitDebugLocEntry(ByteStreamer &Streamer, + const DebugLocEntry &Entry); /// \brief emit a single value for the debug loc section. void emitDebugLocValue(ByteStreamer &Streamer, const DebugLocEntry::Value &Value, @@ -621,8 +597,6 @@ public: void addAccelType(StringRef Name, const DIE &Die, char Flags); const MachineFunction *getCurrentFunction() const { return CurFn; } - const MCSymbol *getFunctionBeginSym() const { return FunctionBeginSym; } - const MCSymbol *getFunctionEndSym() const { return FunctionEndSym; } iterator_range findImportedEntitiesForScope(const MDNode *Scope) const { @@ -642,12 +616,6 @@ public: /// \brief Return Label immediately following the instruction. MCSymbol *getLabelAfterInsn(const MachineInstr *MI); - // FIXME: Consider rolling ranges up into DwarfDebug since we use a single - // range_base anyway, so there's no need to keep them as separate per-CU range - // lists. (though one day we might end up with a range.dwo section, in which - // case it'd go to DwarfFile) - unsigned getNextRangeNumber() { return GlobalRangeCount++; } - // FIXME: Sink these functions down into DwarfFile/Dwarf*Unit. SmallPtrSet &getProcessedSPNodes() { diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h index e8867c0..6eaf707 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.h +++ b/lib/CodeGen/AsmPrinter/DwarfException.h @@ -21,17 +21,24 @@ namespace llvm { class MachineFunction; class ARMTargetStreamer; -class DwarfCFIException : public EHStreamer { - /// shouldEmitPersonality - Per-function flag to indicate if .cfi_personality - /// should be emitted. +class DwarfCFIExceptionBase : public EHStreamer { +protected: + DwarfCFIExceptionBase(AsmPrinter *A); + + /// Per-function flag to indicate if frame CFI info should be emitted. + bool shouldEmitCFI; + + void markFunctionEnd() override; +}; + +class DwarfCFIException : public DwarfCFIExceptionBase { + /// Per-function flag to indicate if .cfi_personality should be emitted. bool shouldEmitPersonality; - /// shouldEmitLSDA - Per-function flag to indicate if .cfi_lsda - /// should be emitted. + /// Per-function flag to indicate if .cfi_lsda should be emitted. bool shouldEmitLSDA; - /// shouldEmitMoves - Per-function flag to indicate if frame moves info - /// should be emitted. + /// Per-function flag to indicate if frame moves info should be emitted. bool shouldEmitMoves; AsmPrinter::CFIMoveType moveTypeModule; @@ -43,26 +50,21 @@ public: DwarfCFIException(AsmPrinter *A); virtual ~DwarfCFIException(); - /// endModule - Emit all exception information that should come after the - /// content. + /// Emit all exception information that should come after the content. void endModule() override; - /// beginFunction - Gather pre-function exception information. Assumes being - /// emitted immediately after the function entry point. + /// Gather pre-function exception information. Assumes being emitted + /// immediately after the function entry point. void beginFunction(const MachineFunction *MF) override; - /// endFunction - Gather and emit post-function exception information. + /// Gather and emit post-function exception information. void endFunction(const MachineFunction *) override; }; -class ARMException : public EHStreamer { +class ARMException : public DwarfCFIExceptionBase { void emitTypeInfos(unsigned TTypeEncoding) override; ARMTargetStreamer &getTargetStreamer(); - /// shouldEmitCFI - Per-function flag to indicate if frame CFI info - /// should be emitted. - bool shouldEmitCFI; - public: //===--------------------------------------------------------------------===// // Main entry points. @@ -70,15 +72,14 @@ public: ARMException(AsmPrinter *A); virtual ~ARMException(); - /// endModule - Emit all exception information that should come after the - /// content. + /// Emit all exception information that should come after the content. void endModule() override; - /// beginFunction - Gather pre-function exception information. Assumes being - /// emitted immediately after the function entry point. + /// Gather pre-function exception information. Assumes being emitted + /// immediately after the function entry point. void beginFunction(const MachineFunction *MF) override; - /// endFunction - Gather and emit post-function exception information. + /// Gather and emit post-function exception information. void endFunction(const MachineFunction *) override; }; } // End of namespace llvm diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp index fcab067..489e455 100644 --- a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp @@ -22,14 +22,6 @@ using namespace llvm; -const TargetRegisterInfo *DwarfExpression::getTRI() const { - return AP.TM.getSubtargetImpl()->getRegisterInfo(); -} - -unsigned DwarfExpression::getDwarfVersion() const { - return AP.getDwarfDebug()->getDwarfVersion(); -} - void DwarfExpression::AddReg(int DwarfReg, const char *Comment) { assert(DwarfReg >= 0 && "invalid negative dwarf register number"); if (DwarfReg < 32) { @@ -74,28 +66,28 @@ void DwarfExpression::AddShr(unsigned ShiftBy) { } bool DwarfExpression::AddMachineRegIndirect(unsigned MachineReg, int Offset) { - int DwarfReg = getTRI()->getDwarfRegNum(MachineReg, false); - if (DwarfReg < 0) - return false; - if (isFrameRegister(MachineReg)) { // If variable offset is based in frame register then use fbreg. EmitOp(dwarf::DW_OP_fbreg); EmitSigned(Offset); - } else { - AddRegIndirect(DwarfReg, Offset); + return true; } + + int DwarfReg = TRI.getDwarfRegNum(MachineReg, false); + if (DwarfReg < 0) + return false; + + AddRegIndirect(DwarfReg, Offset); return true; } bool DwarfExpression::AddMachineRegPiece(unsigned MachineReg, unsigned PieceSizeInBits, unsigned PieceOffsetInBits) { - const TargetRegisterInfo *TRI = getTRI(); - if (!TRI->isPhysicalRegister(MachineReg)) + if (!TRI.isPhysicalRegister(MachineReg)) return false; - int Reg = TRI->getDwarfRegNum(MachineReg, false); + int Reg = TRI.getDwarfRegNum(MachineReg, false); // If this is a valid register number, emit it. if (Reg >= 0) { @@ -107,12 +99,12 @@ bool DwarfExpression::AddMachineRegPiece(unsigned MachineReg, // Walk up the super-register chain until we find a valid number. // For example, EAX on x86_64 is a 32-bit piece of RAX with offset 0. - for (MCSuperRegIterator SR(MachineReg, TRI); SR.isValid(); ++SR) { - Reg = TRI->getDwarfRegNum(*SR, false); + for (MCSuperRegIterator SR(MachineReg, &TRI); SR.isValid(); ++SR) { + Reg = TRI.getDwarfRegNum(*SR, false); if (Reg >= 0) { - unsigned Idx = TRI->getSubRegIndex(*SR, MachineReg); - unsigned Size = TRI->getSubRegIdxSize(Idx); - unsigned RegOffset = TRI->getSubRegIdxOffset(Idx); + unsigned Idx = TRI.getSubRegIndex(*SR, MachineReg); + unsigned Size = TRI.getSubRegIdxSize(Idx); + unsigned RegOffset = TRI.getSubRegIdxOffset(Idx); AddReg(Reg, "super-register"); if (PieceOffsetInBits == RegOffset) { AddOpPiece(Size, RegOffset); @@ -136,15 +128,15 @@ bool DwarfExpression::AddMachineRegPiece(unsigned MachineReg, // efficient DW_OP_piece. unsigned CurPos = PieceOffsetInBits; // The size of the register in bits, assuming 8 bits per byte. - unsigned RegSize = TRI->getMinimalPhysRegClass(MachineReg)->getSize() * 8; + unsigned RegSize = TRI.getMinimalPhysRegClass(MachineReg)->getSize() * 8; // Keep track of the bits in the register we already emitted, so we // can avoid emitting redundant aliasing subregs. SmallBitVector Coverage(RegSize, false); - for (MCSubRegIterator SR(MachineReg, TRI); SR.isValid(); ++SR) { - unsigned Idx = TRI->getSubRegIndex(MachineReg, *SR); - unsigned Size = TRI->getSubRegIdxSize(Idx); - unsigned Offset = TRI->getSubRegIdxOffset(Idx); - Reg = TRI->getDwarfRegNum(*SR, false); + for (MCSubRegIterator SR(MachineReg, &TRI); SR.isValid(); ++SR) { + unsigned Idx = TRI.getSubRegIndex(MachineReg, *SR); + unsigned Size = TRI.getSubRegIdxSize(Idx); + unsigned Offset = TRI.getSubRegIdxOffset(Idx); + Reg = TRI.getDwarfRegNum(*SR, false); // Intersection between the bits we already emitted and the bits // covered by this subregister. @@ -180,7 +172,7 @@ void DwarfExpression::AddSignedConstant(int Value) { // value, so the producers and consumers started to rely on heuristics // to disambiguate the value vs. location status of the expression. // See PR21176 for more details. - if (getDwarfVersion() >= 4) + if (DwarfVersion >= 4) EmitOp(dwarf::DW_OP_stack_value); } @@ -188,7 +180,7 @@ void DwarfExpression::AddUnsignedConstant(unsigned Value) { EmitOp(dwarf::DW_OP_constu); EmitUnsigned(Value); // cf. comment in DwarfExpression::AddSignedConstant(). - if (getDwarfVersion() >= 4) + if (DwarfVersion >= 4) EmitOp(dwarf::DW_OP_stack_value); } @@ -204,11 +196,12 @@ bool DwarfExpression::AddMachineRegExpression(DIExpression Expr, unsigned MachineReg, unsigned PieceOffsetInBits) { auto I = Expr.begin(); - // Pattern-match combinations for which more efficient representations exist - // first. - if (I == Expr.end()) + auto E = Expr.end(); + if (I == E) return AddMachineRegPiece(MachineReg); + // Pattern-match combinations for which more efficient representations exist + // first. bool ValidReg = false; switch (*I) { case dwarf::DW_OP_bit_piece: { @@ -218,20 +211,23 @@ bool DwarfExpression::AddMachineRegExpression(DIExpression Expr, return AddMachineRegPiece(MachineReg, SizeInBits, getOffsetOrZero(OffsetInBits, PieceOffsetInBits)); } - case dwarf::DW_OP_plus: + case dwarf::DW_OP_plus: { // [DW_OP_reg,Offset,DW_OP_plus,DW_OP_deref] --> [DW_OP_breg,Offset]. - if (I->getNext() == dwarf::DW_OP_deref) { + auto N = I->getNext(); + if ((N != E) && (*N == dwarf::DW_OP_deref)) { unsigned Offset = I->getArg(1); ValidReg = AddMachineRegIndirect(MachineReg, Offset); std::advance(I, 2); break; } else ValidReg = AddMachineRegPiece(MachineReg); - case dwarf::DW_OP_deref: - // [DW_OP_reg,DW_OP_deref] --> [DW_OP_breg]. - ValidReg = AddMachineRegIndirect(MachineReg); - ++I; - break; + } + case dwarf::DW_OP_deref: { + // [DW_OP_reg,DW_OP_deref] --> [DW_OP_breg]. + ValidReg = AddMachineRegIndirect(MachineReg); + ++I; + break; + } default: llvm_unreachable("unsupported operand"); } @@ -240,7 +236,7 @@ bool DwarfExpression::AddMachineRegExpression(DIExpression Expr, return false; // Emit remaining elements of the expression. - AddExpression(I, Expr.end(), PieceOffsetInBits); + AddExpression(I, E, PieceOffsetInBits); return true; } diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.h b/lib/CodeGen/AsmPrinter/DwarfExpression.h index b90b7b6..985d52c 100644 --- a/lib/CodeGen/AsmPrinter/DwarfExpression.h +++ b/lib/CodeGen/AsmPrinter/DwarfExpression.h @@ -30,21 +30,22 @@ class DIELoc; /// entry. class DwarfExpression { protected: - const AsmPrinter &AP; // Various convenience accessors that extract things out of AsmPrinter. - const TargetRegisterInfo *getTRI() const; - unsigned getDwarfVersion() const; + const TargetRegisterInfo &TRI; + unsigned DwarfVersion; public: - DwarfExpression(const AsmPrinter &AP) : AP(AP) {} + DwarfExpression(const TargetRegisterInfo &TRI, + unsigned DwarfVersion) + : TRI(TRI), DwarfVersion(DwarfVersion) {} virtual ~DwarfExpression() {} /// Output a dwarf operand and an optional assembler comment. virtual void EmitOp(uint8_t Op, const char *Comment = nullptr) = 0; /// Emit a raw signed value. - virtual void EmitSigned(int Value) = 0; + virtual void EmitSigned(int64_t Value) = 0; /// Emit a raw unsigned value. - virtual void EmitUnsigned(unsigned Value) = 0; + virtual void EmitUnsigned(uint64_t Value) = 0; /// Return whether the given machine register is the frame register in the /// current function. virtual bool isFrameRegister(unsigned MachineReg) = 0; @@ -105,27 +106,27 @@ class DebugLocDwarfExpression : public DwarfExpression { ByteStreamer &BS; public: - DebugLocDwarfExpression(const AsmPrinter &AP, ByteStreamer &BS) - : DwarfExpression(AP), BS(BS) {} + DebugLocDwarfExpression(const TargetRegisterInfo &TRI, + unsigned DwarfVersion, ByteStreamer &BS) + : DwarfExpression(TRI, DwarfVersion), BS(BS) {} void EmitOp(uint8_t Op, const char *Comment = nullptr) override; - void EmitSigned(int Value) override; - void EmitUnsigned(unsigned Value) override; + void EmitSigned(int64_t Value) override; + void EmitUnsigned(uint64_t Value) override; bool isFrameRegister(unsigned MachineReg) override; }; /// DwarfExpression implementation for singular DW_AT_location. class DIEDwarfExpression : public DwarfExpression { +const AsmPrinter &AP; DwarfUnit &DU; DIELoc &DIE; public: - DIEDwarfExpression(const AsmPrinter &AP, DwarfUnit &DU, DIELoc &DIE) - : DwarfExpression(AP), DU(DU), DIE(DIE) {} - + DIEDwarfExpression(const AsmPrinter &AP, DwarfUnit &DU, DIELoc &DIE); void EmitOp(uint8_t Op, const char *Comment = nullptr) override; - void EmitSigned(int Value) override; - void EmitUnsigned(unsigned Value) override; + void EmitSigned(int64_t Value) override; + void EmitUnsigned(uint64_t Value) override; bool isFrameRegister(unsigned MachineReg) override; }; } diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.cpp b/lib/CodeGen/AsmPrinter/DwarfFile.cpp index 3988f0d..60acc58e 100644 --- a/lib/CodeGen/AsmPrinter/DwarfFile.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfFile.cpp @@ -17,9 +17,8 @@ #include "llvm/Target/TargetLoweringObjectFile.h" namespace llvm { -DwarfFile::DwarfFile(AsmPrinter *AP, DwarfDebug &DD, StringRef Pref, - BumpPtrAllocator &DA) - : Asm(AP), DD(DD), StrPool(DA, *Asm, Pref) {} +DwarfFile::DwarfFile(AsmPrinter *AP, StringRef Pref, BumpPtrAllocator &DA) + : Asm(AP), StrPool(DA, *Asm, Pref) {} DwarfFile::~DwarfFile() {} @@ -48,15 +47,15 @@ void DwarfFile::addUnit(std::unique_ptr U) { // Emit the various dwarf units to the unit section USection with // the abbreviations going into ASection. -void DwarfFile::emitUnits(const MCSymbol *ASectionSym) { +void DwarfFile::emitUnits(bool UseOffsets) { for (const auto &TheU : CUs) { DIE &Die = TheU->getUnitDie(); const MCSection *USection = TheU->getSection(); Asm->OutStreamer.SwitchSection(USection); - TheU->emitHeader(ASectionSym); + TheU->emitHeader(UseOffsets); - DD.emitDIE(Die); + Asm->emitDwarfDIE(Die); } } @@ -120,23 +119,13 @@ unsigned DwarfFile::computeSizeAndOffset(DIE &Die, unsigned Offset) { Die.setSize(Offset - Die.getOffset()); return Offset; } + void DwarfFile::emitAbbrevs(const MCSection *Section) { // Check to see if it is worth the effort. if (!Abbreviations.empty()) { // Start the debug abbrev section. Asm->OutStreamer.SwitchSection(Section); - - // For each abbrevation. - for (const DIEAbbrev *Abbrev : Abbreviations) { - // Emit the abbrevations code (base 1 index.) - Asm->EmitULEB128(Abbrev->getNumber(), "Abbreviation Code"); - - // Emit the abbreviations data. - Abbrev->Emit(Asm); - } - - // Mark end of abbreviations. - Asm->EmitULEB128(0, "EOM(3)"); + Asm->emitDwarfAbbrevs(Abbreviations); } } diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.h b/lib/CodeGen/AsmPrinter/DwarfFile.h index 35bf33a..c9de666 100644 --- a/lib/CodeGen/AsmPrinter/DwarfFile.h +++ b/lib/CodeGen/AsmPrinter/DwarfFile.h @@ -37,8 +37,6 @@ class DwarfFile { // Target of Dwarf emission, used for sizing of abbreviations. AsmPrinter *Asm; - DwarfDebug ⅅ - // Used to uniquely define abbreviations. FoldingSet AbbreviationsSet; @@ -62,8 +60,7 @@ class DwarfFile { DenseMap MDTypeNodeToDieMap; public: - DwarfFile(AsmPrinter *AP, DwarfDebug &DD, StringRef Pref, - BumpPtrAllocator &DA); + DwarfFile(AsmPrinter *AP, StringRef Pref, BumpPtrAllocator &DA); ~DwarfFile(); @@ -83,7 +80,7 @@ public: /// \brief Emit all of the units to the section listed with the given /// abbreviation section. - void emitUnits(const MCSymbol *ASectionSym); + void emitUnits(bool UseOffsets); /// \brief Emit a set of abbreviations to the specific section. void emitAbbrevs(const MCSection *); diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp index d76b66c..165ef16 100644 --- a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp @@ -19,7 +19,7 @@ getEntry(AsmPrinter &Asm, std::pair &Entry = Pool[Str]; if (!Entry.first) { Entry.second = Pool.size() - 1; - Entry.first = Asm.GetTempSymbol(Prefix, Entry.second); + Entry.first = Asm.createTempSymbol(Prefix); } return Entry; } diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp index b0c7d48..f6af73f 100644 --- a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -17,6 +17,7 @@ #include "DwarfDebug.h" #include "DwarfExpression.h" #include "llvm/ADT/APFloat.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DataLayout.h" @@ -43,17 +44,23 @@ GenerateDwarfTypeUnits("generate-type-units", cl::Hidden, cl::desc("Generate DWARF4 type units."), cl::init(false)); +DIEDwarfExpression::DIEDwarfExpression(const AsmPrinter &AP, DwarfUnit &DU, + DIELoc &DIE) + : DwarfExpression(*AP.MF->getSubtarget().getRegisterInfo(), + AP.getDwarfDebug()->getDwarfVersion()), + AP(AP), DU(DU), DIE(DIE) {} + void DIEDwarfExpression::EmitOp(uint8_t Op, const char* Comment) { DU.addUInt(DIE, dwarf::DW_FORM_data1, Op); } -void DIEDwarfExpression::EmitSigned(int Value) { +void DIEDwarfExpression::EmitSigned(int64_t Value) { DU.addSInt(DIE, dwarf::DW_FORM_sdata, Value); } -void DIEDwarfExpression::EmitUnsigned(unsigned Value) { +void DIEDwarfExpression::EmitUnsigned(uint64_t Value) { DU.addUInt(DIE, dwarf::DW_FORM_udata, Value); } bool DIEDwarfExpression::isFrameRegister(unsigned MachineReg) { - return MachineReg == getTRI()->getFrameRegister(*AP.MF); + return MachineReg == TRI.getFrameRegister(*AP.MF); } @@ -257,12 +264,14 @@ void DwarfUnit::addIndexedString(DIE &Die, dwarf::Attribute Attribute, /// to be in the local string pool instead of indirected. void DwarfUnit::addLocalString(DIE &Die, dwarf::Attribute Attribute, StringRef String) { + const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); MCSymbol *Symb = DU->getStringPool().getSymbol(*Asm, String); DIEValue *Value; if (Asm->MAI->doesDwarfUseRelocationsAcrossSections()) Value = new (DIEValueAllocator) DIELabel(Symb); else - Value = new (DIEValueAllocator) DIEDelta(Symb, DD->getDebugStrSym()); + Value = new (DIEValueAllocator) + DIEDelta(Symb, TLOF.getDwarfStrSection()->getBeginSymbol()); DIEValue *Str = new (DIEValueAllocator) DIEString(Value, String); Die.addValue(Attribute, dwarf::DW_FORM_strp, Str); } @@ -750,6 +759,15 @@ void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, bool Unsigned) { addBlock(Die, dwarf::DW_AT_const_value, Block); } +// Add a linkage name to the DIE. +void DwarfUnit::addLinkageName(DIE &Die, StringRef LinkageName) { + if (!LinkageName.empty()) + addString(Die, + DD->getDwarfVersion() >= 4 ? dwarf::DW_AT_linkage_name + : dwarf::DW_AT_MIPS_linkage_name, + GlobalValue::getRealLinkageName(LinkageName)); +} + /// addTemplateParams - Add template parameters into buffer. void DwarfUnit::addTemplateParams(DIE &Buffer, DIArray TParams) { // Add template parameters. @@ -1269,9 +1287,8 @@ bool DwarfUnit::applySubprogramDefinitionAttributes(DISubprogram SP, assert(((LinkageName.empty() || DeclLinkageName.empty()) || LinkageName == DeclLinkageName) && "decl has a linkage name and it is different"); - if (!LinkageName.empty() && DeclLinkageName.empty()) - addString(SPDie, dwarf::DW_AT_MIPS_linkage_name, - GlobalValue::getRealLinkageName(LinkageName)); + if (DeclLinkageName.empty()) + addLinkageName(SPDie, LinkageName); if (!DeclDie) return false; @@ -1344,9 +1361,8 @@ void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie, if (SP.isOptimized()) addFlag(SPDie, dwarf::DW_AT_APPLE_optimized); - if (unsigned isa = Asm->getISAEncoding(SP.getFunction())) { + if (unsigned isa = Asm->getISAEncoding()) addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa); - } if (SP.isLValueReference()) addFlag(SPDie, dwarf::DW_AT_reference); @@ -1597,7 +1613,7 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT) { return &StaticMemberDIE; } -void DwarfUnit::emitHeader(const MCSymbol *ASectionSym) const { +void DwarfUnit::emitHeader(bool UseOffsets) { // Emit size of content not including length itself Asm->OutStreamer.AddComment("Length of Unit"); Asm->EmitInt32(getHeaderSize() + UnitDie.getSize()); @@ -1605,14 +1621,16 @@ void DwarfUnit::emitHeader(const MCSymbol *ASectionSym) const { Asm->OutStreamer.AddComment("DWARF version number"); Asm->EmitInt16(DD->getDwarfVersion()); Asm->OutStreamer.AddComment("Offset Into Abbrev. Section"); + // We share one abbreviations table across all units so it's always at the // start of the section. Use a relocatable offset where needed to ensure // linking doesn't invalidate that offset. - if (ASectionSym) - Asm->EmitSectionOffset(ASectionSym, ASectionSym); + const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); + if (!UseOffsets) + Asm->emitSectionOffset(TLOF.getDwarfAbbrevSection()->getBeginSymbol()); else - // Use a constant value when no symbol is provided. Asm->EmitInt32(0); + Asm->OutStreamer.AddComment("Address Size (in bytes)"); Asm->EmitInt8(Asm->getDataLayout().getPointerSize()); } @@ -1622,8 +1640,8 @@ void DwarfUnit::initSection(const MCSection *Section) { this->Section = Section; } -void DwarfTypeUnit::emitHeader(const MCSymbol *ASectionSym) const { - DwarfUnit::emitHeader(ASectionSym); +void DwarfTypeUnit::emitHeader(bool UseOffsets) { + DwarfUnit::emitHeader(UseOffsets); Asm->OutStreamer.AddComment("Type Signature"); Asm->OutStreamer.EmitIntValue(TypeSignature, sizeof(TypeSignature)); Asm->OutStreamer.AddComment("Type DIE Offset"); diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.h b/lib/CodeGen/AsmPrinter/DwarfUnit.h index 7a5e47d..81c5821 100644 --- a/lib/CodeGen/AsmPrinter/DwarfUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfUnit.h @@ -120,7 +120,6 @@ protected: DwarfUnit(unsigned UID, dwarf::Tag, DICompileUnit CU, AsmPrinter *A, DwarfDebug *DW, DwarfFile *DWU); - void initSection(const MCSection *Section); /// Add a string attribute data and value. void addLocalString(DIE &Die, dwarf::Attribute Attribute, StringRef Str); @@ -132,6 +131,8 @@ protected: public: virtual ~DwarfUnit(); + void initSection(const MCSection *Section); + const MCSection *getSection() const { assert(Section); return Section; @@ -251,6 +252,9 @@ public: void addConstantFPValue(DIE &Die, const MachineOperand &MO); void addConstantFPValue(DIE &Die, const ConstantFP *CFP); + /// \brief Add a linkage name, if it isn't empty. + void addLinkageName(DIE &Die, StringRef LinkageName); + /// addTemplateParams - Add template parameters in buffer. void addTemplateParams(DIE &Buffer, DIArray TParams); @@ -321,7 +325,7 @@ public: } /// Emit the header for this unit, not including the initial length field. - virtual void emitHeader(const MCSymbol *ASectionSym) const; + virtual void emitHeader(bool UseOffsets); virtual DwarfCompileUnit &getCU() = 0; @@ -423,12 +427,11 @@ public: void setType(const DIE *Ty) { this->Ty = Ty; } /// Emit the header for this unit, not including the initial length field. - void emitHeader(const MCSymbol *ASectionSym) const override; + void emitHeader(bool UseOffsets) override; unsigned getHeaderSize() const override { return DwarfUnit::getHeaderSize() + sizeof(uint64_t) + // Type Signature sizeof(uint32_t); // Type DIE Offset } - using DwarfUnit::initSection; DwarfCompileUnit &getCU() override { return CU; } }; } // end llvm namespace diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/lib/CodeGen/AsmPrinter/EHStreamer.cpp index 4841814..14df4c9 100644 --- a/lib/CodeGen/AsmPrinter/EHStreamer.cpp +++ b/lib/CodeGen/AsmPrinter/EHStreamer.cpp @@ -436,12 +436,7 @@ void EHStreamer::emitExceptionTable() { Asm->OutContext.GetOrCreateSymbol(Twine("GCC_except_table")+ Twine(Asm->getFunctionNumber())); Asm->OutStreamer.EmitLabel(GCCETSym); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("exception", - Asm->getFunctionNumber())); - - if (IsSJLJ) - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("_LSDA_", - Asm->getFunctionNumber())); + Asm->OutStreamer.EmitLabel(Asm->getCurExceptionSym()); // Emit the LSDA header. Asm->EmitEncodingByte(dwarf::DW_EH_PE_omit, "@LPStart"); @@ -552,16 +547,14 @@ void EHStreamer::emitExceptionTable() { I = CallSites.begin(), E = CallSites.end(); I != E; ++I) { const CallSiteEntry &S = *I; - MCSymbol *EHFuncBeginSym = - Asm->GetTempSymbol("eh_func_begin", Asm->getFunctionNumber()); + MCSymbol *EHFuncBeginSym = Asm->getFunctionBegin(); MCSymbol *BeginLabel = S.BeginLabel; if (!BeginLabel) BeginLabel = EHFuncBeginSym; MCSymbol *EndLabel = S.EndLabel; if (!EndLabel) - EndLabel = Asm->GetTempSymbol("eh_func_end", Asm->getFunctionNumber()); - + EndLabel = Asm->getFunctionEnd(); // Offset of the call site relative to the previous call site, counted in // number of 16-byte bundles. The first call site is counted relative to @@ -689,19 +682,3 @@ void EHStreamer::emitTypeInfos(unsigned TTypeEncoding) { Asm->EmitULEB128(TypeID); } } - -/// Emit all exception information that should come after the content. -void EHStreamer::endModule() { - llvm_unreachable("Should be implemented"); -} - -/// Gather pre-function exception information. Assumes it's being emitted -/// immediately after the function entry point. -void EHStreamer::beginFunction(const MachineFunction *MF) { - llvm_unreachable("Should be implemented"); -} - -/// Gather and emit post-function exception information. -void EHStreamer::endFunction(const MachineFunction *) { - llvm_unreachable("Should be implemented"); -} diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.h b/lib/CodeGen/AsmPrinter/EHStreamer.h index 9b316ff..94d0585 100644 --- a/lib/CodeGen/AsmPrinter/EHStreamer.h +++ b/lib/CodeGen/AsmPrinter/EHStreamer.h @@ -125,16 +125,6 @@ public: EHStreamer(AsmPrinter *A); virtual ~EHStreamer(); - /// Emit all exception information that should come after the content. - void endModule() override; - - /// Gather pre-function exception information. Assumes being emitted - /// immediately after the function entry point. - void beginFunction(const MachineFunction *MF) override; - - /// Gather and emit post-function exception information. - void endFunction(const MachineFunction *) override; - // Unused. void setSymbolSize(const MCSymbol *Sym, uint64_t Size) override {} void beginInstruction(const MachineInstr *MI) override {} diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/lib/CodeGen/AsmPrinter/Win64Exception.cpp index 2b03877..7d76ead 100644 --- a/lib/CodeGen/AsmPrinter/Win64Exception.cpp +++ b/lib/CodeGen/AsmPrinter/Win64Exception.cpp @@ -48,8 +48,6 @@ Win64Exception::~Win64Exception() {} void Win64Exception::endModule() { } -/// beginFunction - Gather pre-function exception information. Assumes it's -/// being emitted immediately after the function entry point. void Win64Exception::beginFunction(const MachineFunction *MF) { shouldEmitMoves = shouldEmitPersonality = shouldEmitLSDA = false; @@ -80,9 +78,6 @@ void Win64Exception::beginFunction(const MachineFunction *MF) { const MCSymbol *PersHandlerSym = TLOF.getCFIPersonalitySymbol(Per, *Asm->Mang, Asm->TM, MMI); Asm->OutStreamer.EmitWinEHHandler(PersHandlerSym, true, true); - - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_begin", - Asm->getFunctionNumber())); } /// endFunction - Gather and emit post-function exception information. @@ -91,9 +86,6 @@ void Win64Exception::endFunction(const MachineFunction *) { if (!shouldEmitPersonality && !shouldEmitMoves) return; - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_end", - Asm->getFunctionNumber())); - // Map all labels and get rid of any dead landing pads. MMI->TidyLandingPads(); @@ -170,10 +162,8 @@ void Win64Exception::emitCSpecificHandlerTable() { SmallVector CallSites; computeCallSiteTable(CallSites, LandingPads, FirstActions); - MCSymbol *EHFuncBeginSym = - Asm->GetTempSymbol("eh_func_begin", Asm->getFunctionNumber()); - MCSymbol *EHFuncEndSym = - Asm->GetTempSymbol("eh_func_end", Asm->getFunctionNumber()); + MCSymbol *EHFuncBeginSym = Asm->getFunctionBegin(); + MCSymbol *EHFuncEndSym = Asm->getFunctionEnd(); // Emit the number of table entries. unsigned NumEntries = 0; diff --git a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp index b5e0929..d2b4eec 100644 --- a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp +++ b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp @@ -190,8 +190,11 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) { return; assert(FI.End && "Don't know where the function ends?"); - StringRef FuncName = getDISubprogram(GV).getDisplayName(), - GVName = GV->getName(); + StringRef GVName = GV->getName(); + StringRef FuncName; + if (DISubprogram SP = getDISubprogram(GV)) + FuncName = SP.getDisplayName(); + // FIXME Clang currently sets DisplayName to "bar" for a C++ // "namespace_foo::bar" function, see PR21528. Luckily, dbghelp.dll is trying // to demangle display names anyways, so let's just put a mangled name into @@ -364,10 +367,7 @@ void WinCodeViewLineTables::endFunction(const MachineFunction *MF) { FnDebugInfo.erase(GV); VisitedFunctions.pop_back(); } else { - // Define end label for subprogram. - MCSymbol *FunctionEndSym = Asm->OutStreamer.getContext().CreateTempSymbol(); - Asm->OutStreamer.EmitLabel(FunctionEndSym); - CurFn->End = FunctionEndSym; + CurFn->End = Asm->getFunctionEnd(); } CurFn = nullptr; } diff --git a/lib/CodeGen/AtomicExpandPass.cpp b/lib/CodeGen/AtomicExpandPass.cpp index 4b64be0..fa17108 100644 --- a/lib/CodeGen/AtomicExpandPass.cpp +++ b/lib/CodeGen/AtomicExpandPass.cpp @@ -48,7 +48,7 @@ namespace { bool expandAtomicLoadToLL(LoadInst *LI); bool expandAtomicLoadToCmpXchg(LoadInst *LI); bool expandAtomicStore(StoreInst *SI); - bool expandAtomicRMW(AtomicRMWInst *AI); + bool tryExpandAtomicRMW(AtomicRMWInst *AI); bool expandAtomicRMWToLLSC(AtomicRMWInst *AI); bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI); bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); @@ -135,9 +135,12 @@ bool AtomicExpand::runOnFunction(Function &F) { // - into a load if it is idempotent // - into a Cmpxchg/LL-SC loop otherwise // we try them in that order. - MadeChange |= - (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) || - (TLI->shouldExpandAtomicRMWInIR(RMWI) && expandAtomicRMW(RMWI)); + + if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) { + MadeChange = true; + } else { + MadeChange |= tryExpandAtomicRMW(RMWI); + } } else if (CASI && TLI->hasLoadLinkedStoreConditional()) { MadeChange |= expandAtomicCmpXchg(CASI); } @@ -211,7 +214,7 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) { // atomic if implemented as a native store. So we replace them by an // atomic swap, that can be implemented for example as a ldrex/strex on ARM // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes. - // It is the responsibility of the target to only return true in + // It is the responsibility of the target to only signal expansion via // shouldExpandAtomicRMW in cases where this is required and possible. IRBuilder<> Builder(SI); AtomicRMWInst *AI = @@ -220,14 +223,26 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) { SI->eraseFromParent(); // Now we have an appropriate swap instruction, lower it as usual. - return expandAtomicRMW(AI); + return tryExpandAtomicRMW(AI); } -bool AtomicExpand::expandAtomicRMW(AtomicRMWInst *AI) { - if (TLI->hasLoadLinkedStoreConditional()) +bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { + switch (TLI->shouldExpandAtomicRMWInIR(AI)) { + case TargetLoweringBase::AtomicRMWExpansionKind::None: + return false; + case TargetLoweringBase::AtomicRMWExpansionKind::LLSC: { + assert(TLI->hasLoadLinkedStoreConditional() && + "TargetLowering requested we expand AtomicRMW instruction into " + "load-linked/store-conditional combos, but such instructions aren't " + "supported"); + return expandAtomicRMWToLLSC(AI); - else + } + case TargetLoweringBase::AtomicRMWExpansionKind::CmpXChg: { return expandAtomicRMWToCmpXchg(AI); + } + } + llvm_unreachable("Unhandled case in tryExpandAtomicRMW"); } /// Emit IR to implement the given atomicrmw operation on values in registers, diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index b8f05cd..abe7ca1 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" @@ -727,6 +728,62 @@ bool BranchFolder::CreateCommonTailOnlyBlock(MachineBasicBlock *&PredBB, return true; } +static bool hasIdenticalMMOs(const MachineInstr *MI1, const MachineInstr *MI2) { + auto I1 = MI1->memoperands_begin(), E1 = MI1->memoperands_end(); + auto I2 = MI2->memoperands_begin(), E2 = MI2->memoperands_end(); + if ((E1 - I1) != (E2 - I2)) + return false; + for (; I1 != E1; ++I1, ++I2) { + if (**I1 != **I2) + return false; + } + return true; +} + +static void +removeMMOsFromMemoryOperations(MachineBasicBlock::iterator MBBIStartPos, + MachineBasicBlock &MBBCommon) { + // Remove MMOs from memory operations in the common block + // when they do not match the ones from the block being tail-merged. + // This ensures later passes conservatively compute dependencies. + MachineBasicBlock *MBB = MBBIStartPos->getParent(); + // Note CommonTailLen does not necessarily matches the size of + // the common BB nor all its instructions because of debug + // instructions differences. + unsigned CommonTailLen = 0; + for (auto E = MBB->end(); MBBIStartPos != E; ++MBBIStartPos) + ++CommonTailLen; + + MachineBasicBlock::reverse_iterator MBBI = MBB->rbegin(); + MachineBasicBlock::reverse_iterator MBBIE = MBB->rend(); + MachineBasicBlock::reverse_iterator MBBICommon = MBBCommon.rbegin(); + MachineBasicBlock::reverse_iterator MBBIECommon = MBBCommon.rend(); + + while (CommonTailLen--) { + assert(MBBI != MBBIE && "Reached BB end within common tail length!"); + (void)MBBIE; + + if (MBBI->isDebugValue()) { + ++MBBI; + continue; + } + + while ((MBBICommon != MBBIECommon) && MBBICommon->isDebugValue()) + ++MBBICommon; + + assert(MBBICommon != MBBIECommon && + "Reached BB end within common tail length!"); + assert(MBBICommon->isIdenticalTo(&*MBBI) && "Expected matching MIIs!"); + + if (MBBICommon->mayLoad() || MBBICommon->mayStore()) + if (!hasIdenticalMMOs(&*MBBI, &*MBBICommon)) + MBBICommon->clearMemRefs(); + + ++MBBI; + ++MBBICommon; + } +} + // See if any of the blocks in MergePotentials (which all have a common single // successor, or all have no successor) can be tail-merged. If there is a // successor, any blocks in MergePotentials that are not tail-merged and @@ -761,7 +818,7 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, // Sort by hash value so that blocks with identical end sequences sort // together. - std::stable_sort(MergePotentials.begin(), MergePotentials.end()); + array_pod_sort(MergePotentials.begin(), MergePotentials.end()); // Walk through equivalence sets looking for actual exact matches. while (MergePotentials.size() > 1) { @@ -840,6 +897,8 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, continue; DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber() << (i == e-1 ? "" : ", ")); + // Remove MMOs from memory operations as needed. + removeMMOsFromMemoryOperations(SameTails[i].getTailStartPos(), *MBB); // Hack the end off BB i, making it jump to BB commonTailIndex instead. ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB); // BB i is no longer a predecessor of SuccBB; remove it from the worklist. diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index f21d4d2..ef57638 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -19,7 +19,6 @@ add_llvm_library(LLVMCodeGen ExecutionDepsFix.cpp ExpandISelPseudos.cpp ExpandPostRAPseudos.cpp - ForwardControlFlowIntegrity.cpp GCMetadata.cpp GCMetadataPrinter.cpp GCRootLowering.cpp @@ -29,7 +28,6 @@ add_llvm_library(LLVMCodeGen InlineSpiller.cpp InterferenceCache.cpp IntrinsicLowering.cpp - JumpInstrTables.cpp LLVMTargetMachine.cpp LatencyPriorityQueue.cpp LexicalScopes.cpp diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp index 7c0068e..da66639 100644 --- a/lib/CodeGen/CodeGen.cpp +++ b/lib/CodeGen/CodeGen.cpp @@ -24,9 +24,10 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeBranchFolderPassPass(Registry); initializeCodeGenPreparePass(Registry); initializeDeadMachineInstructionElimPass(Registry); + initializeDwarfEHPreparePass(Registry); initializeEarlyIfConverterPass(Registry); - initializeExpandPostRAPass(Registry); initializeExpandISelPseudosPass(Registry); + initializeExpandPostRAPass(Registry); initializeFinalizeMachineBundlesPass(Registry); initializeGCMachineCodeAnalysisPass(Registry); initializeGCModuleInfoPass(Registry); @@ -36,31 +37,34 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeLiveStacksPass(Registry); initializeLiveVariablesPass(Registry); initializeLocalStackSlotPassPass(Registry); + initializeLowerIntrinsicsPass(Registry); initializeMachineBlockFrequencyInfoPass(Registry); initializeMachineBlockPlacementPass(Registry); initializeMachineBlockPlacementStatsPass(Registry); - initializeMachineCopyPropagationPass(Registry); - initializeMachineCombinerPass(Registry); initializeMachineCSEPass(Registry); + initializeMachineCombinerPass(Registry); + initializeMachineCopyPropagationPass(Registry); initializeMachineDominatorTreePass(Registry); - initializeMachinePostDominatorTreePass(Registry); + initializeMachineFunctionPrinterPassPass(Registry); initializeMachineLICMPass(Registry); initializeMachineLoopInfoPass(Registry); initializeMachineModuleInfoPass(Registry); + initializeMachinePostDominatorTreePass(Registry); initializeMachineSchedulerPass(Registry); initializeMachineSinkingPass(Registry); initializeMachineVerifierPassPass(Registry); initializeOptimizePHIsPass(Registry); + initializePEIPass(Registry); initializePHIEliminationPass(Registry); initializePeepholeOptimizerPass(Registry); initializePostMachineSchedulerPass(Registry); initializePostRASchedulerPass(Registry); initializeProcessImplicitDefsPass(Registry); - initializePEIPass(Registry); initializeRegisterCoalescerPass(Registry); initializeSlotIndexesPass(Registry); - initializeStackProtectorPass(Registry); initializeStackColoringPass(Registry); + initializeStackMapLivenessPass(Registry); + initializeStackProtectorPass(Registry); initializeStackSlotColoringPass(Registry); initializeTailDuplicatePassPass(Registry); initializeTargetPassConfigPass(Registry); @@ -70,9 +74,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeUnreachableMachineBlockElimPass(Registry); initializeVirtRegMapPass(Registry); initializeVirtRegRewriterPass(Registry); - initializeLowerIntrinsicsPass(Registry); - initializeMachineFunctionPrinterPassPass(Registry); - initializeStackMapLivenessPass(Registry); + initializeWinEHPreparePass(Registry); } void LLVMInitializeCodeGen(LLVMPassRegistryRef R) { diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp index c0d7dca..6c9d048 100644 --- a/lib/CodeGen/CodeGenPrepare.cpp +++ b/lib/CodeGen/CodeGenPrepare.cpp @@ -124,7 +124,6 @@ class TypePromotionTransaction; const TargetLowering *TLI; const TargetTransformInfo *TTI; const TargetLibraryInfo *TLInfo; - DominatorTree *DT; /// CurInstIterator - As we scan instructions optimizing them, this is the /// next instruction to optimize. Xforms that can invalidate this should @@ -142,8 +141,7 @@ class TypePromotionTransaction; /// promotion for the current function. InstrToOrigTy PromotedInsts; - /// ModifiedDT - If CFG is modified in anyway, dominator tree may need to - /// be updated. + /// ModifiedDT - If CFG is modified in anyway. bool ModifiedDT; /// OptSize - True if optimizing for size. @@ -186,7 +184,7 @@ class TypePromotionTransaction; bool ExtLdPromotion(TypePromotionTransaction &TPT, LoadInst *&LI, Instruction *&Inst, const SmallVectorImpl &Exts, - unsigned CreatedInst); + unsigned CreatedInstCost); bool splitBranchCondition(Function &F); bool simplifyOffsetableRelocate(Instruction &I); }; @@ -214,9 +212,6 @@ bool CodeGenPrepare::runOnFunction(Function &F) { TLI = TM->getSubtargetImpl(F)->getTargetLowering(); TLInfo = &getAnalysis().getTLI(); TTI = &getAnalysis().getTTI(F); - DominatorTreeWrapperPass *DTWP = - getAnalysisIfAvailable(); - DT = DTWP ? &DTWP->getDomTree() : nullptr; OptSize = F.hasFnAttribute(Attribute::OptimizeForSize); /// This optimization identifies DIV instructions that can be @@ -255,7 +250,6 @@ bool CodeGenPrepare::runOnFunction(Function &F) { MadeChange |= OptimizeBlock(*BB, ModifiedDTOnIteration); // Restart BB iteration if the dominator tree of the Function was changed - ModifiedDT |= ModifiedDTOnIteration; if (ModifiedDTOnIteration) break; } @@ -298,8 +292,6 @@ bool CodeGenPrepare::runOnFunction(Function &F) { if (EverMadeChange || MadeChange) MadeChange |= EliminateFallThrough(F); - if (MadeChange) - ModifiedDT = true; EverMadeChange |= MadeChange; } @@ -313,9 +305,6 @@ bool CodeGenPrepare::runOnFunction(Function &F) { EverMadeChange |= simplifyOffsetableRelocate(*I); } - if (ModifiedDT && DT) - DT->recalculate(F); - return EverMadeChange; } @@ -341,7 +330,7 @@ bool CodeGenPrepare::EliminateFallThrough(Function &F) { // Remember if SinglePred was the entry block of the function. // If so, we will need to move BB back to the entry position. bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); - MergeBasicBlockIntoOnlyPred(BB, DT); + MergeBasicBlockIntoOnlyPred(BB, nullptr); if (isEntry && BB != &BB->getParent()->getEntryBlock()) BB->moveBefore(&BB->getParent()->getEntryBlock()); @@ -481,7 +470,7 @@ void CodeGenPrepare::EliminateMostlyEmptyBlock(BasicBlock *BB) { // Remember if SinglePred was the entry block of the function. If so, we // will need to move BB back to the entry position. bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); - MergeBasicBlockIntoOnlyPred(DestBB, DT); + MergeBasicBlockIntoOnlyPred(DestBB, nullptr); if (isEntry && BB != &BB->getParent()->getEntryBlock()) BB->moveBefore(&BB->getParent()->getEntryBlock()); @@ -523,13 +512,6 @@ void CodeGenPrepare::EliminateMostlyEmptyBlock(BasicBlock *BB) { // The PHIs are now updated, change everything that refers to BB to use // DestBB and remove BB. BB->replaceAllUsesWith(DestBB); - if (DT && !ModifiedDT) { - BasicBlock *BBIDom = DT->getNode(BB)->getIDom()->getBlock(); - BasicBlock *DestBBIDom = DT->getNode(DestBB)->getIDom()->getBlock(); - BasicBlock *NewIDom = DT->findNearestCommonDominator(BBIDom, DestBBIDom); - DT->changeImmediateDominator(DestBB, NewIDom); - DT->eraseNode(BB); - } BB->eraseFromParent(); ++NumBlocksElim; @@ -561,12 +543,15 @@ static void computeBaseDerivedRelocateMap( IntrinsicInst *I = Item.second; auto BaseKey = std::make_pair(Key.first, Key.first); - IntrinsicInst *Base = RelocateIdxMap[BaseKey]; - if (!Base) + + // We're iterating over RelocateIdxMap so we cannot modify it. + auto MaybeBase = RelocateIdxMap.find(BaseKey); + if (MaybeBase == RelocateIdxMap.end()) // TODO: We might want to insert a new base object relocate and gep off // that, if there are enough derived object relocates. continue; - RelocateInstMap[Base].push_back(I); + + RelocateInstMap[MaybeBase->second].push_back(I); } } @@ -615,8 +600,8 @@ simplifyRelocatesOffABase(IntrinsicInst *RelocatedBase, // Create a Builder and replace the target callsite with a gep IRBuilder<> Builder(ToReplace); Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); - Value *Replacement = - Builder.CreateGEP(RelocatedBase, makeArrayRef(OffsetV)); + Value *Replacement = Builder.CreateGEP( + Derived->getSourceElementType(), RelocatedBase, makeArrayRef(OffsetV)); Instruction *ReplacementInst = cast(Replacement); ReplacementInst->removeFromParent(); ReplacementInst->insertAfter(RelocatedBase); @@ -1225,6 +1210,42 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) { return true; } + const DataLayout *TD = TLI ? TLI->getDataLayout() : nullptr; + + // Align the pointer arguments to this call if the target thinks it's a good + // idea + unsigned MinSize, PrefAlign; + if (TLI && TD && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) { + for (auto &Arg : CI->arg_operands()) { + // We want to align both objects whose address is used directly and + // objects whose address is used in casts and GEPs, though it only makes + // sense for GEPs if the offset is a multiple of the desired alignment and + // if size - offset meets the size threshold. + if (!Arg->getType()->isPointerTy()) + continue; + APInt Offset(TD->getPointerSizeInBits( + cast(Arg->getType())->getAddressSpace()), 0); + Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*TD, Offset); + uint64_t Offset2 = Offset.getLimitedValue(); + AllocaInst *AI; + if ((Offset2 & (PrefAlign-1)) == 0 && + (AI = dyn_cast(Val)) && + AI->getAlignment() < PrefAlign && + TD->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2) + AI->setAlignment(PrefAlign); + // TODO: Also align GlobalVariables + } + // If this is a memcpy (or similar) then we may be able to improve the + // alignment + if (MemIntrinsic *MI = dyn_cast(CI)) { + unsigned Align = getKnownAlignment(MI->getDest(), *TD); + if (MemTransferInst *MTI = dyn_cast(MI)) + Align = std::min(Align, getKnownAlignment(MTI->getSource(), *TD)); + if (Align > MI->getAlignment()) + MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align)); + } + } + IntrinsicInst *II = dyn_cast(CI); if (II) { switch (II->getIntrinsicID()) { @@ -1241,8 +1262,7 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) { WeakVH IterHandle(CurInstIterator); replaceAndRecursivelySimplify(CI, RetVal, - TLI ? TLI->getDataLayout() : nullptr, - TLInfo, ModifiedDT ? nullptr : DT); + TLInfo, nullptr); // If the iterator instruction was recursively deleted, start over at the // start of the block. @@ -1284,15 +1304,11 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) { // From here on out we're working with named functions. if (!CI->getCalledFunction()) return false; - // We'll need DataLayout from here on out. - const DataLayout *TD = TLI ? TLI->getDataLayout() : nullptr; - if (!TD) return false; - // Lower all default uses of _chk calls. This is very similar // to what InstCombineCalls does, but here we are only lowering calls // to fortified library functions (e.g. __memcpy_chk) that have the default // "don't know" as the objectsize. Anything else should be left alone. - FortifiedLibCallSimplifier Simplifier(TD, TLInfo, true); + FortifiedLibCallSimplifier Simplifier(TLInfo, true); if (Value *V = Simplifier.optimizeCall(CI)) { CI->replaceAllUsesWith(V); CI->eraseFromParent(); @@ -2025,7 +2041,7 @@ private: ExtAddrMode &AMBefore, ExtAddrMode &AMAfter); bool ValueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2); - bool IsPromotionProfitable(unsigned MatchedSize, unsigned SizeWithPromotion, + bool IsPromotionProfitable(unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const; }; @@ -2159,7 +2175,7 @@ class TypePromotionHelper { /// \brief Utility function to promote the operand of \p Ext when this /// operand is a promotable trunc or sext or zext. /// \p PromotedInsts maps the instructions to their type before promotion. - /// \p CreatedInsts[out] contains how many non-free instructions have been + /// \p CreatedInstsCost[out] contains the cost of all instructions /// created to promote the operand of Ext. /// Newly added extensions are inserted in \p Exts. /// Newly added truncates are inserted in \p Truncs. @@ -2167,53 +2183,55 @@ class TypePromotionHelper { /// \return The promoted value which is used instead of Ext. static Value *promoteOperandForTruncAndAnyExt( Instruction *Ext, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, unsigned &CreatedInsts, + InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, SmallVectorImpl *Exts, - SmallVectorImpl *Truncs); + SmallVectorImpl *Truncs, const TargetLowering &TLI); /// \brief Utility function to promote the operand of \p Ext when this /// operand is promotable and is not a supported trunc or sext. /// \p PromotedInsts maps the instructions to their type before promotion. - /// \p CreatedInsts[out] contains how many non-free instructions have been + /// \p CreatedInstsCost[out] contains the cost of all the instructions /// created to promote the operand of Ext. /// Newly added extensions are inserted in \p Exts. /// Newly added truncates are inserted in \p Truncs. /// Should never be called directly. /// \return The promoted value which is used instead of Ext. - static Value * - promoteOperandForOther(Instruction *Ext, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, unsigned &CreatedInsts, - SmallVectorImpl *Exts, - SmallVectorImpl *Truncs, bool IsSExt); + static Value *promoteOperandForOther(Instruction *Ext, + TypePromotionTransaction &TPT, + InstrToOrigTy &PromotedInsts, + unsigned &CreatedInstsCost, + SmallVectorImpl *Exts, + SmallVectorImpl *Truncs, + const TargetLowering &TLI, bool IsSExt); /// \see promoteOperandForOther. - static Value * - signExtendOperandForOther(Instruction *Ext, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, - unsigned &CreatedInsts, - SmallVectorImpl *Exts, - SmallVectorImpl *Truncs) { - return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInsts, Exts, - Truncs, true); + static Value *signExtendOperandForOther( + Instruction *Ext, TypePromotionTransaction &TPT, + InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, + SmallVectorImpl *Exts, + SmallVectorImpl *Truncs, const TargetLowering &TLI) { + return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost, + Exts, Truncs, TLI, true); } /// \see promoteOperandForOther. - static Value * - zeroExtendOperandForOther(Instruction *Ext, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, - unsigned &CreatedInsts, - SmallVectorImpl *Exts, - SmallVectorImpl *Truncs) { - return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInsts, Exts, - Truncs, false); + static Value *zeroExtendOperandForOther( + Instruction *Ext, TypePromotionTransaction &TPT, + InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, + SmallVectorImpl *Exts, + SmallVectorImpl *Truncs, const TargetLowering &TLI) { + return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost, + Exts, Truncs, TLI, false); } public: /// Type for the utility function that promotes the operand of Ext. typedef Value *(*Action)(Instruction *Ext, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, unsigned &CreatedInsts, + InstrToOrigTy &PromotedInsts, + unsigned &CreatedInstsCost, SmallVectorImpl *Exts, - SmallVectorImpl *Truncs); + SmallVectorImpl *Truncs, + const TargetLowering &TLI); /// \brief Given a sign/zero extend instruction \p Ext, return the approriate /// action to promote the operand of \p Ext instead of using Ext. /// \return NULL if no promotable action is possible with the current @@ -2330,16 +2348,18 @@ TypePromotionHelper::Action TypePromotionHelper::getAction( Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt( llvm::Instruction *SExt, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, unsigned &CreatedInsts, + InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, SmallVectorImpl *Exts, - SmallVectorImpl *Truncs) { + SmallVectorImpl *Truncs, const TargetLowering &TLI) { // By construction, the operand of SExt is an instruction. Otherwise we cannot // get through it and this method should not be called. Instruction *SExtOpnd = cast(SExt->getOperand(0)); Value *ExtVal = SExt; + bool HasMergedNonFreeExt = false; if (isa(SExtOpnd)) { // Replace s|zext(zext(opnd)) // => zext(opnd). + HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd); Value *ZExt = TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType()); TPT.replaceAllUsesWith(SExt, ZExt); @@ -2350,7 +2370,7 @@ Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt( // => z|sext(opnd). TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0)); } - CreatedInsts = 0; + CreatedInstsCost = 0; // Remove dead code. if (SExtOpnd->use_empty()) @@ -2359,8 +2379,11 @@ Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt( // Check if the extension is still needed. Instruction *ExtInst = dyn_cast(ExtVal); if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) { - if (ExtInst && Exts) - Exts->push_back(ExtInst); + if (ExtInst) { + if (Exts) + Exts->push_back(ExtInst); + CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt; + } return ExtVal; } @@ -2373,13 +2396,14 @@ Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt( Value *TypePromotionHelper::promoteOperandForOther( Instruction *Ext, TypePromotionTransaction &TPT, - InstrToOrigTy &PromotedInsts, unsigned &CreatedInsts, + InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, SmallVectorImpl *Exts, - SmallVectorImpl *Truncs, bool IsSExt) { + SmallVectorImpl *Truncs, const TargetLowering &TLI, + bool IsSExt) { // By construction, the operand of Ext is an instruction. Otherwise we cannot // get through it and this method should not be called. Instruction *ExtOpnd = cast(Ext->getOperand(0)); - CreatedInsts = 0; + CreatedInstsCost = 0; if (!ExtOpnd->hasOneUse()) { // ExtOpnd will be promoted. // All its uses, but Ext, will need to use a truncated value of the @@ -2454,7 +2478,6 @@ Value *TypePromotionHelper::promoteOperandForOther( continue; } ExtForOpnd = cast(ValForExtOpnd); - ++CreatedInsts; } if (Exts) Exts->push_back(ExtForOpnd); @@ -2463,6 +2486,7 @@ Value *TypePromotionHelper::promoteOperandForOther( // Move the sign extension before the insertion point. TPT.moveBefore(ExtForOpnd, ExtOpnd); TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd); + CreatedInstsCost += !TLI.isExtFree(ExtForOpnd); // If more sext are required, new instructions will have to be created. ExtForOpnd = nullptr; } @@ -2475,22 +2499,22 @@ Value *TypePromotionHelper::promoteOperandForOther( /// IsPromotionProfitable - Check whether or not promoting an instruction /// to a wider type was profitable. -/// \p MatchedSize gives the number of instructions that have been matched -/// in the addressing mode after the promotion was applied. -/// \p SizeWithPromotion gives the number of created instructions for -/// the promotion plus the number of instructions that have been -/// matched in the addressing mode before the promotion. +/// \p NewCost gives the cost of extension instructions created by the +/// promotion. +/// \p OldCost gives the cost of extension instructions before the promotion +/// plus the number of instructions that have been +/// matched in the addressing mode the promotion. /// \p PromotedOperand is the value that has been promoted. /// \return True if the promotion is profitable, false otherwise. -bool -AddressingModeMatcher::IsPromotionProfitable(unsigned MatchedSize, - unsigned SizeWithPromotion, - Value *PromotedOperand) const { - // We folded less instructions than what we created to promote the operand. +bool AddressingModeMatcher::IsPromotionProfitable( + unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const { + DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost << '\n'); + // The cost of the new extensions is greater than the cost of the + // old extension plus what we folded. // This is not profitable. - if (MatchedSize < SizeWithPromotion) + if (NewCost > OldCost) return false; - if (MatchedSize > SizeWithPromotion) + if (NewCost < OldCost) return true; // The promotion is neutral but it may help folding the sign extension in // loads for instance. @@ -2688,9 +2712,10 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode, TypePromotionTransaction::ConstRestorationPt LastKnownGood = TPT.getRestorationPoint(); - unsigned CreatedInsts = 0; + unsigned CreatedInstsCost = 0; + unsigned ExtCost = !TLI.isExtFree(Ext); Value *PromotedOperand = - TPH(Ext, TPT, PromotedInsts, CreatedInsts, nullptr, nullptr); + TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI); // SExt has been moved away. // Thus either it will be rematched later in the recursive calls or it is // gone. Anyway, we must not fold it into the addressing mode at this point. @@ -2712,7 +2737,12 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode, unsigned OldSize = AddrModeInsts.size(); if (!MatchAddr(PromotedOperand, Depth) || - !IsPromotionProfitable(AddrModeInsts.size(), OldSize + CreatedInsts, + // The total of the new cost is equals to the cost of the created + // instructions. + // The total of the old cost is equals to the cost of the extension plus + // what we have saved in the addressing mode. + !IsPromotionProfitable(CreatedInstsCost, + ExtCost + (AddrModeInsts.size() - OldSize), PromotedOperand)) { AddrMode = BackupAddrMode; AddrModeInsts.resize(OldSize); @@ -3472,7 +3502,7 @@ static bool hasSameExtUse(Instruction *Inst, const TargetLowering &TLI) { bool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT, LoadInst *&LI, Instruction *&Inst, const SmallVectorImpl &Exts, - unsigned CreatedInsts = 0) { + unsigned CreatedInstsCost = 0) { // Iterate over all the extensions to see if one form an ext(load). for (auto I : Exts) { // Check if we directly have ext(load). @@ -3494,10 +3524,11 @@ bool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT, TypePromotionTransaction::ConstRestorationPt LastKnownGood = TPT.getRestorationPoint(); SmallVector NewExts; - unsigned NewCreatedInsts = 0; + unsigned NewCreatedInstsCost = 0; + unsigned ExtCost = !TLI->isExtFree(I); // Promote. - Value *PromotedVal = - TPH(I, TPT, PromotedInsts, NewCreatedInsts, &NewExts, nullptr); + Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost, + &NewExts, nullptr, *TLI); assert(PromotedVal && "TypePromotionHelper should have filtered out those cases"); @@ -3507,9 +3538,10 @@ bool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT, // With exactly 2, the transformation is neutral, because we will merge // one extension but leave one. However, we optimistically keep going, // because the new extension may be removed too. - unsigned TotalCreatedInsts = CreatedInsts + NewCreatedInsts; + long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost; + TotalCreatedInstsCost -= ExtCost; if (!StressExtLdPromotion && - (TotalCreatedInsts > 1 || + (TotalCreatedInstsCost > 1 || !isPromotedInstructionLegal(*TLI, PromotedVal))) { // The promotion is not profitable, rollback to the previous state. TPT.rollback(LastKnownGood); @@ -3517,8 +3549,8 @@ bool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT, } // The promotion is profitable. // Check if it exposes an ext(load). - (void)ExtLdPromotion(TPT, LI, Inst, NewExts, TotalCreatedInsts); - if (LI && (StressExtLdPromotion || NewCreatedInsts == 0 || + (void)ExtLdPromotion(TPT, LI, Inst, NewExts, TotalCreatedInstsCost); + if (LI && (StressExtLdPromotion || NewCreatedInstsCost <= ExtCost || // If we have created a new extension, i.e., now we have two // extensions. We must make sure one of them is merged with // the load, otherwise we may degrade the code quality. @@ -4193,8 +4225,8 @@ bool CodeGenPrepare::OptimizeInst(Instruction *I, bool& ModifiedDT) { // It is possible for very late stage optimizations (such as SimplifyCFG) // to introduce PHI nodes too late to be cleaned up. If we detect such a // trivial PHI, go ahead and zap it here. - if (Value *V = SimplifyInstruction(P, TLI ? TLI->getDataLayout() : nullptr, - TLInfo, DT)) { + const DataLayout &DL = I->getModule()->getDataLayout(); + if (Value *V = SimplifyInstruction(P, DL, TLInfo, nullptr)) { P->replaceAllUsesWith(V); P->eraseFromParent(); ++NumPHIsElim; @@ -4463,8 +4495,7 @@ static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) { /// FIXME: Remove the (equivalent?) implementation in SelectionDAG. /// bool CodeGenPrepare::splitBranchCondition(Function &F) { - if (!TM || TM->Options.EnableFastISel != true || - !TLI || TLI->isJumpExpensive()) + if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive()) return false; bool MadeChange = false; @@ -4625,10 +4656,8 @@ bool CodeGenPrepare::splitBranchCondition(Function &F) { } } - // Request DOM Tree update. // Note: No point in getting fancy here, since the DT info is never - // available to CodeGenPrepare and the existing update code is broken - // anyways. + // available to CodeGenPrepare. ModifiedDT = true; MadeChange = true; diff --git a/lib/CodeGen/DwarfEHPrepare.cpp b/lib/CodeGen/DwarfEHPrepare.cpp index 7b47a48..42656fb 100644 --- a/lib/CodeGen/DwarfEHPrepare.cpp +++ b/lib/CodeGen/DwarfEHPrepare.cpp @@ -13,13 +13,19 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/Analysis/LibCallSemantics.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Transforms/Utils/Local.h" using namespace llvm; #define DEBUG_TYPE "dwarfehprepare" @@ -33,18 +39,28 @@ namespace { // RewindFunction - _Unwind_Resume or the target equivalent. Constant *RewindFunction; + DominatorTree *DT; + const TargetLowering *TLI; + bool InsertUnwindResumeCalls(Function &Fn); Value *GetExceptionObject(ResumeInst *RI); + size_t + pruneUnreachableResumes(Function &Fn, + SmallVectorImpl &Resumes, + SmallVectorImpl &CleanupLPads); public: static char ID; // Pass identification, replacement for typeid. // INITIALIZE_TM_PASS requires a default constructor, but it isn't used in // practice. - DwarfEHPrepare() : FunctionPass(ID), TM(nullptr), RewindFunction(nullptr) {} + DwarfEHPrepare() + : FunctionPass(ID), TM(nullptr), RewindFunction(nullptr), DT(nullptr), + TLI(nullptr) {} DwarfEHPrepare(const TargetMachine *TM) - : FunctionPass(ID), TM(TM), RewindFunction(nullptr) {} + : FunctionPass(ID), TM(TM), RewindFunction(nullptr), DT(nullptr), + TLI(nullptr) {} bool runOnFunction(Function &Fn) override; @@ -53,6 +69,8 @@ namespace { return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override; + const char *getPassName() const override { return "Exception handling preparation"; } @@ -60,13 +78,22 @@ namespace { } // end anonymous namespace char DwarfEHPrepare::ID = 0; -INITIALIZE_TM_PASS(DwarfEHPrepare, "dwarfehprepare", "Prepare DWARF exceptions", - false, false) +INITIALIZE_TM_PASS_BEGIN(DwarfEHPrepare, "dwarfehprepare", + "Prepare DWARF exceptions", false, false) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) +INITIALIZE_TM_PASS_END(DwarfEHPrepare, "dwarfehprepare", + "Prepare DWARF exceptions", false, false) FunctionPass *llvm::createDwarfEHPass(const TargetMachine *TM) { return new DwarfEHPrepare(TM); } +void DwarfEHPrepare::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); + AU.addRequired(); +} + /// GetExceptionObject - Return the exception object from the value passed into /// the 'resume' instruction (typically an aggregate). Clean up any dead /// instructions, including the 'resume' instruction. @@ -107,21 +134,81 @@ Value *DwarfEHPrepare::GetExceptionObject(ResumeInst *RI) { return ExnObj; } +/// Replace resumes that are not reachable from a cleanup landing pad with +/// unreachable and then simplify those blocks. +size_t DwarfEHPrepare::pruneUnreachableResumes( + Function &Fn, SmallVectorImpl &Resumes, + SmallVectorImpl &CleanupLPads) { + BitVector ResumeReachable(Resumes.size()); + size_t ResumeIndex = 0; + for (auto *RI : Resumes) { + for (auto *LP : CleanupLPads) { + if (isPotentiallyReachable(LP, RI, DT)) { + ResumeReachable.set(ResumeIndex); + break; + } + } + ++ResumeIndex; + } + + // If everything is reachable, there is no change. + if (ResumeReachable.all()) + return Resumes.size(); + + const TargetTransformInfo &TTI = + getAnalysis().getTTI(Fn); + LLVMContext &Ctx = Fn.getContext(); + + // Otherwise, insert unreachable instructions and call simplifycfg. + size_t ResumesLeft = 0; + for (size_t I = 0, E = Resumes.size(); I < E; ++I) { + ResumeInst *RI = Resumes[I]; + if (ResumeReachable[I]) { + Resumes[ResumesLeft++] = RI; + } else { + BasicBlock *BB = RI->getParent(); + new UnreachableInst(Ctx, RI); + RI->eraseFromParent(); + SimplifyCFG(BB, TTI, 1); + } + } + Resumes.resize(ResumesLeft); + return ResumesLeft; +} + /// InsertUnwindResumeCalls - Convert the ResumeInsts that are still present /// into calls to the appropriate _Unwind_Resume function. bool DwarfEHPrepare::InsertUnwindResumeCalls(Function &Fn) { SmallVector Resumes; + SmallVector CleanupLPads; + bool FoundLP = false; for (BasicBlock &BB : Fn) { if (auto *RI = dyn_cast(BB.getTerminator())) Resumes.push_back(RI); + if (auto *LP = BB.getLandingPadInst()) { + if (LP->isCleanup()) + CleanupLPads.push_back(LP); + // Check the personality on the first landingpad. Don't do anything if + // it's for MSVC. + if (!FoundLP) { + FoundLP = true; + EHPersonality Pers = classifyEHPersonality(LP->getPersonalityFn()); + if (isMSVCEHPersonality(Pers)) + return false; + } + } } if (Resumes.empty()) return false; - // Find the rewind function if we didn't already. - const TargetLowering *TLI = TM->getSubtargetImpl(Fn)->getTargetLowering(); LLVMContext &Ctx = Fn.getContext(); + + size_t ResumesLeft = pruneUnreachableResumes(Fn, Resumes, CleanupLPads); + if (ResumesLeft == 0) + return true; // We pruned them all. + + // Find the rewind function if we didn't already. if (!RewindFunction) { FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), Type::getInt8PtrTy(Ctx), false); @@ -130,9 +217,7 @@ bool DwarfEHPrepare::InsertUnwindResumeCalls(Function &Fn) { } // Create the basic block where the _Unwind_Resume call will live. - unsigned ResumesSize = Resumes.size(); - - if (ResumesSize == 1) { + if (ResumesLeft == 1) { // Instead of creating a new BB and PHI node, just append the call to // _Unwind_Resume to the end of the single resume block. ResumeInst *RI = Resumes.front(); @@ -149,7 +234,7 @@ bool DwarfEHPrepare::InsertUnwindResumeCalls(Function &Fn) { } BasicBlock *UnwindBB = BasicBlock::Create(Ctx, "unwind_resume", &Fn); - PHINode *PN = PHINode::Create(Type::getInt8PtrTy(Ctx), ResumesSize, + PHINode *PN = PHINode::Create(Type::getInt8PtrTy(Ctx), ResumesLeft, "exn.obj", UnwindBB); // Extract the exception object from the ResumeInst and add it to the PHI node @@ -175,6 +260,10 @@ bool DwarfEHPrepare::InsertUnwindResumeCalls(Function &Fn) { bool DwarfEHPrepare::runOnFunction(Function &Fn) { assert(TM && "DWARF EH preparation requires a target machine"); + DT = &getAnalysis().getDomTree(); + TLI = TM->getSubtargetImpl(Fn)->getTargetLowering(); bool Changed = InsertUnwindResumeCalls(Fn); + DT = nullptr; + TLI = nullptr; return Changed; } diff --git a/lib/CodeGen/ExecutionDepsFix.cpp b/lib/CodeGen/ExecutionDepsFix.cpp index b3a22c8..5b09cf1 100644 --- a/lib/CodeGen/ExecutionDepsFix.cpp +++ b/lib/CodeGen/ExecutionDepsFix.cpp @@ -113,7 +113,7 @@ struct DomainValue { } namespace { -/// LiveReg - Information about a live register. +/// Information about a live register. struct LiveReg { /// Value currently in this register, or NULL when no value is being tracked. /// This counts as a DomainValue reference. @@ -125,7 +125,7 @@ struct LiveReg { /// will be a negative number. int Def; }; -} // anonynous namespace +} // anonymous namespace namespace { class ExeDepsFix : public MachineFunctionPass { @@ -174,7 +174,7 @@ public: private: iterator_range::const_iterator> - regIndizes(unsigned Reg) const; + regIndices(unsigned Reg) const; // DomainValue allocation. DomainValue *alloc(int domain = -1); @@ -205,10 +205,10 @@ private: char ExeDepsFix::ID = 0; -/// Translate TRI register number to a list of indizes into our stmaller tables +/// Translate TRI register number to a list of indices into our smaller tables /// of interesting registers. iterator_range::const_iterator> -ExeDepsFix::regIndizes(unsigned Reg) const { +ExeDepsFix::regIndices(unsigned Reg) const { assert(Reg < AliasMap.size() && "Invalid register"); const auto &Entry = AliasMap[Reg]; return make_range(Entry.begin(), Entry.end()); @@ -225,7 +225,7 @@ DomainValue *ExeDepsFix::alloc(int domain) { return dv; } -/// release - Release a reference to DV. When the last reference is released, +/// Release a reference to DV. When the last reference is released, /// collapse if needed. void ExeDepsFix::release(DomainValue *DV) { while (DV) { @@ -245,8 +245,8 @@ void ExeDepsFix::release(DomainValue *DV) { } } -/// resolve - Follow the chain of dead DomainValues until a live DomainValue is -/// reached. Update the referenced pointer when necessary. +/// Follow the chain of dead DomainValues until a live DomainValue is reached. +/// Update the referenced pointer when necessary. DomainValue *ExeDepsFix::resolve(DomainValue *&DVRef) { DomainValue *DV = DVRef; if (!DV || !DV->Next) @@ -325,8 +325,7 @@ void ExeDepsFix::collapse(DomainValue *dv, unsigned domain) { setLiveReg(rx, alloc(domain)); } -/// Merge - All instructions and registers in B are moved to A, and B is -/// released. +/// All instructions and registers in B are moved to A, and B is released. bool ExeDepsFix::merge(DomainValue *A, DomainValue *B) { assert(!A->isCollapsed() && "Cannot merge into collapsed"); assert(!B->isCollapsed() && "Cannot merge from collapsed"); @@ -352,7 +351,7 @@ bool ExeDepsFix::merge(DomainValue *A, DomainValue *B) { return true; } -// enterBasicBlock - Set up LiveRegs by merging predecessor live-out values. +/// Set up LiveRegs by merging predecessor live-out values. void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) { // Detect back-edges from predecessors we haven't processed yet. SeenUnknownBackEdge = false; @@ -378,7 +377,7 @@ void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) { if (MBB->pred_empty()) { for (MachineBasicBlock::livein_iterator i = MBB->livein_begin(), e = MBB->livein_end(); i != e; ++i) { - for (int rx : regIndizes(*i)) { + for (int rx : regIndices(*i)) { // Treat function live-ins as if they were defined just before the first // instruction. Usually, function arguments are set up immediately // before the call. @@ -475,7 +474,7 @@ void ExeDepsFix::visitInstr(MachineInstr *MI) { bool ExeDepsFix::shouldBreakDependence(MachineInstr *MI, unsigned OpIdx, unsigned Pref) { unsigned reg = MI->getOperand(OpIdx).getReg(); - for (int rx : regIndizes(reg)) { + for (int rx : regIndices(reg)) { unsigned Clearance = CurInstr - LiveRegs[rx].Def; DEBUG(dbgs() << "Clearance: " << Clearance << ", want " << Pref); @@ -521,7 +520,7 @@ void ExeDepsFix::processDefs(MachineInstr *MI, bool Kill) { break; if (MO.isUse()) continue; - for (int rx : regIndizes(MO.getReg())) { + for (int rx : regIndices(MO.getReg())) { // This instruction explicitly defines rx. DEBUG(dbgs() << TRI->getName(RC->getRegister(rx)) << ":\t" << CurInstr << '\t' << *MI); @@ -587,7 +586,7 @@ void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) { e = mi->getDesc().getNumOperands(); i != e; ++i) { MachineOperand &mo = mi->getOperand(i); if (!mo.isReg()) continue; - for (int rx : regIndizes(mo.getReg())) { + for (int rx : regIndices(mo.getReg())) { force(rx, domain); } } @@ -596,7 +595,7 @@ void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) { for (unsigned i = 0, e = mi->getDesc().getNumDefs(); i != e; ++i) { MachineOperand &mo = mi->getOperand(i); if (!mo.isReg()) continue; - for (int rx : regIndizes(mo.getReg())) { + for (int rx : regIndices(mo.getReg())) { kill(rx); force(rx, domain); } @@ -616,7 +615,7 @@ void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) { e = mi->getDesc().getNumOperands(); i != e; ++i) { MachineOperand &mo = mi->getOperand(i); if (!mo.isReg()) continue; - for (int rx : regIndizes(mo.getReg())) { + for (int rx : regIndices(mo.getReg())) { DomainValue *dv = LiveRegs[rx].Value; if (dv == nullptr) continue; @@ -712,7 +711,7 @@ void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) { ii != ee; ++ii) { MachineOperand &mo = *ii; if (!mo.isReg()) continue; - for (int rx : regIndizes(mo.getReg())) { + for (int rx : regIndices(mo.getReg())) { if (!LiveRegs[rx].Value || (mo.isDef() && LiveRegs[rx].Value != dv)) { kill(rx); setLiveReg(rx, dv); diff --git a/lib/CodeGen/ForwardControlFlowIntegrity.cpp b/lib/CodeGen/ForwardControlFlowIntegrity.cpp deleted file mode 100644 index 63c3699..0000000 --- a/lib/CodeGen/ForwardControlFlowIntegrity.cpp +++ /dev/null @@ -1,374 +0,0 @@ -//===-- ForwardControlFlowIntegrity.cpp: Forward-Edge CFI -----------------===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief A pass that instruments code with fast checks for indirect calls and -/// hooks for a function to check violations. -/// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "cfi" - -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/JumpInstrTableInfo.h" -#include "llvm/CodeGen/ForwardControlFlowIntegrity.h" -#include "llvm/CodeGen/JumpInstrTables.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/CallSite.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/GlobalValue.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/InlineAsm.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Pass.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" - -using namespace llvm; - -STATISTIC(NumCFIIndirectCalls, - "Number of indirect call sites rewritten by the CFI pass"); - -char ForwardControlFlowIntegrity::ID = 0; -INITIALIZE_PASS_BEGIN(ForwardControlFlowIntegrity, "forward-cfi", - "Control-Flow Integrity", true, true) -INITIALIZE_PASS_DEPENDENCY(JumpInstrTableInfo); -INITIALIZE_PASS_DEPENDENCY(JumpInstrTables); -INITIALIZE_PASS_END(ForwardControlFlowIntegrity, "forward-cfi", - "Control-Flow Integrity", true, true) - -ModulePass *llvm::createForwardControlFlowIntegrityPass() { - return new ForwardControlFlowIntegrity(); -} - -ModulePass *llvm::createForwardControlFlowIntegrityPass( - JumpTable::JumpTableType JTT, CFIntegrity CFIType, bool CFIEnforcing, - StringRef CFIFuncName) { - return new ForwardControlFlowIntegrity(JTT, CFIType, CFIEnforcing, - CFIFuncName); -} - -// Checks to see if a given CallSite is making an indirect call, including -// cases where the indirect call is made through a bitcast. -static bool isIndirectCall(CallSite &CS) { - if (CS.getCalledFunction()) - return false; - - // Check the value to see if it is merely a bitcast of a function. In - // this case, it will translate to a direct function call in the resulting - // assembly, so we won't treat it as an indirect call here. - const Value *V = CS.getCalledValue(); - if (const ConstantExpr *CE = dyn_cast(V)) { - return !(CE->isCast() && isa(CE->getOperand(0))); - } - - // Otherwise, since we know it's a call, it must be an indirect call - return true; -} - -static const char cfi_failure_func_name[] = "__llvm_cfi_pointer_warning"; - -ForwardControlFlowIntegrity::ForwardControlFlowIntegrity() - : ModulePass(ID), IndirectCalls(), JTType(JumpTable::Single), - CFIType(CFIntegrity::Sub), CFIEnforcing(false), CFIFuncName("") { - initializeForwardControlFlowIntegrityPass(*PassRegistry::getPassRegistry()); -} - -ForwardControlFlowIntegrity::ForwardControlFlowIntegrity( - JumpTable::JumpTableType JTT, CFIntegrity CFIType, bool CFIEnforcing, - std::string CFIFuncName) - : ModulePass(ID), IndirectCalls(), JTType(JTT), CFIType(CFIType), - CFIEnforcing(CFIEnforcing), CFIFuncName(CFIFuncName) { - initializeForwardControlFlowIntegrityPass(*PassRegistry::getPassRegistry()); -} - -ForwardControlFlowIntegrity::~ForwardControlFlowIntegrity() {} - -void ForwardControlFlowIntegrity::getAnalysisUsage(AnalysisUsage &AU) const { - AU.addRequired(); - AU.addRequired(); -} - -void ForwardControlFlowIntegrity::getIndirectCalls(Module &M) { - // To get the indirect calls, we iterate over all functions and iterate over - // the list of basic blocks in each. We extract a total list of indirect calls - // before modifying any of them, since our modifications will modify the list - // of basic blocks. - for (Function &F : M) { - for (BasicBlock &BB : F) { - for (Instruction &I : BB) { - CallSite CS(&I); - if (!(CS && isIndirectCall(CS))) - continue; - - Value *CalledValue = CS.getCalledValue(); - - // Don't rewrite this instruction if the indirect call is actually just - // inline assembly, since our transformation will generate an invalid - // module in that case. - if (isa(CalledValue)) - continue; - - IndirectCalls.push_back(&I); - } - } - } -} - -void ForwardControlFlowIntegrity::updateIndirectCalls(Module &M, - CFITables &CFIT) { - Type *Int64Ty = Type::getInt64Ty(M.getContext()); - for (Instruction *I : IndirectCalls) { - CallSite CS(I); - Value *CalledValue = CS.getCalledValue(); - - // Get the function type for this call and look it up in the tables. - Type *VTy = CalledValue->getType(); - PointerType *PTy = dyn_cast(VTy); - Type *EltTy = PTy->getElementType(); - FunctionType *FunTy = dyn_cast(EltTy); - FunctionType *TransformedTy = JumpInstrTables::transformType(JTType, FunTy); - ++NumCFIIndirectCalls; - Constant *JumpTableStart = nullptr; - Constant *JumpTableMask = nullptr; - Constant *JumpTableSize = nullptr; - - // Some call sites have function types that don't correspond to any - // address-taken function in the module. This happens when function pointers - // are passed in from external code. - auto it = CFIT.find(TransformedTy); - if (it == CFIT.end()) { - // In this case, make sure that the function pointer will change by - // setting the mask and the start to be 0 so that the transformed - // function is 0. - JumpTableStart = ConstantInt::get(Int64Ty, 0); - JumpTableMask = ConstantInt::get(Int64Ty, 0); - JumpTableSize = ConstantInt::get(Int64Ty, 0); - } else { - JumpTableStart = it->second.StartValue; - JumpTableMask = it->second.MaskValue; - JumpTableSize = it->second.Size; - } - - rewriteFunctionPointer(M, I, CalledValue, JumpTableStart, JumpTableMask, - JumpTableSize); - } - - return; -} - -bool ForwardControlFlowIntegrity::runOnModule(Module &M) { - JumpInstrTableInfo *JITI = &getAnalysis(); - Type *Int64Ty = Type::getInt64Ty(M.getContext()); - Type *VoidPtrTy = Type::getInt8PtrTy(M.getContext()); - - // JumpInstrTableInfo stores information about the alignment of each entry. - // The alignment returned by JumpInstrTableInfo is alignment in bytes, not - // in the exponent. - ByteAlignment = JITI->entryByteAlignment(); - LogByteAlignment = llvm::Log2_64(ByteAlignment); - - // Set up tables for control-flow integrity based on information about the - // jump-instruction tables. - CFITables CFIT; - for (const auto &KV : JITI->getTables()) { - uint64_t Size = static_cast(KV.second.size()); - uint64_t TableSize = NextPowerOf2(Size); - - int64_t MaskValue = ((TableSize << LogByteAlignment) - 1) & -ByteAlignment; - Constant *JumpTableMaskValue = ConstantInt::get(Int64Ty, MaskValue); - Constant *JumpTableSize = ConstantInt::get(Int64Ty, Size); - - // The base of the table is defined to be the first jumptable function in - // the table. - Function *First = KV.second.begin()->second; - Constant *JumpTableStartValue = ConstantExpr::getBitCast(First, VoidPtrTy); - CFIT[KV.first].StartValue = JumpTableStartValue; - CFIT[KV.first].MaskValue = JumpTableMaskValue; - CFIT[KV.first].Size = JumpTableSize; - } - - if (CFIT.empty()) - return false; - - getIndirectCalls(M); - - if (!CFIEnforcing) { - addWarningFunction(M); - } - - // Update the instructions with the check and the indirect jump through our - // table. - updateIndirectCalls(M, CFIT); - - return true; -} - -void ForwardControlFlowIntegrity::addWarningFunction(Module &M) { - PointerType *CharPtrTy = Type::getInt8PtrTy(M.getContext()); - - // Get the type of the Warning Function: void (i8*, i8*), - // where the first argument is the name of the function in which the violation - // occurs, and the second is the function pointer that violates CFI. - SmallVector WarningFunArgs; - WarningFunArgs.push_back(CharPtrTy); - WarningFunArgs.push_back(CharPtrTy); - FunctionType *WarningFunTy = - FunctionType::get(Type::getVoidTy(M.getContext()), WarningFunArgs, false); - - if (!CFIFuncName.empty()) { - Constant *FailureFun = M.getOrInsertFunction(CFIFuncName, WarningFunTy); - if (!FailureFun) - report_fatal_error("Could not get or insert the function specified by" - " -cfi-func-name"); - } else { - // The default warning function swallows the warning and lets the call - // continue, since there's no generic way for it to print out this - // information. - Function *WarningFun = M.getFunction(cfi_failure_func_name); - if (!WarningFun) { - WarningFun = - Function::Create(WarningFunTy, GlobalValue::LinkOnceAnyLinkage, - cfi_failure_func_name, &M); - } - - BasicBlock *Entry = - BasicBlock::Create(M.getContext(), "entry", WarningFun, 0); - ReturnInst::Create(M.getContext(), Entry); - } -} - -void ForwardControlFlowIntegrity::rewriteFunctionPointer( - Module &M, Instruction *I, Value *FunPtr, Constant *JumpTableStart, - Constant *JumpTableMask, Constant *JumpTableSize) { - IRBuilder<> TempBuilder(I); - - Type *OrigFunType = FunPtr->getType(); - - BasicBlock *CurBB = cast(I->getParent()); - Function *CurF = cast(CurBB->getParent()); - Type *Int64Ty = Type::getInt64Ty(M.getContext()); - - Value *TI = TempBuilder.CreatePtrToInt(FunPtr, Int64Ty); - Value *TStartInt = TempBuilder.CreatePtrToInt(JumpTableStart, Int64Ty); - - Value *NewFunPtr = nullptr; - Value *Check = nullptr; - switch (CFIType) { - case CFIntegrity::Sub: { - // This is the subtract, mask, and add version. - // Subtract from the base. - Value *Sub = TempBuilder.CreateSub(TI, TStartInt); - - // Mask the difference to force this to be a table offset. - Value *And = TempBuilder.CreateAnd(Sub, JumpTableMask); - - // Add it back to the base. - Value *Result = TempBuilder.CreateAdd(And, TStartInt); - - // Convert it back into a function pointer that we can call. - NewFunPtr = TempBuilder.CreateIntToPtr(Result, OrigFunType); - break; - } - case CFIntegrity::Ror: { - // This is the subtract and rotate version. - // Rotate right by the alignment value. The optimizer should recognize - // this sequence as a rotation. - - // This cast is safe, since unsigned is always a subset of uint64_t. - uint64_t LogByteAlignment64 = static_cast(LogByteAlignment); - Constant *RightShift = ConstantInt::get(Int64Ty, LogByteAlignment64); - Constant *LeftShift = ConstantInt::get(Int64Ty, 64 - LogByteAlignment64); - - // Subtract from the base. - Value *Sub = TempBuilder.CreateSub(TI, TStartInt); - - // Create the equivalent of a rotate-right instruction. - Value *Shr = TempBuilder.CreateLShr(Sub, RightShift); - Value *Shl = TempBuilder.CreateShl(Sub, LeftShift); - Value *Or = TempBuilder.CreateOr(Shr, Shl); - - // Perform unsigned comparison to check for inclusion in the table. - Check = TempBuilder.CreateICmpULT(Or, JumpTableSize); - NewFunPtr = FunPtr; - break; - } - case CFIntegrity::Add: { - // This is the mask and add version. - // Mask the function pointer to turn it into an offset into the table. - Value *And = TempBuilder.CreateAnd(TI, JumpTableMask); - - // Then or this offset to the base and get the pointer value. - Value *Result = TempBuilder.CreateAdd(And, TStartInt); - - // Convert it back into a function pointer that we can call. - NewFunPtr = TempBuilder.CreateIntToPtr(Result, OrigFunType); - break; - } - } - - if (!CFIEnforcing) { - // If a check hasn't been added (in the rotation version), then check to see - // if it's the same as the original function. This check determines whether - // or not we call the CFI failure function. - if (!Check) - Check = TempBuilder.CreateICmpEQ(NewFunPtr, FunPtr); - BasicBlock *InvalidPtrBlock = - BasicBlock::Create(M.getContext(), "invalid.ptr", CurF, 0); - BasicBlock *ContinuationBB = CurBB->splitBasicBlock(I); - - // Remove the unconditional branch that connects the two blocks. - TerminatorInst *TermInst = CurBB->getTerminator(); - TermInst->eraseFromParent(); - - // Add a conditional branch that depends on the Check above. - BranchInst::Create(ContinuationBB, InvalidPtrBlock, Check, CurBB); - - // Call the warning function for this pointer, then continue. - Instruction *BI = BranchInst::Create(ContinuationBB, InvalidPtrBlock); - insertWarning(M, InvalidPtrBlock, BI, FunPtr); - } else { - // Modify the instruction to call this value. - CallSite CS(I); - CS.setCalledFunction(NewFunPtr); - } -} - -void ForwardControlFlowIntegrity::insertWarning(Module &M, BasicBlock *Block, - Instruction *I, Value *FunPtr) { - Function *ParentFun = cast(Block->getParent()); - - // Get the function to call right before the instruction. - Function *WarningFun = nullptr; - if (CFIFuncName.empty()) { - WarningFun = M.getFunction(cfi_failure_func_name); - } else { - WarningFun = M.getFunction(CFIFuncName); - } - - assert(WarningFun && "Could not find the CFI failure function"); - - Type *VoidPtrTy = Type::getInt8PtrTy(M.getContext()); - - IRBuilder<> WarningInserter(I); - // Create a mergeable GlobalVariable containing the name of the function. - Value *ParentNameGV = - WarningInserter.CreateGlobalString(ParentFun->getName()); - Value *ParentNamePtr = WarningInserter.CreateBitCast(ParentNameGV, VoidPtrTy); - Value *FunVoidPtr = WarningInserter.CreateBitCast(FunPtr, VoidPtrTy); - WarningInserter.CreateCall2(WarningFun, ParentNamePtr, FunVoidPtr); -} diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp index 7a29569..b8799a5 100644 --- a/lib/CodeGen/IfConversion.cpp +++ b/lib/CodeGen/IfConversion.cpp @@ -247,7 +247,7 @@ namespace { return true; else if (Incr1 == Incr2) { // Favors subsumption. - if (C1->NeedSubsumption == false && C2->NeedSubsumption == true) + if (!C1->NeedSubsumption && C2->NeedSubsumption) return true; else if (C1->NeedSubsumption == C2->NeedSubsumption) { // Favors diamond over triangle, etc. @@ -726,6 +726,12 @@ bool IfConverter::FeasibilityAnalysis(BBInfo &BBI, if (BBI.IsDone || BBI.IsUnpredicable) return false; + // If it is already predicated but we couldn't analyze its terminator, the + // latter might fallthrough, but we can't determine where to. + // Conservatively avoid if-converting again. + if (BBI.Predicate.size() && !BBI.IsBrAnalyzable) + return false; + // If it is already predicated, check if the new predicate subsumes // its predicate. if (BBI.Predicate.size() && !TII->SubsumesPredicate(Pred, BBI.Predicate)) @@ -1555,7 +1561,7 @@ void IfConverter::PredicateBlock(BBInfo &BBI, UpdatePredRedefs(I, Redefs); } - std::copy(Cond.begin(), Cond.end(), std::back_inserter(BBI.Predicate)); + BBI.Predicate.append(Cond.begin(), Cond.end()); BBI.IsAnalyzed = false; BBI.NonPredSize = 0; @@ -1620,9 +1626,8 @@ void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, } } - std::copy(FromBBI.Predicate.begin(), FromBBI.Predicate.end(), - std::back_inserter(ToBBI.Predicate)); - std::copy(Cond.begin(), Cond.end(), std::back_inserter(ToBBI.Predicate)); + ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end()); + ToBBI.Predicate.append(Cond.begin(), Cond.end()); ToBBI.ClobbersPred |= FromBBI.ClobbersPred; ToBBI.IsAnalyzed = false; @@ -1661,8 +1666,7 @@ void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) { if (NBB && !FromBBI.BB->isSuccessor(NBB)) FromBBI.BB->addSuccessor(NBB); - std::copy(FromBBI.Predicate.begin(), FromBBI.Predicate.end(), - std::back_inserter(ToBBI.Predicate)); + ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end()); FromBBI.Predicate.clear(); ToBBI.NonPredSize += FromBBI.NonPredSize; diff --git a/lib/CodeGen/InterferenceCache.cpp b/lib/CodeGen/InterferenceCache.cpp index 187e015..fd5749b 100644 --- a/lib/CodeGen/InterferenceCache.cpp +++ b/lib/CodeGen/InterferenceCache.cpp @@ -21,7 +21,8 @@ using namespace llvm; #define DEBUG_TYPE "regalloc" // Static member used for null interference cursors. -InterferenceCache::BlockInterference InterferenceCache::Cursor::NoInterference; +const InterferenceCache::BlockInterference + InterferenceCache::Cursor::NoInterference; // Initializes PhysRegEntries (instead of a SmallVector, PhysRegEntries is a // buffer of size NumPhysRegs to speed up alloc/clear for targets with large diff --git a/lib/CodeGen/InterferenceCache.h b/lib/CodeGen/InterferenceCache.h index 1791afb..6519a80 100644 --- a/lib/CodeGen/InterferenceCache.h +++ b/lib/CodeGen/InterferenceCache.h @@ -170,8 +170,8 @@ public: /// Cursor - The primary query interface for the block interference cache. class Cursor { Entry *CacheEntry; - BlockInterference *Current; - static BlockInterference NoInterference; + const BlockInterference *Current; + static const BlockInterference NoInterference; void setEntry(Entry *E) { Current = nullptr; diff --git a/lib/CodeGen/JumpInstrTables.cpp b/lib/CodeGen/JumpInstrTables.cpp deleted file mode 100644 index 75fa261..0000000 --- a/lib/CodeGen/JumpInstrTables.cpp +++ /dev/null @@ -1,296 +0,0 @@ -//===-- JumpInstrTables.cpp: Jump-Instruction Tables ----------------------===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief An implementation of jump-instruction tables. -/// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "jt" - -#include "llvm/CodeGen/JumpInstrTables.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/JumpInstrTableInfo.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/CallSite.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include - -using namespace llvm; - -char JumpInstrTables::ID = 0; - -INITIALIZE_PASS_BEGIN(JumpInstrTables, "jump-instr-tables", - "Jump-Instruction Tables", true, true) -INITIALIZE_PASS_DEPENDENCY(JumpInstrTableInfo); -INITIALIZE_PASS_END(JumpInstrTables, "jump-instr-tables", - "Jump-Instruction Tables", true, true) - -STATISTIC(NumJumpTables, "Number of indirect call tables generated"); -STATISTIC(NumFuncsInJumpTables, "Number of functions in the jump tables"); - -ModulePass *llvm::createJumpInstrTablesPass() { - // The default implementation uses a single table for all functions. - return new JumpInstrTables(JumpTable::Single); -} - -ModulePass *llvm::createJumpInstrTablesPass(JumpTable::JumpTableType JTT) { - return new JumpInstrTables(JTT); -} - -namespace { -static const char jump_func_prefix[] = "__llvm_jump_instr_table_"; -static const char jump_section_prefix[] = ".jump.instr.table.text."; - -// Checks to see if a given CallSite is making an indirect call, including -// cases where the indirect call is made through a bitcast. -bool isIndirectCall(CallSite &CS) { - if (CS.getCalledFunction()) - return false; - - // Check the value to see if it is merely a bitcast of a function. In - // this case, it will translate to a direct function call in the resulting - // assembly, so we won't treat it as an indirect call here. - const Value *V = CS.getCalledValue(); - if (const ConstantExpr *CE = dyn_cast(V)) { - return !(CE->isCast() && isa(CE->getOperand(0))); - } - - // Otherwise, since we know it's a call, it must be an indirect call - return true; -} - -// Replaces Functions and GlobalAliases with a different Value. -bool replaceGlobalValueIndirectUse(GlobalValue *GV, Value *V, Use *U) { - User *Us = U->getUser(); - if (!Us) - return false; - if (Instruction *I = dyn_cast(Us)) { - CallSite CS(I); - - // Don't do the replacement if this use is a direct call to this function. - // If the use is not the called value, then replace it. - if (CS && (isIndirectCall(CS) || CS.isCallee(U))) { - return false; - } - - U->set(V); - } else if (Constant *C = dyn_cast(Us)) { - // Don't replace calls to bitcasts of function symbols, since they get - // translated to direct calls. - if (ConstantExpr *CE = dyn_cast(Us)) { - if (CE->getOpcode() == Instruction::BitCast) { - // This bitcast must have exactly one user. - if (CE->user_begin() != CE->user_end()) { - User *ParentUs = *CE->user_begin(); - if (CallInst *CI = dyn_cast(ParentUs)) { - CallSite CS(CI); - Use &CEU = *CE->use_begin(); - if (CS.isCallee(&CEU)) { - return false; - } - } - } - } - } - - // GlobalAlias doesn't support replaceUsesOfWithOnConstant. And the verifier - // requires alias to point to a defined function. So, GlobalAlias is handled - // as a separate case in runOnModule. - if (!isa(C)) - C->replaceUsesOfWithOnConstant(GV, V, U); - } else { - llvm_unreachable("The Use of a Function symbol is neither an instruction " - "nor a constant"); - } - - return true; -} - -// Replaces all replaceable address-taken uses of GV with a pointer to a -// jump-instruction table entry. -void replaceValueWithFunction(GlobalValue *GV, Function *F) { - // Go through all uses of this function and replace the uses of GV with the - // jump-table version of the function. Get the uses as a vector before - // replacing them, since replacing them changes the use list and invalidates - // the iterator otherwise. - for (Value::use_iterator I = GV->use_begin(), E = GV->use_end(); I != E;) { - Use &U = *I++; - - // Replacement of constants replaces all instances in the constant. So, some - // uses might have already been handled by the time we reach them here. - if (U.get() == GV) - replaceGlobalValueIndirectUse(GV, F, &U); - } - - return; -} -} // end anonymous namespace - -JumpInstrTables::JumpInstrTables() - : ModulePass(ID), Metadata(), JITI(nullptr), TableCount(0), - JTType(JumpTable::Single) { - initializeJumpInstrTablesPass(*PassRegistry::getPassRegistry()); -} - -JumpInstrTables::JumpInstrTables(JumpTable::JumpTableType JTT) - : ModulePass(ID), Metadata(), JITI(nullptr), TableCount(0), JTType(JTT) { - initializeJumpInstrTablesPass(*PassRegistry::getPassRegistry()); -} - -JumpInstrTables::~JumpInstrTables() {} - -void JumpInstrTables::getAnalysisUsage(AnalysisUsage &AU) const { - AU.addRequired(); -} - -Function *JumpInstrTables::insertEntry(Module &M, Function *Target) { - FunctionType *OrigFunTy = Target->getFunctionType(); - FunctionType *FunTy = transformType(JTType, OrigFunTy); - - JumpMap::iterator it = Metadata.find(FunTy); - if (Metadata.end() == it) { - struct TableMeta Meta; - Meta.TableNum = TableCount; - Meta.Count = 0; - Metadata[FunTy] = Meta; - it = Metadata.find(FunTy); - ++NumJumpTables; - ++TableCount; - } - - it->second.Count++; - - std::string NewName(jump_func_prefix); - NewName += (Twine(it->second.TableNum) + "_" + Twine(it->second.Count)).str(); - Function *JumpFun = - Function::Create(OrigFunTy, GlobalValue::ExternalLinkage, NewName, &M); - // The section for this table - JumpFun->setSection((jump_section_prefix + Twine(it->second.TableNum)).str()); - JITI->insertEntry(FunTy, Target, JumpFun); - - ++NumFuncsInJumpTables; - return JumpFun; -} - -bool JumpInstrTables::hasTable(FunctionType *FunTy) { - FunctionType *TransTy = transformType(JTType, FunTy); - return Metadata.end() != Metadata.find(TransTy); -} - -FunctionType *JumpInstrTables::transformType(JumpTable::JumpTableType JTT, - FunctionType *FunTy) { - // Returning nullptr forces all types into the same table, since all types map - // to the same type - Type *VoidPtrTy = Type::getInt8PtrTy(FunTy->getContext()); - - // Ignore the return type. - Type *RetTy = VoidPtrTy; - bool IsVarArg = FunTy->isVarArg(); - std::vector ParamTys(FunTy->getNumParams()); - FunctionType::param_iterator PI, PE; - int i = 0; - - std::vector EmptyParams; - Type *Int32Ty = Type::getInt32Ty(FunTy->getContext()); - FunctionType *VoidFnTy = FunctionType::get( - Type::getVoidTy(FunTy->getContext()), EmptyParams, false); - switch (JTT) { - case JumpTable::Single: - - return FunctionType::get(RetTy, EmptyParams, false); - case JumpTable::Arity: - // Transform all types to void* so that all functions with the same arity - // end up in the same table. - for (PI = FunTy->param_begin(), PE = FunTy->param_end(); PI != PE; - PI++, i++) { - ParamTys[i] = VoidPtrTy; - } - - return FunctionType::get(RetTy, ParamTys, IsVarArg); - case JumpTable::Simplified: - // Project all parameters types to one of 3 types: composite, integer, and - // function, matching the three subclasses of Type. - for (PI = FunTy->param_begin(), PE = FunTy->param_end(); PI != PE; - ++PI, ++i) { - assert((isa(*PI) || isa(*PI) || - isa(*PI)) && - "This type is not an Integer or a Composite or a Function"); - if (isa(*PI)) { - ParamTys[i] = VoidPtrTy; - } else if (isa(*PI)) { - ParamTys[i] = VoidFnTy; - } else if (isa(*PI)) { - ParamTys[i] = Int32Ty; - } - } - - return FunctionType::get(RetTy, ParamTys, IsVarArg); - case JumpTable::Full: - // Don't transform this type at all. - return FunTy; - } - - return nullptr; -} - -bool JumpInstrTables::runOnModule(Module &M) { - JITI = &getAnalysis(); - - // Get the set of jumptable-annotated functions that have their address taken. - DenseMap Functions; - for (Function &F : M) { - if (F.hasFnAttribute(Attribute::JumpTable) && F.hasAddressTaken()) { - assert(F.hasUnnamedAddr() && - "Attribute 'jumptable' requires 'unnamed_addr'"); - Functions[&F] = nullptr; - } - } - - // Create the jump-table functions. - for (auto &KV : Functions) { - Function *F = KV.first; - KV.second = insertEntry(M, F); - } - - // GlobalAlias is a special case, because the target of an alias statement - // must be a defined function. So, instead of replacing a given function in - // the alias, we replace all uses of aliases that target jumptable functions. - // Note that there's no need to create these functions, since only aliases - // that target known jumptable functions are replaced, and there's no way to - // put the jumptable annotation on a global alias. - DenseMap Aliases; - for (GlobalAlias &GA : M.aliases()) { - Constant *Aliasee = GA.getAliasee(); - if (Function *F = dyn_cast(Aliasee)) { - auto it = Functions.find(F); - if (it != Functions.end()) { - Aliases[&GA] = it->second; - } - } - } - - // Replace each address taken function with its jump-instruction table entry. - for (auto &KV : Functions) - replaceValueWithFunction(KV.first, KV.second); - - for (auto &KV : Aliases) - replaceValueWithFunction(KV.first, KV.second); - - return !Functions.empty(); -} diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 9c23368..0fb0c46 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -12,12 +12,9 @@ //===----------------------------------------------------------------------===// #include "llvm/Target/TargetMachine.h" -#include "llvm/Analysis/JumpInstrTableInfo.h" #include "llvm/Analysis/Passes.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/BasicTTIImpl.h" -#include "llvm/CodeGen/ForwardControlFlowIntegrity.h" -#include "llvm/CodeGen/JumpInstrTables.h" #include "llvm/CodeGen/MachineFunctionAnalysis.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/Passes.h" @@ -33,12 +30,8 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/TargetRegistry.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Transforms/Scalar.h" using namespace llvm; @@ -50,8 +43,16 @@ EnableFastISelOption("fast-isel", cl::Hidden, cl::desc("Enable the \"fast\" instruction selector")); void LLVMTargetMachine::initAsmInfo() { - MCAsmInfo *TmpAsmInfo = TheTarget.createMCAsmInfo( - *getSubtargetImpl()->getRegisterInfo(), getTargetTriple()); + MRI = TheTarget.createMCRegInfo(getTargetTriple()); + MII = TheTarget.createMCInstrInfo(); + // FIXME: Having an MCSubtargetInfo on the target machine is a hack due + // to some backends having subtarget feature dependent module level + // code generation. This is similar to the hack in the AsmPrinter for + // module level assembly etc. + STI = TheTarget.createMCSubtargetInfo(getTargetTriple(), getTargetCPU(), + getTargetFeatureString()); + + MCAsmInfo *TmpAsmInfo = TheTarget.createMCAsmInfo(*MRI, getTargetTriple()); // TargetSelect.h moved to a different directory between LLVM 2.9 and 3.0, // and if the old one gets included then MCAsmInfo will be NULL and // we'll crash later. @@ -69,12 +70,13 @@ void LLVMTargetMachine::initAsmInfo() { AsmInfo = TmpAsmInfo; } -LLVMTargetMachine::LLVMTargetMachine(const Target &T, StringRef Triple, - StringRef CPU, StringRef FS, - TargetOptions Options, +LLVMTargetMachine::LLVMTargetMachine(const Target &T, + StringRef DataLayoutString, + StringRef Triple, StringRef CPU, + StringRef FS, TargetOptions Options, Reloc::Model RM, CodeModel::Model CM, CodeGenOpt::Level OL) - : TargetMachine(T, Triple, CPU, FS, Options) { + : TargetMachine(T, DataLayoutString, Triple, CPU, FS, Options) { CodeGenInfo = T.createMCCodeGenInfo(Triple, RM, CM, OL); } @@ -115,8 +117,7 @@ static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM, // Install a MachineModuleInfo class, which is an immutable pass that holds // all the per-module stuff we're generating, including MCContext. MachineModuleInfo *MMI = new MachineModuleInfo( - *TM->getMCAsmInfo(), *TM->getSubtargetImpl()->getRegisterInfo(), - TM->getObjFileLowering()); + *TM->getMCAsmInfo(), *TM->getMCRegisterInfo(), TM->getObjFileLowering()); PM.add(MMI); // Set up a MachineFunction for the rest of CodeGen to work on. @@ -145,16 +146,6 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, bool DisableVerify, AnalysisID StartAfter, AnalysisID StopAfter) { - // Passes to handle jumptable function annotations. These can't be handled at - // JIT time, so we don't add them directly to addPassesToGenerateCode. - PM.add(createJumpInstrTableInfoPass( - getSubtargetImpl()->getInstrInfo()->getJumpInstrTableEntryBound())); - PM.add(createJumpInstrTablesPass(Options.JTType)); - if (Options.FCFI) - PM.add(createForwardControlFlowIntegrityPass( - Options.JTType, Options.CFIType, Options.CFIEnforcing, - Options.getCFIFuncName())); - // Add common CodeGen passes. MCContext *Context = addPassesToGenerateCode(this, PM, DisableVerify, StartAfter, StopAfter); @@ -174,22 +165,22 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, if (Options.MCOptions.MCSaveTempLabels) Context->setAllowTemporaryLabels(false); - const MCSubtargetInfo &STI = getSubtarget(); + const MCSubtargetInfo &STI = *getMCSubtargetInfo(); const MCAsmInfo &MAI = *getMCAsmInfo(); - const MCRegisterInfo &MRI = *getSubtargetImpl()->getRegisterInfo(); - const MCInstrInfo &MII = *getSubtargetImpl()->getInstrInfo(); + const MCRegisterInfo &MRI = *getMCRegisterInfo(); + const MCInstrInfo &MII = *getMCInstrInfo(); + std::unique_ptr AsmStreamer; switch (FileType) { case CGFT_AssemblyFile: { - MCInstPrinter *InstPrinter = - getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI, - MII, MRI, STI); + MCInstPrinter *InstPrinter = getTarget().createMCInstPrinter( + MAI.getAssemblerDialect(), MAI, MII, MRI, STI); // Create a code emitter if asked to show the encoding. MCCodeEmitter *MCE = nullptr; if (Options.MCOptions.ShowMCEncoding) - MCE = getTarget().createMCCodeEmitter(MII, MRI, STI, *Context); + MCE = getTarget().createMCCodeEmitter(MII, MRI, *Context); MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(), TargetCPU); @@ -203,17 +194,16 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, case CGFT_ObjectFile: { // Create the code emitter for the target if it exists. If not, .o file // emission fails. - MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(MII, MRI, STI, - *Context); + MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(MII, MRI, *Context); MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(), TargetCPU); if (!MCE || !MAB) return true; - AsmStreamer.reset( - getTarget() - .createMCObjectStreamer(getTargetTriple(), *Context, *MAB, Out, MCE, - STI, Options.MCOptions.MCRelaxAll)); + Triple T(getTargetTriple()); + AsmStreamer.reset(getTarget().createMCObjectStreamer( + T, *Context, *MAB, Out, MCE, STI, Options.MCOptions.MCRelaxAll, + /*DWARFMustBeAtTheEnd*/ true)); break; } case CGFT_Null: @@ -253,18 +243,19 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, // Create the code emitter for the target if it exists. If not, .o file // emission fails. - const MCRegisterInfo &MRI = *getSubtargetImpl()->getRegisterInfo(); - const MCSubtargetInfo &STI = getSubtarget(); - MCCodeEmitter *MCE = getTarget().createMCCodeEmitter( - *getSubtargetImpl()->getInstrInfo(), MRI, STI, *Ctx); + const MCRegisterInfo &MRI = *getMCRegisterInfo(); + MCCodeEmitter *MCE = + getTarget().createMCCodeEmitter(*getMCInstrInfo(), MRI, *Ctx); MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(), TargetCPU); if (!MCE || !MAB) return true; + Triple T(getTargetTriple()); + const MCSubtargetInfo &STI = *getMCSubtargetInfo(); std::unique_ptr AsmStreamer(getTarget().createMCObjectStreamer( - getTargetTriple(), *Ctx, *MAB, Out, MCE, STI, - Options.MCOptions.MCRelaxAll)); + T, *Ctx, *MAB, Out, MCE, STI, Options.MCOptions.MCRelaxAll, + /*DWARFMustBeAtTheEnd*/ true)); // Create the AsmPrinter, which takes ownership of AsmStreamer if successful. FunctionPass *Printer = diff --git a/lib/CodeGen/LatencyPriorityQueue.cpp b/lib/CodeGen/LatencyPriorityQueue.cpp index cdf505e..4321849 100644 --- a/lib/CodeGen/LatencyPriorityQueue.cpp +++ b/lib/CodeGen/LatencyPriorityQueue.cpp @@ -138,16 +138,3 @@ void LatencyPriorityQueue::remove(SUnit *SU) { std::swap(*I, Queue.back()); Queue.pop_back(); } - -#ifdef NDEBUG -void LatencyPriorityQueue::dump(ScheduleDAG *DAG) const {} -#else -void LatencyPriorityQueue::dump(ScheduleDAG *DAG) const { - LatencyPriorityQueue q = *this; - while (!q.empty()) { - SUnit *su = q.pop(); - dbgs() << "Height " << su->getHeight() << ": "; - su->dump(DAG); - } -} -#endif diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp index dc936a3..e3791be 100644 --- a/lib/CodeGen/LiveDebugVariables.cpp +++ b/lib/CodeGen/LiveDebugVariables.cpp @@ -36,6 +36,7 @@ #include "llvm/IR/Value.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -276,7 +277,7 @@ public: /// getDebugLoc - Return DebugLoc of this UserValue. DebugLoc getDebugLoc() { return dl;} - void print(raw_ostream&, const TargetMachine*); + void print(raw_ostream &, const TargetRegisterInfo *); }; } // namespace @@ -362,7 +363,7 @@ public: }; } // namespace -void UserValue::print(raw_ostream &OS, const TargetMachine *TM) { +void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) { DIVariable DV(Variable); OS << "!\""; DV.printExtendedName(OS); @@ -378,7 +379,7 @@ void UserValue::print(raw_ostream &OS, const TargetMachine *TM) { } for (unsigned i = 0, e = locations.size(); i != e; ++i) { OS << " Loc" << i << '='; - locations[i].print(OS, TM); + locations[i].print(OS, TRI); } OS << '\n'; } @@ -386,7 +387,7 @@ void UserValue::print(raw_ostream &OS, const TargetMachine *TM) { void LDVImpl::print(raw_ostream &OS) { OS << "********** DEBUG VARIABLES **********\n"; for (unsigned i = 0, e = userValues.size(); i != e; ++i) - userValues[i]->print(OS, &MF->getTarget()); + userValues[i]->print(OS, TRI); } void UserValue::coalesceLocation(unsigned LocNo) { @@ -1004,7 +1005,7 @@ void LDVImpl::emitDebugValues(VirtRegMap *VRM) { return; const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); for (unsigned i = 0, e = userValues.size(); i != e; ++i) { - DEBUG(userValues[i]->print(dbgs(), &MF->getTarget())); + DEBUG(userValues[i]->print(dbgs(), TRI)); userValues[i]->rewriteLocations(*VRM, *TRI); userValues[i]->emitDebugValues(VRM, *LIS, *TII); } diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp index fd7516d..2afd7fa 100644 --- a/lib/CodeGen/LiveInterval.cpp +++ b/lib/CodeGen/LiveInterval.cpp @@ -32,6 +32,7 @@ #include using namespace llvm; +namespace { //===----------------------------------------------------------------------===// // Implementation of various methods necessary for calculation of live ranges. // The implementation of the methods abstracts from the concrete type of the @@ -293,6 +294,7 @@ private: return I; } }; +} // namespace //===----------------------------------------------------------------------===// // LiveRange methods @@ -567,13 +569,9 @@ void LiveRange::removeSegment(SlotIndex Start, SlotIndex End, /// Also remove the value# from value# list. void LiveRange::removeValNo(VNInfo *ValNo) { if (empty()) return; - iterator I = end(); - iterator E = begin(); - do { - --I; - if (I->valno == ValNo) - segments.erase(I); - } while (I != E); + segments.erase(std::remove_if(begin(), end(), [ValNo](const Segment &S) { + return S.valno == ValNo; + }), end()); // Now that ValNo is dead, remove it. markValNoForDeletion(ValNo); } @@ -747,7 +745,6 @@ void LiveRange::flushSegmentSet() { segments.empty() && "segment set can be used only initially before switching to the array"); segments.append(segmentSet->begin(), segmentSet->end()); - delete segmentSet; segmentSet = nullptr; verify(); } diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp index cc08045..adca4cc 100644 --- a/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -199,7 +199,7 @@ void LiveIntervals::computeVirtRegInterval(LiveInterval &LI) { assert(LRCalc && "LRCalc not initialized."); assert(LI.empty() && "Should only compute empty intervals."); LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator()); - LRCalc->calculate(LI); + LRCalc->calculate(LI, MRI->shouldTrackSubRegLiveness(LI.reg)); computeDeadValues(LI, nullptr); } @@ -466,7 +466,7 @@ bool LiveIntervals::computeDeadValues(LiveInterval &LI, // Is the register live before? Otherwise we may have to add a read-undef // flag for subregister defs. - if (MRI->tracksSubRegLiveness()) { + if (MRI->shouldTrackSubRegLiveness(LI.reg)) { if ((I == LI.begin() || std::prev(I)->end < Def) && !VNI->isPHIDef()) { MachineInstr *MI = getInstructionFromIndex(Def); MI->addRegisterDefReadUndef(LI.reg); @@ -662,7 +662,7 @@ void LiveIntervals::addKillFlags(const VirtRegMap *VRM) { RU.push_back(std::make_pair(&RURange, RURange.find(LI.begin()->end))); } - if (MRI->tracksSubRegLiveness()) { + if (MRI->subRegLivenessEnabled()) { SRs.clear(); for (const LiveInterval::SubRange &SR : LI.subranges()) { SRs.push_back(std::make_pair(&SR, SR.find(LI.begin()->end))); @@ -700,7 +700,7 @@ void LiveIntervals::addKillFlags(const VirtRegMap *VRM) { goto CancelKill; } - if (MRI->tracksSubRegLiveness()) { + if (MRI->subRegLivenessEnabled()) { // When reading a partial undefined value we must not add a kill flag. // The regalloc might have used the undef lane for something else. // Example: diff --git a/lib/CodeGen/LivePhysRegs.cpp b/lib/CodeGen/LivePhysRegs.cpp index 7efd941..89567ef 100644 --- a/lib/CodeGen/LivePhysRegs.cpp +++ b/lib/CodeGen/LivePhysRegs.cpp @@ -16,6 +16,7 @@ #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp index d804b39..45e7265 100644 --- a/lib/CodeGen/LiveRangeCalc.cpp +++ b/lib/CodeGen/LiveRangeCalc.cpp @@ -50,7 +50,7 @@ static void createDeadDef(SlotIndexes &Indexes, VNInfo::Allocator &Alloc, LR.createDeadDef(DefIdx, Alloc); } -void LiveRangeCalc::calculate(LiveInterval &LI) { +void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) { assert(MRI && Indexes && "call reset() first"); // Step 1: Create minimal live segments for every definition of Reg. @@ -63,7 +63,7 @@ void LiveRangeCalc::calculate(LiveInterval &LI) { continue; unsigned SubReg = MO.getSubReg(); - if (LI.hasSubRanges() || (SubReg != 0 && MRI->tracksSubRegLiveness())) { + if (LI.hasSubRanges() || (SubReg != 0 && TrackSubRegs)) { unsigned Mask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg) : MRI->getMaxLaneMaskForVReg(Reg); diff --git a/lib/CodeGen/LiveRangeCalc.h b/lib/CodeGen/LiveRangeCalc.h index 90bf971..34d9953 100644 --- a/lib/CodeGen/LiveRangeCalc.h +++ b/lib/CodeGen/LiveRangeCalc.h @@ -187,7 +187,7 @@ public: /// Calculates liveness for the register specified in live interval @p LI. /// Creates subregister live ranges as needed if subreg liveness tracking is /// enabled. - void calculate(LiveInterval &LI); + void calculate(LiveInterval &LI, bool TrackSubRegs); //===--------------------------------------------------------------------===// // Low-level interface. diff --git a/lib/CodeGen/LiveStackAnalysis.cpp b/lib/CodeGen/LiveStackAnalysis.cpp index 8a6ac25..5c9c679 100644 --- a/lib/CodeGen/LiveStackAnalysis.cpp +++ b/lib/CodeGen/LiveStackAnalysis.cpp @@ -61,8 +61,10 @@ LiveStacks::getOrCreateInterval(int Slot, const TargetRegisterClass *RC) { assert(Slot >= 0 && "Spill slot indice must be >= 0"); SS2IntervalMap::iterator I = S2IMap.find(Slot); if (I == S2IMap.end()) { - I = S2IMap.insert(I, std::make_pair(Slot, - LiveInterval(TargetRegisterInfo::index2StackSlot(Slot), 0.0F))); + I = S2IMap.emplace(std::piecewise_construct, std::forward_as_tuple(Slot), + std::forward_as_tuple( + TargetRegisterInfo::index2StackSlot(Slot), 0.0F)) + .first; S2RCMap.insert(std::make_pair(Slot, RC)); } else { // Use the largest common subclass register class. diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp index c4bca5f..11deb81 100644 --- a/lib/CodeGen/LiveVariables.cpp +++ b/lib/CodeGen/LiveVariables.cpp @@ -36,6 +36,7 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include using namespace llvm; diff --git a/lib/CodeGen/LocalStackSlotAllocation.cpp b/lib/CodeGen/LocalStackSlotAllocation.cpp index e8bf687..8378429 100644 --- a/lib/CodeGen/LocalStackSlotAllocation.cpp +++ b/lib/CodeGen/LocalStackSlotAllocation.cpp @@ -252,7 +252,8 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) { } static inline bool -lookupCandidateBaseReg(int64_t BaseOffset, +lookupCandidateBaseReg(unsigned BaseReg, + int64_t BaseOffset, int64_t FrameSizeAdjust, int64_t LocalFrameOffset, const MachineInstr *MI, @@ -260,7 +261,7 @@ lookupCandidateBaseReg(int64_t BaseOffset, // Check if the relative offset from the where the base register references // to the target address is in range for the instruction. int64_t Offset = FrameSizeAdjust + LocalFrameOffset - BaseOffset; - return TRI->isFrameOffsetLegal(MI, Offset); + return TRI->isFrameOffsetLegal(MI, BaseReg, Offset); } bool LocalStackSlotPass::insertFrameReferenceRegisters(MachineFunction &Fn) { @@ -362,8 +363,9 @@ bool LocalStackSlotPass::insertFrameReferenceRegisters(MachineFunction &Fn) { // instruction itself will be taken into account by the target, // so we don't have to adjust for it here when reusing a base // register. - if (UsedBaseReg && lookupCandidateBaseReg(BaseOffset, FrameSizeAdjust, - LocalOffset, MI, TRI)) { + if (UsedBaseReg && lookupCandidateBaseReg(BaseReg, BaseOffset, + FrameSizeAdjust, LocalOffset, MI, + TRI)) { DEBUG(dbgs() << " Reusing base register " << BaseReg << "\n"); // We found a register to reuse. Offset = FrameSizeAdjust + LocalOffset - BaseOffset; @@ -382,7 +384,7 @@ bool LocalStackSlotPass::insertFrameReferenceRegisters(MachineFunction &Fn) { // then don't bother creating it. if (ref + 1 >= e || !lookupCandidateBaseReg( - BaseOffset, FrameSizeAdjust, + BaseReg, BaseOffset, FrameSizeAdjust, FrameReferenceInsns[ref + 1].getLocalOffset(), FrameReferenceInsns[ref + 1].getMachineInstr(), TRI)) { BaseOffset = PrevBaseOffset; diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index 3c73905..98359b1 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -307,7 +307,7 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const { OS << '\t'; if (I->isInsideBundle()) OS << " * "; - I->print(OS, &getParent()->getTarget()); + I->print(OS); } // Print the successors of this block according to the CFG. diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp index 1b5c1f1..ecc50c9 100644 --- a/lib/CodeGen/MachineBlockPlacement.cpp +++ b/lib/CodeGen/MachineBlockPlacement.cpp @@ -33,6 +33,7 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" @@ -40,13 +41,14 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetSubtargetInfo.h" #include using namespace llvm; -#define DEBUG_TYPE "block-placement2" +#define DEBUG_TYPE "block-placement" STATISTIC(NumCondBranches, "Number of conditional branches"); STATISTIC(NumUncondBranches, "Number of uncondittional branches"); @@ -61,11 +63,23 @@ static cl::opt AlignAllBlock("align-all-blocks", cl::init(0), cl::Hidden); // FIXME: Find a good default for this flag and remove the flag. -static cl::opt -ExitBlockBias("block-placement-exit-block-bias", - cl::desc("Block frequency percentage a loop exit block needs " - "over the original exit to be considered the new exit."), - cl::init(0), cl::Hidden); +static cl::opt ExitBlockBias( + "block-placement-exit-block-bias", + cl::desc("Block frequency percentage a loop exit block needs " + "over the original exit to be considered the new exit."), + cl::init(0), cl::Hidden); + +static cl::opt OutlineOptionalBranches( + "outline-optional-branches", + cl::desc("Put completely optional branches, i.e. branches with a common " + "post dominator, out of line."), + cl::init(false), cl::Hidden); + +static cl::opt OutlineOptionalThreshold( + "outline-optional-threshold", + cl::desc("Don't outline optional branches that are a single block with an " + "instruction count below this threshold"), + cl::init(4), cl::Hidden); namespace { class BlockChain; @@ -107,7 +121,7 @@ public: /// function. It also registers itself as the chain that block participates /// in with the BlockToChain mapping. BlockChain(BlockToChainMapType &BlockToChain, MachineBasicBlock *BB) - : Blocks(1, BB), BlockToChain(BlockToChain), LoopPredecessors(0) { + : Blocks(1, BB), BlockToChain(BlockToChain), LoopPredecessors(0) { assert(BB && "Cannot create a chain with a null basic block"); BlockToChain[BB] = this; } @@ -144,19 +158,18 @@ public: // Update the incoming blocks to point to this chain, and add them to the // chain structure. - for (BlockChain::iterator BI = Chain->begin(), BE = Chain->end(); - BI != BE; ++BI) { - Blocks.push_back(*BI); - assert(BlockToChain[*BI] == Chain && "Incoming blocks not in chain"); - BlockToChain[*BI] = this; + for (MachineBasicBlock *ChainBB : *Chain) { + Blocks.push_back(ChainBB); + assert(BlockToChain[ChainBB] == Chain && "Incoming blocks not in chain"); + BlockToChain[ChainBB] = this; } } #ifndef NDEBUG /// \brief Dump the blocks in this chain. LLVM_DUMP_METHOD void dump() { - for (iterator I = begin(), E = end(); I != E; ++I) - (*I)->dump(); + for (MachineBasicBlock *MBB : *this) + MBB->dump(); } #endif // NDEBUG @@ -188,6 +201,13 @@ class MachineBlockPlacement : public MachineFunctionPass { /// \brief A handle to the target's lowering info. const TargetLoweringBase *TLI; + /// \brief A handle to the post dominator tree. + MachineDominatorTree *MDT; + + /// \brief A set of blocks that are unavoidably execute, i.e. they dominate + /// all terminators of the MachineFunction. + SmallPtrSet UnavoidableBlocks; + /// \brief Allocator and owner of BlockChain structures. /// /// We build BlockChains lazily while processing the loop structure of @@ -205,28 +225,26 @@ class MachineBlockPlacement : public MachineFunctionPass { /// between basic blocks. DenseMap BlockToChain; - void markChainSuccessors(BlockChain &Chain, - MachineBasicBlock *LoopHeaderBB, + void markChainSuccessors(BlockChain &Chain, MachineBasicBlock *LoopHeaderBB, SmallVectorImpl &BlockWorkList, const BlockFilterSet *BlockFilter = nullptr); MachineBasicBlock *selectBestSuccessor(MachineBasicBlock *BB, BlockChain &Chain, const BlockFilterSet *BlockFilter); - MachineBasicBlock *selectBestCandidateBlock( - BlockChain &Chain, SmallVectorImpl &WorkList, - const BlockFilterSet *BlockFilter); - MachineBasicBlock *getFirstUnplacedBlock( - MachineFunction &F, - const BlockChain &PlacedChain, - MachineFunction::iterator &PrevUnplacedBlockIt, - const BlockFilterSet *BlockFilter); + MachineBasicBlock * + selectBestCandidateBlock(BlockChain &Chain, + SmallVectorImpl &WorkList, + const BlockFilterSet *BlockFilter); + MachineBasicBlock * + getFirstUnplacedBlock(MachineFunction &F, const BlockChain &PlacedChain, + MachineFunction::iterator &PrevUnplacedBlockIt, + const BlockFilterSet *BlockFilter); void buildChain(MachineBasicBlock *BB, BlockChain &Chain, SmallVectorImpl &BlockWorkList, const BlockFilterSet *BlockFilter = nullptr); MachineBasicBlock *findBestLoopTop(MachineLoop &L, const BlockFilterSet &LoopBlockSet); - MachineBasicBlock *findBestLoopExit(MachineFunction &F, - MachineLoop &L, + MachineBasicBlock *findBestLoopExit(MachineFunction &F, MachineLoop &L, const BlockFilterSet &LoopBlockSet); void buildLoopChains(MachineFunction &F, MachineLoop &L); void rotateLoop(BlockChain &LoopChain, MachineBasicBlock *ExitingBB, @@ -244,6 +262,7 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); AU.addRequired(); + AU.addRequired(); AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } @@ -252,12 +271,13 @@ public: char MachineBlockPlacement::ID = 0; char &llvm::MachineBlockPlacementID = MachineBlockPlacement::ID; -INITIALIZE_PASS_BEGIN(MachineBlockPlacement, "block-placement2", +INITIALIZE_PASS_BEGIN(MachineBlockPlacement, "block-placement", "Branch Probability Basic Block Placement", false, false) INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) -INITIALIZE_PASS_END(MachineBlockPlacement, "block-placement2", +INITIALIZE_PASS_END(MachineBlockPlacement, "block-placement", "Branch Probability Basic Block Placement", false, false) #ifndef NDEBUG @@ -267,8 +287,8 @@ INITIALIZE_PASS_END(MachineBlockPlacement, "block-placement2", static std::string getBlockName(MachineBasicBlock *BB) { std::string Result; raw_string_ostream OS(Result); - OS << "BB#" << BB->getNumber() - << " (derived from LLVM BB '" << BB->getName() << "')"; + OS << "BB#" << BB->getNumber(); + OS << " (derived from LLVM BB '" << BB->getName() << "')"; OS.flush(); return Result; } @@ -292,26 +312,22 @@ static std::string getBlockNum(MachineBasicBlock *BB) { /// having one fewer active predecessor. It also adds any successors of this /// chain which reach the zero-predecessor state to the worklist passed in. void MachineBlockPlacement::markChainSuccessors( - BlockChain &Chain, - MachineBasicBlock *LoopHeaderBB, + BlockChain &Chain, MachineBasicBlock *LoopHeaderBB, SmallVectorImpl &BlockWorkList, const BlockFilterSet *BlockFilter) { // Walk all the blocks in this chain, marking their successors as having // a predecessor placed. - for (BlockChain::iterator CBI = Chain.begin(), CBE = Chain.end(); - CBI != CBE; ++CBI) { + for (MachineBasicBlock *MBB : Chain) { // Add any successors for which this is the only un-placed in-loop // predecessor to the worklist as a viable candidate for CFG-neutral // placement. No subsequent placement of this block will violate the CFG // shape, so we get to use heuristics to choose a favorable placement. - for (MachineBasicBlock::succ_iterator SI = (*CBI)->succ_begin(), - SE = (*CBI)->succ_end(); - SI != SE; ++SI) { - if (BlockFilter && !BlockFilter->count(*SI)) + for (MachineBasicBlock *Succ : MBB->successors()) { + if (BlockFilter && !BlockFilter->count(Succ)) continue; - BlockChain &SuccChain = *BlockToChain[*SI]; + BlockChain &SuccChain = *BlockToChain[Succ]; // Disregard edges within a fixed chain, or edges to the loop header. - if (&Chain == &SuccChain || *SI == LoopHeaderBB) + if (&Chain == &SuccChain || Succ == LoopHeaderBB) continue; // This is a cross-chain edge that is within the loop, so decrement the @@ -331,9 +347,10 @@ void MachineBlockPlacement::markChainSuccessors( /// very hot successor edges. /// /// \returns The best successor block found, or null if none are viable. -MachineBasicBlock *MachineBlockPlacement::selectBestSuccessor( - MachineBasicBlock *BB, BlockChain &Chain, - const BlockFilterSet *BlockFilter) { +MachineBasicBlock * +MachineBlockPlacement::selectBestSuccessor(MachineBasicBlock *BB, + BlockChain &Chain, + const BlockFilterSet *BlockFilter) { const BranchProbability HotProb(4, 5); // 80% MachineBasicBlock *BestSucc = nullptr; @@ -363,6 +380,30 @@ MachineBasicBlock *MachineBlockPlacement::selectBestSuccessor( uint32_t SuccWeight = MBPI->getEdgeWeight(BB, Succ); BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight); + // If we outline optional branches, look whether Succ is unavoidable, i.e. + // dominates all terminators of the MachineFunction. If it does, other + // successors must be optional. Don't do this for cold branches. + if (OutlineOptionalBranches && SuccProb > HotProb.getCompl() && + UnavoidableBlocks.count(Succ) > 0) { + auto HasShortOptionalBranch = [&]() { + for (MachineBasicBlock *Pred : Succ->predecessors()) { + // Check whether there is an unplaced optional branch. + if (Pred == Succ || (BlockFilter && !BlockFilter->count(Pred)) || + BlockToChain[Pred] == &Chain) + continue; + // Check whether the optional branch has exactly one BB. + if (Pred->pred_size() > 1 || *Pred->pred_begin() != BB) + continue; + // Check whether the optional branch is small. + if (Pred->size() < OutlineOptionalThreshold) + return true; + } + return false; + }; + if (!HasShortOptionalBranch()) + return Succ; + } + // Only consider successors which are either "hot", or wouldn't violate // any CFG constraints. if (SuccChain.LoopPredecessors != 0) { @@ -426,29 +467,26 @@ MachineBasicBlock *MachineBlockPlacement::selectBestCandidateBlock( // some code complexity) into the loop below. WorkList.erase(std::remove_if(WorkList.begin(), WorkList.end(), [&](MachineBasicBlock *BB) { - return BlockToChain.lookup(BB) == &Chain; - }), + return BlockToChain.lookup(BB) == &Chain; + }), WorkList.end()); MachineBasicBlock *BestBlock = nullptr; BlockFrequency BestFreq; - for (SmallVectorImpl::iterator WBI = WorkList.begin(), - WBE = WorkList.end(); - WBI != WBE; ++WBI) { - BlockChain &SuccChain = *BlockToChain[*WBI]; + for (MachineBasicBlock *MBB : WorkList) { + BlockChain &SuccChain = *BlockToChain[MBB]; if (&SuccChain == &Chain) { - DEBUG(dbgs() << " " << getBlockName(*WBI) - << " -> Already merged!\n"); + DEBUG(dbgs() << " " << getBlockName(MBB) << " -> Already merged!\n"); continue; } assert(SuccChain.LoopPredecessors == 0 && "Found CFG-violating block"); - BlockFrequency CandidateFreq = MBFI->getBlockFreq(*WBI); - DEBUG(dbgs() << " " << getBlockName(*WBI) << " -> "; - MBFI->printBlockFreq(dbgs(), CandidateFreq) << " (freq)\n"); + BlockFrequency CandidateFreq = MBFI->getBlockFreq(MBB); + DEBUG(dbgs() << " " << getBlockName(MBB) << " -> "; + MBFI->printBlockFreq(dbgs(), CandidateFreq) << " (freq)\n"); if (BestBlock && BestFreq >= CandidateFreq) continue; - BestBlock = *WBI; + BestBlock = MBB; BestFreq = CandidateFreq; } return BestBlock; @@ -481,8 +519,7 @@ MachineBasicBlock *MachineBlockPlacement::getFirstUnplacedBlock( } void MachineBlockPlacement::buildChain( - MachineBasicBlock *BB, - BlockChain &Chain, + MachineBasicBlock *BB, BlockChain &Chain, SmallVectorImpl &BlockWorkList, const BlockFilterSet *BlockFilter) { assert(BB); @@ -509,8 +546,8 @@ void MachineBlockPlacement::buildChain( BestSucc = selectBestCandidateBlock(Chain, BlockWorkList, BlockFilter); if (!BestSucc) { - BestSucc = getFirstUnplacedBlock(F, Chain, PrevUnplacedBlockIt, - BlockFilter); + BestSucc = + getFirstUnplacedBlock(F, Chain, PrevUnplacedBlockIt, BlockFilter); if (!BestSucc) break; @@ -523,8 +560,8 @@ void MachineBlockPlacement::buildChain( // Zero out LoopPredecessors for the successor we're about to merge in case // we selected a successor that didn't fit naturally into the CFG. SuccChain.LoopPredecessors = 0; - DEBUG(dbgs() << "Merging from " << getBlockNum(BB) - << " to " << getBlockNum(BestSucc) << "\n"); + DEBUG(dbgs() << "Merging from " << getBlockNum(BB) << " to " + << getBlockNum(BestSucc) << "\n"); markChainSuccessors(SuccChain, LoopHeaderBB, BlockWorkList, BlockFilter); Chain.merge(BestSucc, &SuccChain); BB = *std::prev(Chain.end()); @@ -554,20 +591,17 @@ MachineBlockPlacement::findBestLoopTop(MachineLoop &L, if (!LoopBlockSet.count(*HeaderChain.begin())) return L.getHeader(); - DEBUG(dbgs() << "Finding best loop top for: " - << getBlockName(L.getHeader()) << "\n"); + DEBUG(dbgs() << "Finding best loop top for: " << getBlockName(L.getHeader()) + << "\n"); BlockFrequency BestPredFreq; MachineBasicBlock *BestPred = nullptr; - for (MachineBasicBlock::pred_iterator PI = L.getHeader()->pred_begin(), - PE = L.getHeader()->pred_end(); - PI != PE; ++PI) { - MachineBasicBlock *Pred = *PI; + for (MachineBasicBlock *Pred : L.getHeader()->predecessors()) { if (!LoopBlockSet.count(Pred)) continue; DEBUG(dbgs() << " header pred: " << getBlockName(Pred) << ", " << Pred->succ_size() << " successors, "; - MBFI->printBlockFreq(dbgs(), Pred) << " freq\n"); + MBFI->printBlockFreq(dbgs(), Pred) << " freq\n"); if (Pred->succ_size() > 1) continue; @@ -594,15 +628,13 @@ MachineBlockPlacement::findBestLoopTop(MachineLoop &L, return BestPred; } - /// \brief Find the best loop exiting block for layout. /// /// This routine implements the logic to analyze the loop looking for the best /// block to layout at the top of the loop. Typically this is done to maximize /// fallthrough opportunities. MachineBasicBlock * -MachineBlockPlacement::findBestLoopExit(MachineFunction &F, - MachineLoop &L, +MachineBlockPlacement::findBestLoopExit(MachineFunction &F, MachineLoop &L, const BlockFilterSet &LoopBlockSet) { // We don't want to layout the loop linearly in all cases. If the loop header // is just a normal basic block in the loop, we want to look for what block @@ -624,15 +656,13 @@ MachineBlockPlacement::findBestLoopExit(MachineFunction &F, // blocks where rotating to exit with that block will reach an outer loop. SmallPtrSet BlocksExitingToOuterLoop; - DEBUG(dbgs() << "Finding best loop exit for: " - << getBlockName(L.getHeader()) << "\n"); - for (MachineLoop::block_iterator I = L.block_begin(), - E = L.block_end(); - I != E; ++I) { - BlockChain &Chain = *BlockToChain[*I]; + DEBUG(dbgs() << "Finding best loop exit for: " << getBlockName(L.getHeader()) + << "\n"); + for (MachineBasicBlock *MBB : L.getBlocks()) { + BlockChain &Chain = *BlockToChain[MBB]; // Ensure that this block is at the end of a chain; otherwise it could be // mid-way through an inner loop or a successor of an analyzable branch. - if (*I != *std::prev(Chain.end())) + if (MBB != *std::prev(Chain.end())) continue; // Now walk the successors. We need to establish whether this has a viable @@ -646,43 +676,40 @@ MachineBlockPlacement::findBestLoopExit(MachineFunction &F, // the MBPI analysis, we use the internal weights and manually compute the // probabilities to avoid quadratic behavior. uint32_t WeightScale = 0; - uint32_t SumWeight = MBPI->getSumForBlock(*I, WeightScale); - for (MachineBasicBlock::succ_iterator SI = (*I)->succ_begin(), - SE = (*I)->succ_end(); - SI != SE; ++SI) { - if ((*SI)->isLandingPad()) + uint32_t SumWeight = MBPI->getSumForBlock(MBB, WeightScale); + for (MachineBasicBlock *Succ : MBB->successors()) { + if (Succ->isLandingPad()) continue; - if (*SI == *I) + if (Succ == MBB) continue; - BlockChain &SuccChain = *BlockToChain[*SI]; + BlockChain &SuccChain = *BlockToChain[Succ]; // Don't split chains, either this chain or the successor's chain. if (&Chain == &SuccChain) { - DEBUG(dbgs() << " exiting: " << getBlockName(*I) << " -> " - << getBlockName(*SI) << " (chain conflict)\n"); + DEBUG(dbgs() << " exiting: " << getBlockName(MBB) << " -> " + << getBlockName(Succ) << " (chain conflict)\n"); continue; } - uint32_t SuccWeight = MBPI->getEdgeWeight(*I, *SI); - if (LoopBlockSet.count(*SI)) { - DEBUG(dbgs() << " looping: " << getBlockName(*I) << " -> " - << getBlockName(*SI) << " (" << SuccWeight << ")\n"); + uint32_t SuccWeight = MBPI->getEdgeWeight(MBB, Succ); + if (LoopBlockSet.count(Succ)) { + DEBUG(dbgs() << " looping: " << getBlockName(MBB) << " -> " + << getBlockName(Succ) << " (" << SuccWeight << ")\n"); HasLoopingSucc = true; continue; } unsigned SuccLoopDepth = 0; - if (MachineLoop *ExitLoop = MLI->getLoopFor(*SI)) { + if (MachineLoop *ExitLoop = MLI->getLoopFor(Succ)) { SuccLoopDepth = ExitLoop->getLoopDepth(); if (ExitLoop->contains(&L)) - BlocksExitingToOuterLoop.insert(*I); + BlocksExitingToOuterLoop.insert(MBB); } BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight); - BlockFrequency ExitEdgeFreq = MBFI->getBlockFreq(*I) * SuccProb; - DEBUG(dbgs() << " exiting: " << getBlockName(*I) << " -> " - << getBlockName(*SI) << " [L:" << SuccLoopDepth - << "] ("; - MBFI->printBlockFreq(dbgs(), ExitEdgeFreq) << ")\n"); + BlockFrequency ExitEdgeFreq = MBFI->getBlockFreq(MBB) * SuccProb; + DEBUG(dbgs() << " exiting: " << getBlockName(MBB) << " -> " + << getBlockName(Succ) << " [L:" << SuccLoopDepth << "] ("; + MBFI->printBlockFreq(dbgs(), ExitEdgeFreq) << ")\n"); // Note that we bias this toward an existing layout successor to retain // incoming order in the absence of better information. The exit must have // a frequency higher than the current exit before we consider breaking @@ -690,10 +717,10 @@ MachineBlockPlacement::findBestLoopExit(MachineFunction &F, BranchProbability Bias(100 - ExitBlockBias, 100); if (!ExitingBB || BestExitLoopDepth < SuccLoopDepth || ExitEdgeFreq > BestExitEdgeFreq || - ((*I)->isLayoutSuccessor(*SI) && + (MBB->isLayoutSuccessor(Succ) && !(ExitEdgeFreq < BestExitEdgeFreq * Bias))) { BestExitEdgeFreq = ExitEdgeFreq; - ExitingBB = *I; + ExitingBB = MBB; } } @@ -734,12 +761,10 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, MachineBasicBlock *Top = *LoopChain.begin(); bool ViableTopFallthrough = false; - for (MachineBasicBlock::pred_iterator PI = Top->pred_begin(), - PE = Top->pred_end(); - PI != PE; ++PI) { - BlockChain *PredChain = BlockToChain[*PI]; - if (!LoopBlockSet.count(*PI) && - (!PredChain || *PI == *std::prev(PredChain->end()))) { + for (MachineBasicBlock *Pred : Top->predecessors()) { + BlockChain *PredChain = BlockToChain[Pred]; + if (!LoopBlockSet.count(Pred) && + (!PredChain || Pred == *std::prev(PredChain->end()))) { ViableTopFallthrough = true; break; } @@ -750,18 +775,16 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, // introduce an unnecessary branch. if (ViableTopFallthrough) { MachineBasicBlock *Bottom = *std::prev(LoopChain.end()); - for (MachineBasicBlock::succ_iterator SI = Bottom->succ_begin(), - SE = Bottom->succ_end(); - SI != SE; ++SI) { - BlockChain *SuccChain = BlockToChain[*SI]; - if (!LoopBlockSet.count(*SI) && - (!SuccChain || *SI == *SuccChain->begin())) + for (MachineBasicBlock *Succ : Bottom->successors()) { + BlockChain *SuccChain = BlockToChain[Succ]; + if (!LoopBlockSet.count(Succ) && + (!SuccChain || Succ == *SuccChain->begin())) return; } } - BlockChain::iterator ExitIt = std::find(LoopChain.begin(), LoopChain.end(), - ExitingBB); + BlockChain::iterator ExitIt = + std::find(LoopChain.begin(), LoopChain.end(), ExitingBB); if (ExitIt == LoopChain.end()) return; @@ -778,8 +801,8 @@ void MachineBlockPlacement::buildLoopChains(MachineFunction &F, MachineLoop &L) { // First recurse through any nested loops, building chains for those inner // loops. - for (MachineLoop::iterator LI = L.begin(), LE = L.end(); LI != LE; ++LI) - buildLoopChains(F, **LI); + for (MachineLoop *InnerLoop : L) + buildLoopChains(F, *InnerLoop); SmallVector BlockWorkList; BlockFilterSet LoopBlockSet(L.block_begin(), L.block_end()); @@ -805,21 +828,16 @@ void MachineBlockPlacement::buildLoopChains(MachineFunction &F, SmallPtrSet UpdatedPreds; assert(LoopChain.LoopPredecessors == 0); UpdatedPreds.insert(&LoopChain); - for (MachineLoop::block_iterator BI = L.block_begin(), - BE = L.block_end(); - BI != BE; ++BI) { - BlockChain &Chain = *BlockToChain[*BI]; + for (MachineBasicBlock *LoopBB : L.getBlocks()) { + BlockChain &Chain = *BlockToChain[LoopBB]; if (!UpdatedPreds.insert(&Chain).second) continue; assert(Chain.LoopPredecessors == 0); - for (BlockChain::iterator BCI = Chain.begin(), BCE = Chain.end(); - BCI != BCE; ++BCI) { - assert(BlockToChain[*BCI] == &Chain); - for (MachineBasicBlock::pred_iterator PI = (*BCI)->pred_begin(), - PE = (*BCI)->pred_end(); - PI != PE; ++PI) { - if (BlockToChain[*PI] == &Chain || !LoopBlockSet.count(*PI)) + for (MachineBasicBlock *ChainBB : Chain) { + assert(BlockToChain[ChainBB] == &Chain); + for (MachineBasicBlock *Pred : ChainBB->predecessors()) { + if (BlockToChain[Pred] == &Chain || !LoopBlockSet.count(Pred)) continue; ++Chain.LoopPredecessors; } @@ -841,29 +859,26 @@ void MachineBlockPlacement::buildLoopChains(MachineFunction &F, << " Loop header: " << getBlockName(*L.block_begin()) << "\n" << " Chain header: " << getBlockName(*LoopChain.begin()) << "\n"; } - for (BlockChain::iterator BCI = LoopChain.begin(), BCE = LoopChain.end(); - BCI != BCE; ++BCI) { - dbgs() << " ... " << getBlockName(*BCI) << "\n"; - if (!LoopBlockSet.erase(*BCI)) { + for (MachineBasicBlock *ChainBB : LoopChain) { + dbgs() << " ... " << getBlockName(ChainBB) << "\n"; + if (!LoopBlockSet.erase(ChainBB)) { // We don't mark the loop as bad here because there are real situations // where this can occur. For example, with an unanalyzable fallthrough // from a loop block to a non-loop block or vice versa. dbgs() << "Loop chain contains a block not contained by the loop!\n" << " Loop header: " << getBlockName(*L.block_begin()) << "\n" << " Chain header: " << getBlockName(*LoopChain.begin()) << "\n" - << " Bad block: " << getBlockName(*BCI) << "\n"; + << " Bad block: " << getBlockName(ChainBB) << "\n"; } } if (!LoopBlockSet.empty()) { BadLoop = true; - for (BlockFilterSet::iterator LBI = LoopBlockSet.begin(), - LBE = LoopBlockSet.end(); - LBI != LBE; ++LBI) + for (MachineBasicBlock *LoopBB : LoopBlockSet) dbgs() << "Loop contains blocks never placed into a chain!\n" << " Loop header: " << getBlockName(*L.block_begin()) << "\n" << " Chain header: " << getBlockName(*LoopChain.begin()) << "\n" - << " Bad block: " << getBlockName(*LBI) << "\n"; + << " Bad block: " << getBlockName(LoopBB) << "\n"; } assert(!BadLoop && "Detected problems with the placement of this loop."); }); @@ -875,8 +890,8 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { SmallVector Cond; // For AnalyzeBranch. for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) { MachineBasicBlock *BB = FI; - BlockChain *Chain - = new (ChainAllocator.Allocate()) BlockChain(BlockToChain, BB); + BlockChain *Chain = + new (ChainAllocator.Allocate()) BlockChain(BlockToChain, BB); // Also, merge any blocks which we cannot reason about and must preserve // the exact fallthrough behavior for. for (;;) { @@ -899,28 +914,44 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { } } + if (OutlineOptionalBranches) { + // Find the nearest common dominator of all of F's terminators. + MachineBasicBlock *Terminator = nullptr; + for (MachineBasicBlock &MBB : F) { + if (MBB.succ_size() == 0) { + if (Terminator == nullptr) + Terminator = &MBB; + else + Terminator = MDT->findNearestCommonDominator(Terminator, &MBB); + } + } + + // MBBs dominating this common dominator are unavoidable. + UnavoidableBlocks.clear(); + for (MachineBasicBlock &MBB : F) { + if (MDT->dominates(&MBB, Terminator)) { + UnavoidableBlocks.insert(&MBB); + } + } + } + // Build any loop-based chains. - for (MachineLoopInfo::iterator LI = MLI->begin(), LE = MLI->end(); LI != LE; - ++LI) - buildLoopChains(F, **LI); + for (MachineLoop *L : *MLI) + buildLoopChains(F, *L); SmallVector BlockWorkList; SmallPtrSet UpdatedPreds; - for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) { - MachineBasicBlock *BB = &*FI; - BlockChain &Chain = *BlockToChain[BB]; + for (MachineBasicBlock &MBB : F) { + BlockChain &Chain = *BlockToChain[&MBB]; if (!UpdatedPreds.insert(&Chain).second) continue; assert(Chain.LoopPredecessors == 0); - for (BlockChain::iterator BCI = Chain.begin(), BCE = Chain.end(); - BCI != BCE; ++BCI) { - assert(BlockToChain[*BCI] == &Chain); - for (MachineBasicBlock::pred_iterator PI = (*BCI)->pred_begin(), - PE = (*BCI)->pred_end(); - PI != PE; ++PI) { - if (BlockToChain[*PI] == &Chain) + for (MachineBasicBlock *ChainBB : Chain) { + assert(BlockToChain[ChainBB] == &Chain); + for (MachineBasicBlock *Pred : ChainBB->predecessors()) { + if (BlockToChain[Pred] == &Chain) continue; ++Chain.LoopPredecessors; } @@ -940,46 +971,40 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { // Crash at the end so we get all of the debugging output first. bool BadFunc = false; FunctionBlockSetType FunctionBlockSet; - for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) - FunctionBlockSet.insert(FI); + for (MachineBasicBlock &MBB : F) + FunctionBlockSet.insert(&MBB); - for (BlockChain::iterator BCI = FunctionChain.begin(), - BCE = FunctionChain.end(); - BCI != BCE; ++BCI) - if (!FunctionBlockSet.erase(*BCI)) { + for (MachineBasicBlock *ChainBB : FunctionChain) + if (!FunctionBlockSet.erase(ChainBB)) { BadFunc = true; dbgs() << "Function chain contains a block not in the function!\n" - << " Bad block: " << getBlockName(*BCI) << "\n"; + << " Bad block: " << getBlockName(ChainBB) << "\n"; } if (!FunctionBlockSet.empty()) { BadFunc = true; - for (FunctionBlockSetType::iterator FBI = FunctionBlockSet.begin(), - FBE = FunctionBlockSet.end(); - FBI != FBE; ++FBI) + for (MachineBasicBlock *RemainingBB : FunctionBlockSet) dbgs() << "Function contains blocks never placed into a chain!\n" - << " Bad block: " << getBlockName(*FBI) << "\n"; + << " Bad block: " << getBlockName(RemainingBB) << "\n"; } assert(!BadFunc && "Detected problems with the block placement."); }); // Splice the blocks into place. MachineFunction::iterator InsertPos = F.begin(); - for (BlockChain::iterator BI = FunctionChain.begin(), - BE = FunctionChain.end(); - BI != BE; ++BI) { - DEBUG(dbgs() << (BI == FunctionChain.begin() ? "Placing chain " - : " ... ") - << getBlockName(*BI) << "\n"); - if (InsertPos != MachineFunction::iterator(*BI)) - F.splice(InsertPos, *BI); + for (MachineBasicBlock *ChainBB : FunctionChain) { + DEBUG(dbgs() << (ChainBB == *FunctionChain.begin() ? "Placing chain " + : " ... ") + << getBlockName(ChainBB) << "\n"); + if (InsertPos != MachineFunction::iterator(ChainBB)) + F.splice(InsertPos, ChainBB); else ++InsertPos; // Update the terminator of the previous block. - if (BI == FunctionChain.begin()) + if (ChainBB == *FunctionChain.begin()) continue; - MachineBasicBlock *PrevBB = std::prev(MachineFunction::iterator(*BI)); + MachineBasicBlock *PrevBB = std::prev(MachineFunction::iterator(ChainBB)); // FIXME: It would be awesome of updateTerminator would just return rather // than assert when the branch cannot be analyzed in order to remove this @@ -989,16 +1014,16 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { if (!TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond)) { // The "PrevBB" is not yet updated to reflect current code layout, so, // o. it may fall-through to a block without explict "goto" instruction - // before layout, and no longer fall-through it after layout; or + // before layout, and no longer fall-through it after layout; or // o. just opposite. - // + // // AnalyzeBranch() may return erroneous value for FBB when these two // situations take place. For the first scenario FBB is mistakenly set // NULL; for the 2nd scenario, the FBB, which is expected to be NULL, // is mistakenly pointing to "*BI". // bool needUpdateBr = true; - if (!Cond.empty() && (!FBB || FBB == *BI)) { + if (!Cond.empty() && (!FBB || FBB == ChainBB)) { PrevBB->updateTerminator(); needUpdateBr = false; Cond.clear(); @@ -1018,7 +1043,7 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { << getBlockName(PrevBB) << "\n"); DEBUG(dbgs() << " Edge weight: " << MBPI->getEdgeWeight(PrevBB, FBB) << " vs " << MBPI->getEdgeWeight(PrevBB, TBB) << "\n"); - DebugLoc dl; // FIXME: this is nowhere + DebugLoc dl; // FIXME: this is nowhere TII->RemoveBranch(*PrevBB); TII->InsertBranch(*PrevBB, FBB, TBB, Cond, dl); needUpdateBr = true; @@ -1042,29 +1067,30 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { if (F.getFunction()->hasFnAttribute(Attribute::OptimizeForSize)) return; if (FunctionChain.begin() == FunctionChain.end()) - return; // Empty chain. + return; // Empty chain. const BranchProbability ColdProb(1, 5); // 20% BlockFrequency EntryFreq = MBFI->getBlockFreq(F.begin()); BlockFrequency WeightedEntryFreq = EntryFreq * ColdProb; - for (BlockChain::iterator BI = std::next(FunctionChain.begin()), - BE = FunctionChain.end(); - BI != BE; ++BI) { + for (MachineBasicBlock *ChainBB : FunctionChain) { + if (ChainBB == *FunctionChain.begin()) + continue; + // Don't align non-looping basic blocks. These are unlikely to execute // enough times to matter in practice. Note that we'll still handle // unnatural CFGs inside of a natural outer loop (the common case) and // rotated loops. - MachineLoop *L = MLI->getLoopFor(*BI); + MachineLoop *L = MLI->getLoopFor(ChainBB); if (!L) continue; unsigned Align = TLI->getPrefLoopAlignment(L); if (!Align) - continue; // Don't care about loop alignment. + continue; // Don't care about loop alignment. // If the block is cold relative to the function entry don't waste space // aligning it. - BlockFrequency Freq = MBFI->getBlockFreq(*BI); + BlockFrequency Freq = MBFI->getBlockFreq(ChainBB); if (Freq < WeightedEntryFreq) continue; @@ -1077,12 +1103,13 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { // Check for the existence of a non-layout predecessor which would benefit // from aligning this block. - MachineBasicBlock *LayoutPred = *std::prev(BI); + MachineBasicBlock *LayoutPred = + &*std::prev(MachineFunction::iterator(ChainBB)); // Force alignment if all the predecessors are jumps. We already checked // that the block isn't cold above. - if (!LayoutPred->isSuccessor(*BI)) { - (*BI)->setAlignment(Align); + if (!LayoutPred->isSuccessor(ChainBB)) { + ChainBB->setAlignment(Align); continue; } @@ -1090,10 +1117,11 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { // cold relative to the block. When this is true, other predecessors make up // all of the hot entries into the block and thus alignment is likely to be // important. - BranchProbability LayoutProb = MBPI->getEdgeProbability(LayoutPred, *BI); + BranchProbability LayoutProb = + MBPI->getEdgeProbability(LayoutPred, ChainBB); BlockFrequency LayoutEdgeFreq = MBFI->getBlockFreq(LayoutPred) * LayoutProb; if (LayoutEdgeFreq <= (Freq * ColdProb)) - (*BI)->setAlignment(Align); + ChainBB->setAlignment(Align); } } @@ -1110,6 +1138,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &F) { MLI = &getAnalysis(); TII = F.getSubtarget().getInstrInfo(); TLI = F.getSubtarget().getTargetLowering(); + MDT = &getAnalysis(); assert(BlockToChain.empty()); buildCFGChains(F); @@ -1119,9 +1148,8 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &F) { if (AlignAllBlock) // Align all of the blocks in the function to a specific alignment. - for (MachineFunction::iterator FI = F.begin(), FE = F.end(); - FI != FE; ++FI) - FI->setAlignment(AlignAllBlock); + for (MachineBasicBlock &MBB : F) + MBB.setAlignment(AlignAllBlock); // We always return true as we have no way to track whether the final order // differs from the original order. @@ -1176,20 +1204,19 @@ bool MachineBlockPlacementStats::runOnMachineFunction(MachineFunction &F) { MBPI = &getAnalysis(); MBFI = &getAnalysis(); - for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { - BlockFrequency BlockFreq = MBFI->getBlockFreq(I); - Statistic &NumBranches = (I->succ_size() > 1) ? NumCondBranches - : NumUncondBranches; - Statistic &BranchTakenFreq = (I->succ_size() > 1) ? CondBranchTakenFreq - : UncondBranchTakenFreq; - for (MachineBasicBlock::succ_iterator SI = I->succ_begin(), - SE = I->succ_end(); - SI != SE; ++SI) { + for (MachineBasicBlock &MBB : F) { + BlockFrequency BlockFreq = MBFI->getBlockFreq(&MBB); + Statistic &NumBranches = + (MBB.succ_size() > 1) ? NumCondBranches : NumUncondBranches; + Statistic &BranchTakenFreq = + (MBB.succ_size() > 1) ? CondBranchTakenFreq : UncondBranchTakenFreq; + for (MachineBasicBlock *Succ : MBB.successors()) { // Skip if this successor is a fallthrough. - if (I->isLayoutSuccessor(*SI)) + if (MBB.isLayoutSuccessor(Succ)) continue; - BlockFrequency EdgeFreq = BlockFreq * MBPI->getEdgeProbability(I, *SI); + BlockFrequency EdgeFreq = + BlockFreq * MBPI->getEdgeProbability(&MBB, Succ); ++NumBranches; BranchTakenFreq += EdgeFreq.getFrequency(); } diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp index 21b9c5a..f72d72a 100644 --- a/lib/CodeGen/MachineCSE.cpp +++ b/lib/CodeGen/MachineCSE.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/RecyclingAllocator.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; diff --git a/lib/CodeGen/MachineCopyPropagation.cpp b/lib/CodeGen/MachineCopyPropagation.cpp index cbd6272..9611122 100644 --- a/lib/CodeGen/MachineCopyPropagation.cpp +++ b/lib/CodeGen/MachineCopyPropagation.cpp @@ -75,10 +75,9 @@ MachineCopyPropagation::SourceNoLongerAvailable(unsigned Reg, I != E; ++I) { unsigned MappedDef = *I; // Source of copy is no longer available for propagation. - if (AvailCopyMap.erase(MappedDef)) { - for (MCSubRegIterator SR(MappedDef, TRI); SR.isValid(); ++SR) - AvailCopyMap.erase(*SR); - } + AvailCopyMap.erase(MappedDef); + for (MCSubRegIterator SR(MappedDef, TRI); SR.isValid(); ++SR) + AvailCopyMap.erase(*SR); } } } diff --git a/lib/CodeGen/MachineDominators.cpp b/lib/CodeGen/MachineDominators.cpp index df60cf3..467a2e4 100644 --- a/lib/CodeGen/MachineDominators.cpp +++ b/lib/CodeGen/MachineDominators.cpp @@ -14,6 +14,7 @@ #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/SmallBitVector.h" using namespace llvm; @@ -59,3 +60,68 @@ void MachineDominatorTree::releaseMemory() { void MachineDominatorTree::print(raw_ostream &OS, const Module*) const { DT->print(OS); } + +void MachineDominatorTree::applySplitCriticalEdges() const { + // Bail out early if there is nothing to do. + if (CriticalEdgesToSplit.empty()) + return; + + // For each element in CriticalEdgesToSplit, remember whether or not element + // is the new immediate domminator of its successor. The mapping is done by + // index, i.e., the information for the ith element of CriticalEdgesToSplit is + // the ith element of IsNewIDom. + SmallBitVector IsNewIDom(CriticalEdgesToSplit.size(), true); + size_t Idx = 0; + + // Collect all the dominance properties info, before invalidating + // the underlying DT. + for (CriticalEdge &Edge : CriticalEdgesToSplit) { + // Update dominator information. + MachineBasicBlock *Succ = Edge.ToBB; + MachineDomTreeNode *SuccDTNode = DT->getNode(Succ); + + for (MachineBasicBlock *PredBB : Succ->predecessors()) { + if (PredBB == Edge.NewBB) + continue; + // If we are in this situation: + // FromBB1 FromBB2 + // + + + // + + + + + // + + + + + // ... Split1 Split2 ... + // + + + // + + + // + + // Succ + // Instead of checking the domiance property with Split2, we check it with + // FromBB2 since Split2 is still unknown of the underlying DT structure. + if (NewBBs.count(PredBB)) { + assert(PredBB->pred_size() == 1 && "A basic block resulting from a " + "critical edge split has more " + "than one predecessor!"); + PredBB = *PredBB->pred_begin(); + } + if (!DT->dominates(SuccDTNode, DT->getNode(PredBB))) { + IsNewIDom[Idx] = false; + break; + } + } + ++Idx; + } + + // Now, update DT with the collected dominance properties info. + Idx = 0; + for (CriticalEdge &Edge : CriticalEdgesToSplit) { + // We know FromBB dominates NewBB. + MachineDomTreeNode *NewDTNode = DT->addNewBlock(Edge.NewBB, Edge.FromBB); + + // If all the other predecessors of "Succ" are dominated by "Succ" itself + // then the new block is the new immediate dominator of "Succ". Otherwise, + // the new block doesn't dominate anything. + if (IsNewIDom[Idx]) + DT->changeImmediateDominator(DT->getNode(Edge.ToBB), NewDTNode); + ++Idx; + } + NewBBs.clear(); + CriticalEdgesToSplit.clear(); +} diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index 151a260..6ceace8 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -54,7 +54,7 @@ void ilist_traits::deleteNode(MachineBasicBlock *MBB) { MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM, unsigned FunctionNum, MachineModuleInfo &mmi) - : Fn(F), Target(TM), STI(TM.getSubtargetImpl()), Ctx(mmi.getContext()), + : Fn(F), Target(TM), STI(TM.getSubtargetImpl(*F)), Ctx(mmi.getContext()), MMI(mmi) { if (STI->getRegisterInfo()) RegInfo = new (Allocator) MachineRegisterInfo(this); @@ -584,14 +584,6 @@ int MachineFrameInfo::CreateFixedSpillStackObject(uint64_t Size, return -++NumFixedObjects; } -int MachineFrameInfo::CreateFrameAllocation(uint64_t Size) { - // Force the use of a frame pointer. The intention is that this intrinsic be - // used in conjunction with unwind mechanisms that leak the frame pointer. - setFrameAddressIsTaken(true); - Size = RoundUpToAlignment(Size, StackAlignment); - return CreateStackObject(Size, StackAlignment, false); -} - BitVector MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const { assert(MBB && "MBB must be valid"); @@ -903,16 +895,16 @@ static bool CanShareConstantPoolEntry(const Constant *A, const Constant *B, // DataLayout. if (isa(A->getType())) A = ConstantFoldInstOperands(Instruction::PtrToInt, IntTy, - const_cast(A), TD); + const_cast(A), *TD); else if (A->getType() != IntTy) A = ConstantFoldInstOperands(Instruction::BitCast, IntTy, - const_cast(A), TD); + const_cast(A), *TD); if (isa(B->getType())) B = ConstantFoldInstOperands(Instruction::PtrToInt, IntTy, - const_cast(B), TD); + const_cast(B), *TD); else if (B->getType() != IntTy) B = ConstantFoldInstOperands(Instruction::BitCast, IntTy, - const_cast(B), TD); + const_cast(B), *TD); return A == B; } diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 981e4a3..1240efb 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -276,17 +276,8 @@ hash_code llvm::hash_value(const MachineOperand &MO) { /// print - Print the specified machine operand. /// -void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { - // If the instruction is embedded into a basic block, we can find the - // target info for the instruction. - if (!TM) - if (const MachineInstr *MI = getParent()) - if (const MachineBasicBlock *MBB = MI->getParent()) - if (const MachineFunction *MF = MBB->getParent()) - TM = &MF->getTarget(); - const TargetRegisterInfo *TRI = - TM ? TM->getSubtargetImpl()->getRegisterInfo() : nullptr; - +void MachineOperand::print(raw_ostream &OS, + const TargetRegisterInfo *TRI) const { switch (getType()) { case MachineOperand::MO_Register: OS << PrintReg(getReg(), TRI, getSubReg()); @@ -1512,23 +1503,19 @@ void MachineInstr::dump() const { #endif } -static void printDebugLoc(DebugLoc DL, const MachineFunction *MF, - raw_ostream &CommentOS) { - const LLVMContext &Ctx = MF->getFunction()->getContext(); - DL.print(Ctx, CommentOS); -} - -void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, - bool SkipOpers) const { - // We can be a bit tidier if we know the TargetMachine and/or MachineFunction. +void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const { + // We can be a bit tidier if we know the MachineFunction. const MachineFunction *MF = nullptr; + const TargetRegisterInfo *TRI = nullptr; const MachineRegisterInfo *MRI = nullptr; + const TargetInstrInfo *TII = nullptr; if (const MachineBasicBlock *MBB = getParent()) { MF = MBB->getParent(); - if (!TM && MF) - TM = &MF->getTarget(); - if (MF) + if (MF) { MRI = &MF->getRegInfo(); + TRI = MF->getSubtarget().getRegisterInfo(); + TII = MF->getSubtarget().getInstrInfo(); + } } // Save a list of virtual registers. @@ -1541,7 +1528,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, !getOperand(StartOp).isImplicit(); ++StartOp) { if (StartOp != 0) OS << ", "; - getOperand(StartOp).print(OS, TM); + getOperand(StartOp).print(OS, TRI); unsigned Reg = getOperand(StartOp).getReg(); if (TargetRegisterInfo::isVirtualRegister(Reg)) VirtRegs.push_back(Reg); @@ -1551,8 +1538,8 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, OS << " = "; // Print the opcode name. - if (TM && TM->getSubtargetImpl()->getInstrInfo()) - OS << TM->getSubtargetImpl()->getInstrInfo()->getName(getOpcode()); + if (TII) + OS << TII->getName(getOpcode()); else OS << "UNKNOWN"; @@ -1568,7 +1555,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, if (isInlineAsm() && e >= InlineAsm::MIOp_FirstOperand) { // Print asm string. OS << " "; - getOperand(InlineAsm::MIOp_AsmString).print(OS, TM); + getOperand(InlineAsm::MIOp_AsmString).print(OS, TRI); // Print HasSideEffects, MayLoad, MayStore, IsAlignStack unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); @@ -1606,9 +1593,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, if (TargetRegisterInfo::isPhysicalRegister(Reg)) { if (MRI->use_empty(Reg)) { bool HasAliasLive = false; - for (MCRegAliasIterator AI( - Reg, TM->getSubtargetImpl()->getRegisterInfo(), true); - AI.isValid(); ++AI) { + for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { unsigned AliasReg = *AI; if (!MRI->use_empty(AliasReg)) { HasAliasLive = true; @@ -1641,10 +1626,9 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, if (DI.isVariable() && !DIV.getName().empty()) OS << "!\"" << DIV.getName() << '\"'; else - MO.print(OS, TM); - } else if (TM && (isInsertSubreg() || isRegSequence()) && MO.isImm()) { - OS << TM->getSubtargetImpl()->getRegisterInfo()->getSubRegIndexName( - MO.getImm()); + MO.print(OS, TRI); + } else if (TRI && (isInsertSubreg() || isRegSequence()) && MO.isImm()) { + OS << TRI->getSubRegIndexName(MO.getImm()); } else if (i == AsmDescOp && MO.isImm()) { // Pretty print the inline asm operand descriptor. OS << '$' << AsmOpCount++; @@ -1661,11 +1645,8 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, unsigned RCID = 0; if (InlineAsm::hasRegClassConstraint(Flag, RCID)) { - if (TM) { - const TargetRegisterInfo *TRI = - TM->getSubtargetImpl()->getRegisterInfo(); - OS << ':' - << TRI->getRegClassName(TRI->getRegClass(RCID)); + if (TRI) { + OS << ':' << TRI->getRegClassName(TRI->getRegClass(RCID)); } else OS << ":RC" << RCID; } @@ -1679,7 +1660,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, // Compute the index of the next operand descriptor. AsmDescOp += 1 + InlineAsm::getNumOperandRegisters(Flag); } else - MO.print(OS, TM); + MO.print(OS, TRI); } // Briefly indicate whether any call clobbers were omitted. @@ -1715,7 +1696,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, if (!HaveSemi) OS << ";"; HaveSemi = true; for (unsigned i = 0; i != VirtRegs.size(); ++i) { const TargetRegisterClass *RC = MRI->getRegClass(VirtRegs[i]); - OS << " " << MRI->getTargetRegisterInfo()->getRegClassName(RC) + OS << " " << TRI->getRegClassName(RC) << ':' << PrintReg(VirtRegs[i]); for (unsigned j = i+1; j != VirtRegs.size();) { if (MRI->getRegClass(VirtRegs[j]) != RC) { @@ -1738,7 +1719,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(InlinedAt); if (!InlinedAtDL.isUnknown() && MF) { OS << " inlined @[ "; - printDebugLoc(InlinedAtDL, MF, OS); + InlinedAtDL.print(OS); OS << " ]"; } } @@ -1747,7 +1728,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, } else if (!debugLoc.isUnknown() && MF) { if (!HaveSemi) OS << ";"; OS << " dbg:"; - printDebugLoc(debugLoc, MF, OS); + debugLoc.print(OS); } OS << '\n'; diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 64d0932..2f65a2e 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -54,6 +54,12 @@ HoistCheapInsts("hoist-cheap-insts", cl::desc("MachineLICM should hoist even cheap instructions"), cl::init(false), cl::Hidden); +static cl::opt +SinkInstsToAvoidSpills("sink-insts-to-avoid-spills", + cl::desc("MachineLICM should sink instructions into " + "loops to avoid register spills"), + cl::init(false), cl::Hidden); + STATISTIC(NumHoisted, "Number of machine instructions hoisted out of loops"); STATISTIC(NumLowRP, @@ -243,6 +249,11 @@ namespace { void HoistOutOfLoop(MachineDomTreeNode *LoopHeaderNode); void HoistRegion(MachineDomTreeNode *N, bool IsHeader); + /// SinkIntoLoop - Sink instructions into loops if profitable. This + /// especially tries to prevent register spills caused by register pressure + /// if there is little to no overhead moving instructions into loops. + void SinkIntoLoop(); + /// getRegisterClassIDAndCost - For a given MI, register, and the operand /// index, return the ID and cost of its representative register class by /// reference. @@ -381,6 +392,9 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { FirstInLoop = true; HoistOutOfLoop(N); CSEMap.clear(); + + if (SinkInstsToAvoidSpills) + SinkIntoLoop(); } } @@ -771,6 +785,53 @@ void MachineLICM::HoistOutOfLoop(MachineDomTreeNode *HeaderN) { } } +void MachineLICM::SinkIntoLoop() { + MachineBasicBlock *Preheader = getCurPreheader(); + if (!Preheader) + return; + + SmallVector Candidates; + for (MachineBasicBlock::instr_iterator I = Preheader->instr_begin(); + I != Preheader->instr_end(); ++I) { + // We need to ensure that we can safely move this instruction into the loop. + // As such, it must not have side-effects, e.g. such as a call has. + if (IsLoopInvariantInst(*I) && !HasLoopPHIUse(I)) + Candidates.push_back(I); + } + + for (MachineInstr *I : Candidates) { + const MachineOperand &MO = I->getOperand(0); + if (!MO.isDef() || !MO.isReg() || !MO.getReg()) + continue; + if (!MRI->hasOneDef(MO.getReg())) + continue; + bool CanSink = true; + MachineBasicBlock *B = nullptr; + for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) { + // FIXME: Come up with a proper cost model that estimates whether sinking + // the instruction (and thus possibly executing it on every loop + // iteration) is more expensive than a register. + // For now assumes that copies are cheap and thus almost always worth it. + if (!MI.isCopy()) { + CanSink = false; + break; + } + if (!B) { + B = MI.getParent(); + continue; + } + B = DT->findNearestCommonDominator(B, MI.getParent()); + if (!B) { + CanSink = false; + break; + } + } + if (!CanSink || !B || B == Preheader) + continue; + B->splice(B->getFirstNonPHI(), Preheader, I); + } +} + static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) { return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg()); } diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp index 89054d4..ce6abdd 100644 --- a/lib/CodeGen/MachineLoopInfo.cpp +++ b/lib/CodeGen/MachineLoopInfo.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/Passes.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; // Explicitly instantiate methods in LoopInfoImpl.h for MI-level Loops. diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp index 32b7db1..278a8f2 100644 --- a/lib/CodeGen/MachineRegisterInfo.cpp +++ b/lib/CodeGen/MachineRegisterInfo.cpp @@ -65,7 +65,7 @@ MachineRegisterInfo::recomputeRegClass(unsigned Reg) { const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); const TargetRegisterClass *OldRC = getRegClass(Reg); const TargetRegisterClass *NewRC = - getTargetRegisterInfo()->getLargestLegalSuperClass(OldRC); + getTargetRegisterInfo()->getLargestLegalSuperClass(OldRC, *MF); // Stop early if there is no room to grow. if (NewRC == OldRC) diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp index 89ac6a8..7a3c80b 100644 --- a/lib/CodeGen/MachineScheduler.cpp +++ b/lib/CodeGen/MachineScheduler.cpp @@ -209,6 +209,11 @@ static MachineSchedRegistry DefaultSchedRegistry("default", "Use the target's default scheduler choice.", useDefaultMachineSched); +static cl::opt EnableMachineSched( + "enable-misched", + cl::desc("Enable the machine instruction scheduling pass."), cl::init(true), + cl::Hidden); + /// Forward declare the standard machine scheduler. This will be used as the /// default scheduler if the target does not set a default. static ScheduleDAGInstrs *createGenericSchedLive(MachineSchedContext *C); @@ -304,6 +309,12 @@ ScheduleDAGInstrs *PostMachineScheduler::createPostMachineScheduler() { /// design would be to split blocks at scheduling boundaries, but LLVM has a /// general bias against block splitting purely for implementation simplicity. bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { + if (EnableMachineSched.getNumOccurrences()) { + if (!EnableMachineSched) + return false; + } else if (!mf.getSubtarget().enableMachineScheduler()) + return false; + DEBUG(dbgs() << "Before MISsched:\n"; mf.print(dbgs())); // Initialize the context of the pass. diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index bdb094f..991241e 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -397,7 +397,7 @@ void MachineVerifier::report(const char *msg, assert(MO); report(msg, MO->getParent()); errs() << "- operand " << MONum << ": "; - MO->print(errs(), TM); + MO->print(errs(), TRI); errs() << "\n"; } @@ -739,7 +739,7 @@ void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) { if (!isUInt<5>(MI->getOperand(1).getImm())) report("Unknown asm flags", &MI->getOperand(1), 1); - assert(InlineAsm::MIOp_FirstOperand == 2 && "Asm format changed"); + static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed"); unsigned OpNo = InlineAsm::MIOp_FirstOperand; unsigned NumOps; @@ -927,7 +927,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { TII->getRegClass(MCID, MONum, TRI, *MF)) { if (SubIdx) { const TargetRegisterClass *SuperRC = - TRI->getLargestLegalSuperClass(RC); + TRI->getLargestLegalSuperClass(RC, *MF); if (!SuperRC) { report("No largest legal super class exists.", MO, MONum); return; @@ -1573,7 +1573,8 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, if (!hasRead) { // When tracking subregister liveness, the main range must start new // values on partial register writes, even if there is no read. - if (!MRI->tracksSubRegLiveness() || LaneMask != 0 || !hasSubRegDef) { + if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask != 0 || + !hasSubRegDef) { report("Instruction ending live segment doesn't read the register", MI); errs() << S << " in " << LR << '\n'; @@ -1649,40 +1650,35 @@ void MachineVerifier::verifyLiveRange(const LiveRange &LR, unsigned Reg, } void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) { - verifyLiveRange(LI, LI.reg); - unsigned Reg = LI.reg; - if (TargetRegisterInfo::isVirtualRegister(Reg)) { - unsigned Mask = 0; - unsigned MaxMask = MRI->getMaxLaneMaskForVReg(Reg); - for (const LiveInterval::SubRange &SR : LI.subranges()) { - if ((Mask & SR.LaneMask) != 0) - report("Lane masks of sub ranges overlap in live interval", MF, LI); - if ((SR.LaneMask & ~MaxMask) != 0) - report("Subrange lanemask is invalid", MF, LI); - Mask |= SR.LaneMask; - verifyLiveRange(SR, LI.reg, SR.LaneMask); - if (!LI.covers(SR)) - report("A Subrange is not covered by the main range", MF, LI); - } - } else if (LI.hasSubRanges()) { - report("subregister liveness only allowed for virtual registers", MF, LI); + assert(TargetRegisterInfo::isVirtualRegister(Reg)); + verifyLiveRange(LI, Reg); + + unsigned Mask = 0; + unsigned MaxMask = MRI->getMaxLaneMaskForVReg(Reg); + for (const LiveInterval::SubRange &SR : LI.subranges()) { + if ((Mask & SR.LaneMask) != 0) + report("Lane masks of sub ranges overlap in live interval", MF, LI); + if ((SR.LaneMask & ~MaxMask) != 0) + report("Subrange lanemask is invalid", MF, LI); + Mask |= SR.LaneMask; + verifyLiveRange(SR, LI.reg, SR.LaneMask); + if (!LI.covers(SR)) + report("A Subrange is not covered by the main range", MF, LI); } // Check the LI only has one connected component. - if (TargetRegisterInfo::isVirtualRegister(LI.reg)) { - ConnectedVNInfoEqClasses ConEQ(*LiveInts); - unsigned NumComp = ConEQ.Classify(&LI); - if (NumComp > 1) { - report("Multiple connected components in live interval", MF, LI); - for (unsigned comp = 0; comp != NumComp; ++comp) { - errs() << comp << ": valnos"; - for (LiveInterval::const_vni_iterator I = LI.vni_begin(), - E = LI.vni_end(); I!=E; ++I) - if (comp == ConEQ.getEqClass(*I)) - errs() << ' ' << (*I)->id; - errs() << '\n'; - } + ConnectedVNInfoEqClasses ConEQ(*LiveInts); + unsigned NumComp = ConEQ.Classify(&LI); + if (NumComp > 1) { + report("Multiple connected components in live interval", MF, LI); + for (unsigned comp = 0; comp != NumComp; ++comp) { + errs() << comp << ": valnos"; + for (LiveInterval::const_vni_iterator I = LI.vni_begin(), + E = LI.vni_end(); I!=E; ++I) + if (comp == ConEQ.getEqClass(*I)) + errs() << ' ' << (*I)->id; + errs() << '\n'; } } } diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp index def2e3d..d514190 100644 --- a/lib/CodeGen/PHIElimination.cpp +++ b/lib/CodeGen/PHIElimination.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" #include @@ -46,6 +47,10 @@ SplitAllCriticalEdges("phi-elim-split-all-critical-edges", cl::init(false), cl::Hidden, cl::desc("Split all critical edges during " "PHI elimination")); +static cl::opt NoPhiElimLiveOutEarlyExit( + "no-phi-elim-live-out-early-exit", cl::init(false), cl::Hidden, + cl::desc("Do not use an early exit if isLiveOutPastPHIs returns true.")); + namespace { class PHIElimination : public MachineFunctionPass { MachineRegisterInfo *MRI; // Machine register information @@ -573,12 +578,14 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, // there is a risk it may not be coalesced away. // // If the copy would be a kill, there is no need to split the edge. - if (!isLiveOutPastPHIs(Reg, PreMBB) && !SplitAllCriticalEdges) + bool ShouldSplit = isLiveOutPastPHIs(Reg, PreMBB); + if (!ShouldSplit && !NoPhiElimLiveOutEarlyExit) continue; - - DEBUG(dbgs() << PrintReg(Reg) << " live-out before critical edge BB#" - << PreMBB->getNumber() << " -> BB#" << MBB.getNumber() - << ": " << *BBI); + if (ShouldSplit) { + DEBUG(dbgs() << PrintReg(Reg) << " live-out before critical edge BB#" + << PreMBB->getNumber() << " -> BB#" << MBB.getNumber() + << ": " << *BBI); + } // If Reg is not live-in to MBB, it means it must be live-in to some // other PreMBB successor, and we can avoid the interference by splitting @@ -588,7 +595,7 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, // is likely to be left after coalescing. If we are looking at a loop // exiting edge, split it so we won't insert code in the loop, otherwise // don't bother. - bool ShouldSplit = !isLiveIn(Reg, &MBB) || SplitAllCriticalEdges; + ShouldSplit = ShouldSplit && !isLiveIn(Reg, &MBB); // Check for a loop exiting edge. if (!ShouldSplit && CurLoop != PreLoop) { @@ -603,7 +610,7 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, // Split unless this edge is entering CurLoop from an outer loop. ShouldSplit = PreLoop && !PreLoop->contains(CurLoop); } - if (!ShouldSplit) + if (!ShouldSplit && !SplitAllCriticalEdges) continue; if (!PreMBB->SplitCriticalEdge(&MBB, this)) { DEBUG(dbgs() << "Failed to split critical edge.\n"); diff --git a/lib/CodeGen/Passes.cpp b/lib/CodeGen/Passes.cpp index 272d068..c128414 100644 --- a/lib/CodeGen/Passes.cpp +++ b/lib/CodeGen/Passes.cpp @@ -23,8 +23,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/SymbolRewriter.h" @@ -55,9 +54,6 @@ static cl::opt DisableMachineCSE("disable-machine-cse", cl::Hidden, static cl::opt OptimizeRegAlloc("optimize-regalloc", cl::Hidden, cl::desc("Enable optimized register allocation compilation path.")); -static cl::opt -EnableMachineSched("enable-misched", - cl::desc("Enable the machine instruction scheduling pass.")); static cl::opt DisablePostRAMachineLICM("disable-postra-machine-licm", cl::Hidden, cl::desc("Disable Machine LICM")); @@ -116,28 +112,6 @@ static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID, return PassID; } -/// Allow Pass selection to be overriden by command line options. This supports -/// flags with ternary conditions. TargetID is passed through by default. The -/// pass is suppressed when the option is false. When the option is true, the -/// StandardID is selected if the target provides no default. -static IdentifyingPassPtr applyOverride(IdentifyingPassPtr TargetID, - cl::boolOrDefault Override, - AnalysisID StandardID) { - switch (Override) { - case cl::BOU_UNSET: - return TargetID; - case cl::BOU_TRUE: - if (TargetID.isValid()) - return TargetID; - if (StandardID == nullptr) - report_fatal_error("Target cannot enable pass"); - return StandardID; - case cl::BOU_FALSE: - return IdentifyingPassPtr(); - } - llvm_unreachable("Invalid command line option state"); -} - /// Allow standard passes to be disabled by the command line, regardless of who /// is adding the pass. /// @@ -182,9 +156,6 @@ static IdentifyingPassPtr overridePass(AnalysisID StandardID, if (StandardID == &MachineCSEID) return applyDisable(TargetID, DisableMachineCSE); - if (StandardID == &MachineSchedulerID) - return applyOverride(TargetID, EnableMachineSched, StandardID); - if (StandardID == &TargetPassConfig::PostRAMachineLICMID) return applyDisable(TargetID, DisablePostRAMachineLICM); @@ -249,11 +220,6 @@ TargetPassConfig::TargetPassConfig(TargetMachine *tm, PassManagerBase &pm) // Substitute Pseudo Pass IDs for real ones. substitutePass(&EarlyTailDuplicateID, &TailDuplicateID); substitutePass(&PostRAMachineLICMID, &MachineLICMID); - - // Temporarily disable experimental passes. - const TargetSubtargetInfo &ST = *TM->getSubtargetImpl(); - if (!ST.useMachineScheduler()) - disablePass(&MachineSchedulerID); } /// Insert InsertedPassID pass after TargetPassID. @@ -409,10 +375,8 @@ void TargetPassConfig::addIRPasses() { // Before running any passes, run the verifier to determine if the input // coming from the front-end and/or optimizer is valid. - if (!DisableVerify) { + if (!DisableVerify) addPass(createVerifierPass()); - addPass(createDebugInfoVerifierPass()); - } // Run loop strength reduction before anything else. if (getOptLevel() != CodeGenOpt::None && !DisableLSR) { @@ -455,7 +419,11 @@ void TargetPassConfig::addPassesToHandleExceptions() { addPass(createDwarfEHPass(TM)); break; case ExceptionHandling::WinEH: + // We support using both GCC-style and MSVC-style exceptions on Windows, so + // add both preparation passes. Each pass will only actually run if it + // recognizes the personality function. addPass(createWinEHPass(TM)); + addPass(createDwarfEHPass(TM)); break; case ExceptionHandling::None: addPass(createLowerInvokePass()); @@ -479,12 +447,6 @@ void TargetPassConfig::addCodeGenPrepare() { void TargetPassConfig::addISelPrepare() { addPreISel(); - // Need to verify DebugInfo *before* creating the stack protector analysis. - // It's a function pass, and verifying between it and its users causes a - // crash. - if (!DisableVerify) - addPass(createDebugInfoVerifierPass()); - addPass(createStackProtectorPass(TM)); if (PrintISelInput) diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp index 283d1f2..ebe05e3 100644 --- a/lib/CodeGen/PeepholeOptimizer.cpp +++ b/lib/CodeGen/PeepholeOptimizer.cpp @@ -76,6 +76,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" @@ -411,8 +412,7 @@ optimizeExtInstr(MachineInstr *MI, MachineBasicBlock *MBB, if (ExtendLife && !ExtendedUses.empty()) // Extend the liveness of the extension result. - std::copy(ExtendedUses.begin(), ExtendedUses.end(), - std::back_inserter(Uses)); + Uses.append(ExtendedUses.begin(), ExtendedUses.end()); // Now replace all uses. bool Changed = false; @@ -916,7 +916,7 @@ bool PeepholeOptimizer::optimizeCoalescableCopy(MachineInstr *MI) { // => v0 = COPY v1 // Currently we haven't seen motivating example for that and we // want to avoid untested code. - NumRewrittenCopies += Changed == true; + NumRewrittenCopies += Changed; return Changed; } diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index 6d29b98..e073e6a 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -16,7 +16,6 @@ // //===----------------------------------------------------------------------===// -#include "PrologEpilogInserter.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" @@ -28,6 +27,7 @@ #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/StackProtector.h" #include "llvm/IR/DiagnosticInfo.h" @@ -48,6 +48,53 @@ using namespace llvm; #define DEBUG_TYPE "pei" +namespace { +class PEI : public MachineFunctionPass { +public: + static char ID; + PEI() : MachineFunctionPass(ID) { + initializePEIPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + /// runOnMachineFunction - Insert prolog/epilog code and replace abstract + /// frame indexes with appropriate references. + /// + bool runOnMachineFunction(MachineFunction &Fn) override; + +private: + RegScavenger *RS; + + // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved + // stack frame indexes. + unsigned MinCSFrameIndex, MaxCSFrameIndex; + + // Entry and return blocks of the current function. + MachineBasicBlock *EntryBlock; + SmallVector ReturnBlocks; + + // Flag to control whether to use the register scavenger to resolve + // frame index materialization registers. Set according to + // TRI->requiresFrameIndexScavenging() for the current function. + bool FrameIndexVirtualScavenging; + + void calculateSets(MachineFunction &Fn); + void calculateCallsInformation(MachineFunction &Fn); + void calculateCalleeSavedRegisters(MachineFunction &Fn); + void insertCSRSpillsAndRestores(MachineFunction &Fn); + void calculateFrameObjectOffsets(MachineFunction &Fn); + void replaceFrameIndices(MachineFunction &Fn); + void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, + int &SPAdj); + void scavengeFrameVirtualRegs(MachineFunction &Fn); + void insertPrologEpilogCode(MachineFunction &Fn); + + // Convenience for recognizing return blocks. + bool isReturnBlock(MachineBasicBlock *MBB); +}; +} // namespace + char PEI::ID = 0; char &llvm::PrologEpilogCodeInserterID = PEI::ID; @@ -810,17 +857,6 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, continue; } - // Frame allocations are target independent. Simply swap the index with - // the offset. - if (MI->getOpcode() == TargetOpcode::FRAME_ALLOC) { - assert(TFI->hasFP(Fn) && "frame alloc requires FP"); - MachineOperand &FI = MI->getOperand(i); - unsigned Reg; - int FrameOffset = TFI->getFrameIndexReference(Fn, FI.getIndex(), Reg); - FI.ChangeToImmediate(FrameOffset); - continue; - } - // Some instructions (e.g. inline asm instructions) can have // multiple frame indices and/or cause eliminateFrameIndex // to insert more than one instruction. We need the register diff --git a/lib/CodeGen/PrologEpilogInserter.h b/lib/CodeGen/PrologEpilogInserter.h deleted file mode 100644 index f88b8ef..0000000 --- a/lib/CodeGen/PrologEpilogInserter.h +++ /dev/null @@ -1,78 +0,0 @@ -//===-- PrologEpilogInserter.h - Prolog/Epilog code insertion -*- C++ -*---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This pass is responsible for finalizing the functions frame layout, saving -// callee saved registers, and for emitting prolog & epilog code for the -// function. -// -// This pass must be run after register allocation. After this pass is -// executed, it is illegal to construct MO_FrameIndex operands. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_CODEGEN_PROLOGEPILOGINSERTER_H -#define LLVM_LIB_CODEGEN_PROLOGEPILOGINSERTER_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SparseBitVector.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetRegisterInfo.h" - -namespace llvm { - class RegScavenger; - class MachineBasicBlock; - - class PEI : public MachineFunctionPass { - public: - static char ID; - PEI() : MachineFunctionPass(ID) { - initializePEIPass(*PassRegistry::getPassRegistry()); - } - - void getAnalysisUsage(AnalysisUsage &AU) const override; - - /// runOnMachineFunction - Insert prolog/epilog code and replace abstract - /// frame indexes with appropriate references. - /// - bool runOnMachineFunction(MachineFunction &Fn) override; - - private: - RegScavenger *RS; - - // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved - // stack frame indexes. - unsigned MinCSFrameIndex, MaxCSFrameIndex; - - // Entry and return blocks of the current function. - MachineBasicBlock* EntryBlock; - SmallVector ReturnBlocks; - - // Flag to control whether to use the register scavenger to resolve - // frame index materialization registers. Set according to - // TRI->requiresFrameIndexScavenging() for the curren function. - bool FrameIndexVirtualScavenging; - - void calculateSets(MachineFunction &Fn); - void calculateCallsInformation(MachineFunction &Fn); - void calculateCalleeSavedRegisters(MachineFunction &Fn); - void insertCSRSpillsAndRestores(MachineFunction &Fn); - void calculateFrameObjectOffsets(MachineFunction &Fn); - void replaceFrameIndices(MachineFunction &Fn); - void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, - int &SPAdj); - void scavengeFrameVirtualRegs(MachineFunction &Fn); - void insertPrologEpilogCode(MachineFunction &Fn); - - // Convenience for recognizing return blocks. - bool isReturnBlock(MachineBasicBlock* MBB); - }; -} // End llvm namespace -#endif diff --git a/lib/CodeGen/RegAllocBase.cpp b/lib/CodeGen/RegAllocBase.cpp index 6b346f4..16ff48e 100644 --- a/lib/CodeGen/RegAllocBase.cpp +++ b/lib/CodeGen/RegAllocBase.cpp @@ -27,6 +27,7 @@ #endif #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/Timer.h" diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp index edc3294..e94f1bb 100644 --- a/lib/CodeGen/RegAllocGreedy.cpp +++ b/lib/CodeGen/RegAllocGreedy.cpp @@ -1554,7 +1554,8 @@ RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order, DEBUG(dbgs() << "Split around " << Uses.size() << " individual instrs.\n"); - const TargetRegisterClass *SuperRC = TRI->getLargestLegalSuperClass(CurRC); + const TargetRegisterClass *SuperRC = + TRI->getLargestLegalSuperClass(CurRC, *MF); unsigned SuperRCNumAllocatableRegs = RCI.getNumAllocatableRegs(SuperRC); // Split around every non-copy instruction if this split will relax // the constraints on the virtual register. diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 77a42b3..eeff73d 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -178,8 +178,40 @@ class Interference : public PBQPRAConstraint { private: typedef const PBQP::RegAlloc::AllowedRegVector* AllowedRegVecPtr; - typedef std::pair IMatrixKey; - typedef DenseMap IMatrixCache; + typedef std::pair IKey; + typedef DenseMap IMatrixCache; + typedef DenseSet DisjointAllowedRegsCache; + typedef std::pair IEdgeKey; + typedef DenseSet IEdgeCache; + + bool haveDisjointAllowedRegs(const PBQPRAGraph &G, PBQPRAGraph::NodeId NId, + PBQPRAGraph::NodeId MId, + const DisjointAllowedRegsCache &D) const { + const auto *NRegs = &G.getNodeMetadata(NId).getAllowedRegs(); + const auto *MRegs = &G.getNodeMetadata(MId).getAllowedRegs(); + + if (NRegs == MRegs) + return false; + + if (NRegs < MRegs) + return D.count(IKey(NRegs, MRegs)) > 0; + + return D.count(IKey(MRegs, NRegs)) > 0; + } + + void setDisjointAllowedRegs(const PBQPRAGraph &G, PBQPRAGraph::NodeId NId, + PBQPRAGraph::NodeId MId, + DisjointAllowedRegsCache &D) { + const auto *NRegs = &G.getNodeMetadata(NId).getAllowedRegs(); + const auto *MRegs = &G.getNodeMetadata(MId).getAllowedRegs(); + + assert(NRegs != MRegs && "AllowedRegs can not be disjoint with itself"); + + if (NRegs < MRegs) + D.insert(IKey(NRegs, MRegs)); + else + D.insert(IKey(MRegs, NRegs)); + } // Holds (Interval, CurrentSegmentID, and NodeId). The first two are required // for the fast interference graph construction algorithm. The last is there @@ -247,6 +279,13 @@ public: // and uniquing them. IMatrixCache C; + // Finding an edge is expensive in the worst case (O(max_clique(G))). So + // cache locally edges we have already seen. + IEdgeCache EC; + + // Cache known disjoint allowed registers pairs + DisjointAllowedRegsCache D; + typedef std::set IntervalSet; typedef std::priority_queue, decltype(&lowestStartPoint)> IntervalQueue; @@ -290,14 +329,21 @@ public: for (const auto &A : Active) { PBQP::GraphBase::NodeId MId = getNodeId(A); + // Do not add an edge when the nodes' allowed registers do not + // intersect: there is obviously no interference. + if (haveDisjointAllowedRegs(G, NId, MId, D)) + continue; + // Check that we haven't already added this edge - // FIXME: findEdge is expensive in the worst case (O(max_clique(G))). - // It might be better to replace this with a local bit-matrix. - if (G.findEdge(NId, MId) != PBQPRAGraph::invalidEdgeId()) + IEdgeKey EK(std::min(NId, MId), std::max(NId, MId)); + if (EC.count(EK)) continue; // This is a new edge - add it to the graph. - createInterferenceEdge(G, NId, MId, C); + if (!createInterferenceEdge(G, NId, MId, C)) + setDisjointAllowedRegs(G, NId, MId, D); + else + EC.insert(EK); } // Finally, add Cur to the Active set. @@ -307,35 +353,48 @@ public: private: - void createInterferenceEdge(PBQPRAGraph &G, PBQPRAGraph::NodeId NId, - PBQPRAGraph::NodeId MId, IMatrixCache &C) { + // Create an Interference edge and add it to the graph, unless it is + // a null matrix, meaning the nodes' allowed registers do not have any + // interference. This case occurs frequently between integer and floating + // point registers for example. + // return true iff both nodes interferes. + bool createInterferenceEdge(PBQPRAGraph &G, + PBQPRAGraph::NodeId NId, PBQPRAGraph::NodeId MId, + IMatrixCache &C) { const TargetRegisterInfo &TRI = *G.getMetadata().MF.getSubtarget().getRegisterInfo(); - const auto &NRegs = G.getNodeMetadata(NId).getAllowedRegs(); const auto &MRegs = G.getNodeMetadata(MId).getAllowedRegs(); // Try looking the edge costs up in the IMatrixCache first. - IMatrixKey K(&NRegs, &MRegs); + IKey K(&NRegs, &MRegs); IMatrixCache::iterator I = C.find(K); if (I != C.end()) { G.addEdgeBypassingCostAllocator(NId, MId, I->second); - return; + return true; } PBQPRAGraph::RawMatrix M(NRegs.size() + 1, MRegs.size() + 1, 0); + bool NodesInterfere = false; for (unsigned I = 0; I != NRegs.size(); ++I) { unsigned PRegN = NRegs[I]; for (unsigned J = 0; J != MRegs.size(); ++J) { unsigned PRegM = MRegs[J]; - if (TRI.regsOverlap(PRegN, PRegM)) + if (TRI.regsOverlap(PRegN, PRegM)) { M[I + 1][J + 1] = std::numeric_limits::infinity(); + NodesInterfere = true; + } } } + if (!NodesInterfere) + return false; + PBQPRAGraph::EdgeId EId = G.addEdge(NId, MId, std::move(M)); C[K] = G.getEdgeCostsPtr(EId); + + return true; } }; diff --git a/lib/CodeGen/RegisterClassInfo.cpp b/lib/CodeGen/RegisterClassInfo.cpp index ab33672..178fa18 100644 --- a/lib/CodeGen/RegisterClassInfo.cpp +++ b/lib/CodeGen/RegisterClassInfo.cpp @@ -131,7 +131,8 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const { RCI.NumRegs = StressRA; // Check if RC is a proper sub-class. - if (const TargetRegisterClass *Super = TRI->getLargestLegalSuperClass(RC)) + if (const TargetRegisterClass *Super = + TRI->getLargestLegalSuperClass(RC, *MF)) if (Super != RC && getNumAllocatableRegs(Super) > RCI.NumRegs) RCI.ProperSubClass = true; @@ -175,6 +176,6 @@ unsigned RegisterClassInfo::computePSetLimit(unsigned Idx) const { } compute(RC); unsigned NReserved = RC->getNumRegs() - getNumAllocatableRegs(RC); - return TRI->getRegPressureSetLimit(Idx) - - TRI->getRegClassWeight(RC).RegWeight * NReserved; + return TRI->getRegPressureSetLimit(*MF, Idx) - + TRI->getRegClassWeight(RC).RegWeight * NReserved; } diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp index 1e4cfe8..9e3cf41 100644 --- a/lib/CodeGen/RegisterCoalescer.cpp +++ b/lib/CodeGen/RegisterCoalescer.cpp @@ -58,6 +58,10 @@ EnableJoining("join-liveintervals", cl::desc("Coalesce copies (default=true)"), cl::init(true)); +static cl::opt UseTerminalRule("terminal-rule", + cl::desc("Apply the terminal rule"), + cl::init(false)); + /// Temporary flag to test critical edge unsplitting. static cl::opt EnableJoinSplits("join-splitedges", @@ -160,12 +164,14 @@ namespace { /// LaneMask are split as necessary. @p LaneMask are the lanes that /// @p ToMerge will occupy in the coalescer register. @p LI has its subrange /// lanemasks already adjusted to the coalesced register. - void mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge, + /// @returns false if live range conflicts couldn't get resolved. + bool mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge, unsigned LaneMask, CoalescerPair &CP); /// Join the liveranges of two subregisters. Joins @p RRange into /// @p LRange, @p RRange may be invalid afterwards. - void joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, + /// @returns false if live range conflicts couldn't get resolved. + bool joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, unsigned LaneMask, const CoalescerPair &CP); /// We found a non-trivially-coalescable copy. If the source value number is @@ -204,6 +210,20 @@ namespace { /// Returns true if @p CopyMI was a copy of an undef value and eliminated. bool eliminateUndefCopy(MachineInstr *CopyMI); + /// Check whether or not we should apply the terminal rule on the + /// destination (Dst) of \p Copy. + /// When the terminal rule applies, Copy is not profitable to + /// coalesce. + /// Dst is terminal if it has exactly one affinity (Dst, Src) and + /// at least one interference (Dst, Dst2). If Dst is terminal, the + /// terminal rule consists in checking that at least one of + /// interfering node, say Dst2, has an affinity of equal or greater + /// weight with Src. + /// In that case, Dst2 and Dst will not be able to be both coalesced + /// with Src. Since Dst2 exposes more coalescing opportunities than + /// Dst, we can drop \p Copy. + bool applyTerminalRule(const MachineInstr &Copy) const; + public: static char ID; ///< Class identification, replacement for typeinfo RegisterCoalescer() : MachineFunctionPass(ID) { @@ -1143,7 +1163,7 @@ void RegisterCoalescer::updateRegDefsUses(unsigned SrcReg, // A subreg use of a partially undef (super) register may be a complete // undef use now and then has to be marked that way. - if (SubIdx != 0 && MO.isUse() && MRI->tracksSubRegLiveness()) { + if (SubIdx != 0 && MO.isUse() && MRI->shouldTrackSubRegLiveness(DstReg)) { if (!DstInt->hasSubRanges()) { BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); unsigned Mask = MRI->getMaxLaneMaskForVReg(DstInt->reg); @@ -1756,6 +1776,9 @@ public: void eraseInstrs(SmallPtrSetImpl &ErasedInstrs, SmallVectorImpl &ShrinkRegs); + /// Remove liverange defs at places where implicit defs will be removed. + void removeImplicitDefs(); + /// Get the value assignments suitable for passing to LiveInterval::join. const int *getAssignments() const { return Assignments.data(); } }; @@ -1856,7 +1879,11 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) { assert(DefMI != nullptr); if (SubRangeJoin) { // We don't care about the lanes when joining subregister ranges. - V.ValidLanes = V.WriteLanes = 1; + V.WriteLanes = V.ValidLanes = 1; + if (DefMI->isImplicitDef()) { + V.ValidLanes = 0; + V.ErasableImplicitDef = true; + } } else { bool Redef = false; V.ValidLanes = V.WriteLanes = computeWriteLanes(DefMI, Redef); @@ -2339,6 +2366,18 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, unsigned &ShrinkMask) LI.removeEmptySubRanges(); } +void JoinVals::removeImplicitDefs() { + for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { + Val &V = Vals[i]; + if (V.Resolution != CR_Keep || !V.ErasableImplicitDef || !V.Pruned) + continue; + + VNInfo *VNI = LR.getValNumInfo(i); + VNI->markUnused(); + LR.removeValNo(VNI); + } +} + void JoinVals::eraseInstrs(SmallPtrSetImpl &ErasedInstrs, SmallVectorImpl &ShrinkRegs) { for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { @@ -2382,7 +2421,7 @@ void JoinVals::eraseInstrs(SmallPtrSetImpl &ErasedInstrs, } } -void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, +bool RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, unsigned LaneMask, const CoalescerPair &CP) { SmallVector NewVNInfo; @@ -2392,12 +2431,19 @@ void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, NewVNInfo, CP, LIS, TRI, true, true); // Compute NewVNInfo and resolve conflicts (see also joinVirtRegs()) - // Conflicts should already be resolved so the mapping/resolution should - // always succeed. - if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals)) - llvm_unreachable("Can't join subrange although main ranges are compatible"); - if (!LHSVals.resolveConflicts(RHSVals) || !RHSVals.resolveConflicts(LHSVals)) - llvm_unreachable("Can't join subrange although main ranges are compatible"); + // We should be able to resolve all conflicts here as we could successfully do + // it on the mainrange already. There is however a problem when multiple + // ranges get mapped to the "overflow" lane mask bit which creates unexpected + // interferences. + if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals)) { + DEBUG(dbgs() << "*** Couldn't join subrange!\n"); + return false; + } + if (!LHSVals.resolveConflicts(RHSVals) || + !RHSVals.resolveConflicts(LHSVals)) { + DEBUG(dbgs() << "*** Couldn't join subrange!\n"); + return false; + } // The merging algorithm in LiveInterval::join() can't handle conflicting // value mappings, so we need to remove any live ranges that overlap a @@ -2407,6 +2453,9 @@ void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, LHSVals.pruneValues(RHSVals, EndPoints, false); RHSVals.pruneValues(LHSVals, EndPoints, false); + LHSVals.removeImplicitDefs(); + RHSVals.removeImplicitDefs(); + LRange.verify(); RRange.verify(); @@ -2416,16 +2465,17 @@ void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, DEBUG(dbgs() << "\t\tjoined lanes: " << LRange << "\n"); if (EndPoints.empty()) - return; + return true; // Recompute the parts of the live range we had to remove because of // CR_Replace conflicts. DEBUG(dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: " << LRange << '\n'); LIS->extendToIndices(LRange, EndPoints); + return true; } -void RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI, +bool RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge, unsigned LaneMask, CoalescerPair &CP) { BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); @@ -2453,7 +2503,8 @@ void RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI, CommonRange = &R; } LiveRange RangeCopy(ToMerge, Allocator); - joinSubRegRanges(*CommonRange, RangeCopy, Common, CP); + if (!joinSubRegRanges(*CommonRange, RangeCopy, Common, CP)) + return false; LaneMask &= ~RMask; } @@ -2461,13 +2512,14 @@ void RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI, DEBUG(dbgs() << format("\t\tNew Lane %04X\n", LaneMask)); LI.createSubRangeFrom(Allocator, LaneMask, ToMerge); } + return true; } bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) { SmallVector NewVNInfo; LiveInterval &RHS = LIS->getInterval(CP.getSrcReg()); LiveInterval &LHS = LIS->getInterval(CP.getDstReg()); - bool TrackSubRegLiveness = MRI->tracksSubRegLiveness(); + bool TrackSubRegLiveness = MRI->shouldTrackSubRegLiveness(*CP.getNewRC()); JoinVals RHSVals(RHS, CP.getSrcReg(), CP.getSrcIdx(), 0, NewVNInfo, CP, LIS, TRI, false, TrackSubRegLiveness); JoinVals LHSVals(LHS, CP.getDstReg(), CP.getDstIdx(), 0, NewVNInfo, CP, LIS, @@ -2511,22 +2563,40 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) { // Determine lanemasks of RHS in the coalesced register and merge subranges. unsigned SrcIdx = CP.getSrcIdx(); + bool Abort = false; if (!RHS.hasSubRanges()) { unsigned Mask = SrcIdx == 0 ? CP.getNewRC()->getLaneMask() : TRI->getSubRegIndexLaneMask(SrcIdx); - mergeSubRangeInto(LHS, RHS, Mask, CP); + if (!mergeSubRangeInto(LHS, RHS, Mask, CP)) + Abort = true; } else { // Pair up subranges and merge. for (LiveInterval::SubRange &R : RHS.subranges()) { unsigned Mask = TRI->composeSubRegIndexLaneMask(SrcIdx, R.LaneMask); - mergeSubRangeInto(LHS, R, Mask, CP); + if (!mergeSubRangeInto(LHS, R, Mask, CP)) { + Abort = true; + break; + } } } + if (Abort) { + // This shouldn't have happened :-( + // However we are aware of at least one existing problem where we + // can't merge subranges when multiple ranges end up in the + // "overflow bit" 32. As a workaround we drop all subregister ranges + // which means we loose some precision but are back to a well defined + // state. + assert((CP.getNewRC()->getLaneMask() & 0x80000000u) + && "SubRange merge should only fail when merging into bit 32."); + DEBUG(dbgs() << "\tSubrange join aborted!\n"); + LHS.clearSubRanges(); + RHS.clearSubRanges(); + } else { + DEBUG(dbgs() << "\tJoined SubRanges " << LHS << "\n"); - DEBUG(dbgs() << "\tJoined SubRanges " << LHS << "\n"); - - LHSVals.pruneSubRegValues(LHS, ShrinkMask); - RHSVals.pruneSubRegValues(LHS, ShrinkMask); + LHSVals.pruneSubRegValues(LHS, ShrinkMask); + RHSVals.pruneSubRegValues(LHS, ShrinkMask); + } } // The merging algorithm in LiveInterval::join() can't handle conflicting @@ -2645,6 +2715,58 @@ copyCoalesceWorkList(MutableArrayRef CurrList) { return Progress; } +/// Check if DstReg is a terminal node. +/// I.e., it does not have any affinity other than \p Copy. +static bool isTerminalReg(unsigned DstReg, const MachineInstr &Copy, + const MachineRegisterInfo *MRI) { + assert(Copy.isCopyLike()); + // Check if the destination of this copy as any other affinity. + for (const MachineInstr &MI : MRI->reg_nodbg_instructions(DstReg)) + if (&MI != &Copy && MI.isCopyLike()) + return false; + return true; +} + +bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const { + assert(Copy.isCopyLike()); + if (!UseTerminalRule) + return false; + // Check if the destination of this copy has any other affinity. + unsigned DstReg = Copy.getOperand(0).getReg(); + if (TargetRegisterInfo::isPhysicalRegister(DstReg) || + !isTerminalReg(DstReg, Copy, MRI)) + return false; + + // DstReg is a terminal node. Check if it inteferes with any other + // copy involving SrcReg. + unsigned SrcReg = Copy.getOperand(1).getReg(); + const MachineBasicBlock *OrigBB = Copy.getParent(); + const LiveInterval &DstLI = LIS->getInterval(DstReg); + for (const MachineInstr &MI : MRI->reg_nodbg_instructions(SrcReg)) { + // Technically we should check if the weight of the new copy is + // interesting compared to the other one and update the weight + // of the copies accordingly. However, this would only work if + // we would gather all the copies first then coalesce, whereas + // right now we interleave both actions. + // For now, just consider the copies that are in the same block. + if (&MI == &Copy || !MI.isCopyLike() || MI.getParent() != OrigBB) + continue; + unsigned OtherReg = MI.getOperand(0).getReg(); + if (OtherReg == SrcReg) + OtherReg = MI.getOperand(1).getReg(); + // Check if OtherReg is a non-terminal. + if (TargetRegisterInfo::isPhysicalRegister(OtherReg) || + isTerminalReg(OtherReg, MI, MRI)) + continue; + // Check that OtherReg interfere with DstReg. + if (LIS->getInterval(OtherReg).overlaps(DstLI)) { + DEBUG(dbgs() << "Apply terminal rule for: " << PrintReg(DstReg) << '\n'); + return true; + } + } + return false; +} + void RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) { DEBUG(dbgs() << MBB->getName() << ":\n"); @@ -2659,7 +2781,7 @@ RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) { // cmp+jmp macro fusion. for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) { - if (!MII->isCopyLike()) + if (!MII->isCopyLike() || applyTerminalRule(*MII)) continue; if (isLocalCopy(&(*MII), LIS)) LocalWorkList.push_back(&(*MII)); @@ -2670,7 +2792,7 @@ RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) { else { for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) - if (MII->isCopyLike()) + if (MII->isCopyLike() && !applyTerminalRule(*MII)) WorkList.push_back(MII); } // Try coalescing the collected copies immediately, and remove the nulls. @@ -2741,7 +2863,7 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { AA = &getAnalysis(); Loops = &getAnalysis(); if (EnableGlobalCopies == cl::BOU_UNSET) - JoinGlobalCopies = STI.useMachineScheduler(); + JoinGlobalCopies = STI.enableJoinGlobalCopies(); else JoinGlobalCopies = (EnableGlobalCopies == cl::BOU_TRUE); diff --git a/lib/CodeGen/RegisterPressure.cpp b/lib/CodeGen/RegisterPressure.cpp index 9925efb..3634103 100644 --- a/lib/CodeGen/RegisterPressure.cpp +++ b/lib/CodeGen/RegisterPressure.cpp @@ -304,6 +304,7 @@ static bool containsReg(ArrayRef RegUnits, unsigned RegUnit) { return std::find(RegUnits.begin(), RegUnits.end(), RegUnit) != RegUnits.end(); } +namespace { /// Collect this instruction's unique uses and defs into SmallVectors for /// processing defs and uses in order. /// @@ -354,6 +355,7 @@ protected: } } }; +} // namespace /// Collect physical and virtual register operands. static void collectOperands(const MachineInstr *MI, diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 78bfd23..17dd729 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -96,14 +96,15 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) { /// getUnderlyingObjects - This is a wrapper around GetUnderlyingObjects /// and adds support for basic ptrtoint+arithmetic+inttoptr sequences. static void getUnderlyingObjects(const Value *V, - SmallVectorImpl &Objects) { + SmallVectorImpl &Objects, + const DataLayout &DL) { SmallPtrSet Visited; SmallVector Working(1, V); do { V = Working.pop_back_val(); SmallVector Objs; - GetUnderlyingObjects(const_cast(V), Objs); + GetUnderlyingObjects(const_cast(V), Objs, DL); for (SmallVectorImpl::iterator I = Objs.begin(), IE = Objs.end(); I != IE; ++I) { @@ -132,7 +133,8 @@ UnderlyingObjectsVector; /// object, return the Value for that object. static void getUnderlyingObjectsForInstr(const MachineInstr *MI, const MachineFrameInfo *MFI, - UnderlyingObjectsVector &Objects) { + UnderlyingObjectsVector &Objects, + const DataLayout &DL) { if (!MI->hasOneMemOperand() || (!(*MI->memoperands_begin())->getValue() && !(*MI->memoperands_begin())->getPseudoValue()) || @@ -156,7 +158,7 @@ static void getUnderlyingObjectsForInstr(const MachineInstr *MI, return; SmallVector Objs; - getUnderlyingObjects(V, Objs); + getUnderlyingObjects(V, Objs, DL); for (SmallVectorImpl::iterator I = Objs.begin(), IE = Objs.end(); I != IE; ++I) { @@ -468,7 +470,8 @@ static inline bool isGlobalMemoryObject(AliasAnalysis *AA, MachineInstr *MI) { // This MI might have either incomplete info, or known to be unsafe // to deal with (i.e. volatile object). static inline bool isUnsafeMemoryObject(MachineInstr *MI, - const MachineFrameInfo *MFI) { + const MachineFrameInfo *MFI, + const DataLayout &DL) { if (!MI || MI->memoperands_empty()) return true; // We purposefully do no check for hasOneMemOperand() here @@ -491,7 +494,7 @@ static inline bool isUnsafeMemoryObject(MachineInstr *MI, return true; SmallVector Objs; - getUnderlyingObjects(V, Objs); + getUnderlyingObjects(V, Objs, DL); for (SmallVectorImpl::iterator I = Objs.begin(), IE = Objs.end(); I != IE; ++I) { // Does this pointer refer to a distinct and identifiable object? @@ -508,7 +511,7 @@ static inline bool isUnsafeMemoryObject(MachineInstr *MI, /// these two MIs be reordered during scheduling from memory dependency /// point of view. static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI, - MachineInstr *MIa, + const DataLayout &DL, MachineInstr *MIa, MachineInstr *MIb) { const MachineFunction *MF = MIa->getParent()->getParent(); const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); @@ -527,7 +530,7 @@ static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI, if (!MIa->hasOneMemOperand() || !MIb->hasOneMemOperand()) return true; - if (isUnsafeMemoryObject(MIa, MFI) || isUnsafeMemoryObject(MIb, MFI)) + if (isUnsafeMemoryObject(MIa, MFI, DL) || isUnsafeMemoryObject(MIb, MFI, DL)) return true; // If we are dealing with two "normal" loads, we do not need an edge @@ -579,10 +582,10 @@ static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI, /// This recursive function iterates over chain deps of SUb looking for /// "latest" node that needs a chain edge to SUa. -static unsigned -iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, - SUnit *SUa, SUnit *SUb, SUnit *ExitSU, unsigned *Depth, - SmallPtrSetImpl &Visited) { +static unsigned iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, + const DataLayout &DL, SUnit *SUa, SUnit *SUb, + SUnit *ExitSU, unsigned *Depth, + SmallPtrSetImpl &Visited) { if (!SUa || !SUb || SUb == ExitSU) return *Depth; @@ -607,7 +610,7 @@ iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, // add that edge to the predecessors chain of SUb, // and stop descending. if (*Depth > 200 || - MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) { + MIsNeedChainEdge(AA, MFI, DL, SUa->getInstr(), SUb->getInstr())) { SUb->addPred(SDep(SUa, SDep::MayAliasMem)); return *Depth; } @@ -617,7 +620,7 @@ iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, for (SUnit::const_succ_iterator I = SUb->Succs.begin(), E = SUb->Succs.end(); I != E; ++I) if (I->isNormalMemoryOrBarrier()) - iterateChainSucc (AA, MFI, SUa, I->getSUnit(), ExitSU, Depth, Visited); + iterateChainSucc(AA, MFI, DL, SUa, I->getSUnit(), ExitSU, Depth, Visited); return *Depth; } @@ -626,7 +629,8 @@ iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, /// checks whether SU can be aliasing any node dominated /// by it. static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI, - SUnit *SU, SUnit *ExitSU, std::set &CheckList, + const DataLayout &DL, SUnit *SU, SUnit *ExitSU, + std::set &CheckList, unsigned LatencyToLoad) { if (!SU) return; @@ -638,7 +642,7 @@ static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI, I != IE; ++I) { if (SU == *I) continue; - if (MIsNeedChainEdge(AA, MFI, SU->getInstr(), (*I)->getInstr())) { + if (MIsNeedChainEdge(AA, MFI, DL, SU->getInstr(), (*I)->getInstr())) { SDep Dep(SU, SDep::MayAliasMem); Dep.setLatency(((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0); (*I)->addPred(Dep); @@ -649,22 +653,22 @@ static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI, for (SUnit::const_succ_iterator J = (*I)->Succs.begin(), JE = (*I)->Succs.end(); J != JE; ++J) if (J->isNormalMemoryOrBarrier()) - iterateChainSucc (AA, MFI, SU, J->getSUnit(), - ExitSU, &Depth, Visited); + iterateChainSucc(AA, MFI, DL, SU, J->getSUnit(), ExitSU, &Depth, + Visited); } } /// Check whether two objects need a chain edge, if so, add it /// otherwise remember the rejected SU. -static inline -void addChainDependency (AliasAnalysis *AA, const MachineFrameInfo *MFI, - SUnit *SUa, SUnit *SUb, - std::set &RejectList, - unsigned TrueMemOrderLatency = 0, - bool isNormalMemory = false) { +static inline void addChainDependency(AliasAnalysis *AA, + const MachineFrameInfo *MFI, + const DataLayout &DL, SUnit *SUa, + SUnit *SUb, std::set &RejectList, + unsigned TrueMemOrderLatency = 0, + bool isNormalMemory = false) { // If this is a false dependency, // do not add the edge, but rememeber the rejected node. - if (MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) { + if (MIsNeedChainEdge(AA, MFI, DL, SUa->getInstr(), SUb->getInstr())) { SDep Dep(SUa, isNormalMemory ? SDep::MayAliasMem : SDep::Barrier); Dep.setLatency(TrueMemOrderLatency); SUb->addPred(Dep); @@ -883,7 +887,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, BarrierChain = SU; // This is a barrier event that acts as a pivotal node in the DAG, // so it is safe to clear list of exposed nodes. - adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, + adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes, TrueMemOrderLatency); RejectMemNodes.clear(); NonAliasMemDefs.clear(); @@ -896,25 +900,27 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, unsigned ChainLatency = 0; if (AliasChain->getInstr()->mayLoad()) ChainLatency = TrueMemOrderLatency; - addChainDependency(AAForDep, MFI, SU, AliasChain, RejectMemNodes, - ChainLatency); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain, + RejectMemNodes, ChainLatency); } AliasChain = SU; for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) - addChainDependency(AAForDep, MFI, SU, PendingLoads[k], RejectMemNodes, + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + PendingLoads[k], RejectMemNodes, TrueMemOrderLatency); for (MapVector >::iterator I = AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) - addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + I->second[i], RejectMemNodes); } for (MapVector >::iterator I = AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) - addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes, - TrueMemOrderLatency); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + I->second[i], RejectMemNodes, TrueMemOrderLatency); } - adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, + adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes, TrueMemOrderLatency); PendingLoads.clear(); AliasMemDefs.clear(); @@ -928,7 +934,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, BarrierChain->addPred(SDep(SU, SDep::Barrier)); UnderlyingObjectsVector Objs; - getUnderlyingObjectsForInstr(MI, MFI, Objs); + getUnderlyingObjectsForInstr(MI, MFI, Objs, *TM.getDataLayout()); if (Objs.empty()) { // Treat all other stores conservatively. @@ -952,8 +958,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); if (I != IE) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) - addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes, - 0, true); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + I->second[i], RejectMemNodes, 0, true); // If we're not using AA, then we only need one store per object. if (!AAForDep) @@ -977,7 +983,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end()); if (J != JE) { for (unsigned i = 0, e = J->second.size(); i != e; ++i) - addChainDependency(AAForDep, MFI, SU, J->second[i], RejectMemNodes, + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + J->second[i], RejectMemNodes, TrueMemOrderLatency, true); J->second.clear(); } @@ -986,13 +993,15 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, // Add dependencies from all the PendingLoads, i.e. loads // with no underlying object. for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) - addChainDependency(AAForDep, MFI, SU, PendingLoads[k], RejectMemNodes, + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + PendingLoads[k], RejectMemNodes, TrueMemOrderLatency); // Add dependence on alias chain, if needed. if (AliasChain) - addChainDependency(AAForDep, MFI, SU, AliasChain, RejectMemNodes); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain, + RejectMemNodes); } - adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, + adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes, TrueMemOrderLatency); } else if (MI->mayLoad()) { bool MayAlias = true; @@ -1000,7 +1009,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, // Invariant load, no chain dependencies needed! } else { UnderlyingObjectsVector Objs; - getUnderlyingObjectsForInstr(MI, MFI, Objs); + getUnderlyingObjectsForInstr(MI, MFI, Objs, *TM.getDataLayout()); if (Objs.empty()) { // A load with no underlying object. Depend on all @@ -1008,8 +1017,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, for (MapVector >::iterator I = AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) for (unsigned i = 0, e = I->second.size(); i != e; ++i) - addChainDependency(AAForDep, MFI, SU, I->second[i], - RejectMemNodes); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + I->second[i], RejectMemNodes); PendingLoads.push_back(SU); MayAlias = true; @@ -1032,18 +1041,20 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); if (I != IE) for (unsigned i = 0, e = I->second.size(); i != e; ++i) - addChainDependency(AAForDep, MFI, SU, I->second[i], - RejectMemNodes, 0, true); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, + I->second[i], RejectMemNodes, 0, true); if (ThisMayAlias) AliasMemUses[V].push_back(SU); else NonAliasMemUses[V].push_back(SU); } if (MayAlias) - adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, /*Latency=*/0); + adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, + RejectMemNodes, /*Latency=*/0); // Add dependencies on alias and barrier chains, if needed. if (MayAlias && AliasChain) - addChainDependency(AAForDep, MFI, SU, AliasChain, RejectMemNodes); + addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain, + RejectMemNodes); if (BarrierChain) BarrierChain->addPred(SDep(SU, SDep::Barrier)); } @@ -1211,7 +1222,7 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { else if (SU == &ExitSU) oss << ""; else - SU->getInstr()->print(oss, &TM, /*SkipOpers=*/true); + SU->getInstr()->print(oss, /*SkipOpers=*/true); return oss.str(); } diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 6129401..a1c84c5 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -246,10 +246,11 @@ namespace { SDValue visitSDIVREM(SDNode *N); SDValue visitUDIVREM(SDNode *N); SDValue visitAND(SDNode *N); + SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *LocReference); SDValue visitOR(SDNode *N); + SDValue visitORLike(SDValue N0, SDValue N1, SDNode *LocReference); SDValue visitXOR(SDNode *N); SDValue SimplifyVBinOp(SDNode *N); - SDValue SimplifyVUnaryOp(SDNode *N); SDValue visitSHL(SDNode *N); SDValue visitSRA(SDNode *N); SDValue visitSRL(SDNode *N); @@ -302,6 +303,7 @@ namespace { SDValue visitCONCAT_VECTORS(SDNode *N); SDValue visitEXTRACT_SUBVECTOR(SDNode *N); SDValue visitVECTOR_SHUFFLE(SDNode *N); + SDValue visitSCALAR_TO_VECTOR(SDNode *N); SDValue visitINSERT_SUBVECTOR(SDNode *N); SDValue visitMLOAD(SDNode *N); SDValue visitMSTORE(SDNode *N); @@ -713,6 +715,22 @@ static SDNode *isConstantBuildVectorOrConstantInt(SDValue N) { return nullptr; } +static SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) { + if (isa(N)) + return N.getNode(); + if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) + return N.getNode(); + return nullptr; +} + +static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) { + if (isa(N)) + return N.getNode(); + if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) + return N.getNode(); + return nullptr; +} + // \brief Returns the SDNode if it is a constant splat BuildVector or constant // int. static ConstantSDNode *isConstOrConstSplat(SDValue N) { @@ -1180,11 +1198,6 @@ void DAGCombiner::Run(CombineLevel AtLevel) { LegalOperations = Level >= AfterLegalizeVectorOps; LegalTypes = Level >= AfterLegalizeTypes; - // Early exit if this basic block is in an optnone function. - if (DAG.getMachineFunction().getFunction()->hasFnAttribute( - Attribute::OptimizeNone)) - return; - // Add all the dag nodes to the worklist. for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), E = DAG.allnodes_end(); I != E; ++I) @@ -1369,6 +1382,7 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); + case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N); case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N); case ISD::MLOAD: return visitMLOAD(N); case ISD::MSTORE: return visitMSTORE(N); @@ -2685,6 +2699,109 @@ SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { return SDValue(); } +/// This contains all DAGCombine rules which reduce two values combined by +/// an And operation to a single value. This makes them reusable in the context +/// of visitSELECT(). Rules involving constants are not included as +/// visitSELECT() already handles those cases. +SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, + SDNode *LocReference) { + EVT VT = N1.getValueType(); + + // fold (and x, undef) -> 0 + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getConstant(0, VT); + // fold (and (setcc x), (setcc y)) -> (setcc (and x, y)) + SDValue LL, LR, RL, RR, CC0, CC1; + if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ + ISD::CondCode Op0 = cast(CC0)->get(); + ISD::CondCode Op1 = cast(CC1)->get(); + + if (LR == RR && isa(LR) && Op0 == Op1 && + LL.getValueType().isInteger()) { + // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0) + if (cast(LR)->isNullValue() && Op1 == ISD::SETEQ) { + SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), + LR.getValueType(), LL, RL); + AddToWorklist(ORNode.getNode()); + return DAG.getSetCC(SDLoc(LocReference), VT, ORNode, LR, Op1); + } + // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1) + if (cast(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) { + SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0), + LR.getValueType(), LL, RL); + AddToWorklist(ANDNode.getNode()); + return DAG.getSetCC(SDLoc(LocReference), VT, ANDNode, LR, Op1); + } + // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1) + if (cast(LR)->isAllOnesValue() && Op1 == ISD::SETGT) { + SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), + LR.getValueType(), LL, RL); + AddToWorklist(ORNode.getNode()); + return DAG.getSetCC(SDLoc(LocReference), VT, ORNode, LR, Op1); + } + } + // Simplify (and (setne X, 0), (setne X, -1)) -> (setuge (add X, 1), 2) + if (LL == RL && isa(LR) && isa(RR) && + Op0 == Op1 && LL.getValueType().isInteger() && + Op0 == ISD::SETNE && ((cast(LR)->isNullValue() && + cast(RR)->isAllOnesValue()) || + (cast(LR)->isAllOnesValue() && + cast(RR)->isNullValue()))) { + SDValue ADDNode = DAG.getNode(ISD::ADD, SDLoc(N0), LL.getValueType(), + LL, DAG.getConstant(1, LL.getValueType())); + AddToWorklist(ADDNode.getNode()); + return DAG.getSetCC(SDLoc(LocReference), VT, ADDNode, + DAG.getConstant(2, LL.getValueType()), ISD::SETUGE); + } + // canonicalize equivalent to ll == rl + if (LL == RR && LR == RL) { + Op1 = ISD::getSetCCSwappedOperands(Op1); + std::swap(RL, RR); + } + if (LL == RL && LR == RR) { + bool isInteger = LL.getValueType().isInteger(); + ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger); + if (Result != ISD::SETCC_INVALID && + (!LegalOperations || + (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && + TLI.isOperationLegal(ISD::SETCC, + getSetCCResultType(N0.getSimpleValueType()))))) + return DAG.getSetCC(SDLoc(LocReference), N0.getValueType(), + LL, LR, Result); + } + } + + if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && + VT.getSizeInBits() <= 64) { + if (ConstantSDNode *ADDI = dyn_cast(N0.getOperand(1))) { + APInt ADDC = ADDI->getAPIntValue(); + if (!TLI.isLegalAddImmediate(ADDC.getSExtValue())) { + // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal + // immediate for an add, but it is legal if its top c2 bits are set, + // transform the ADD so the immediate doesn't need to be materialized + // in a register. + if (ConstantSDNode *SRLI = dyn_cast(N1.getOperand(1))) { + APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), + SRLI->getZExtValue()); + if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) { + ADDC |= Mask; + if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) { + SDValue NewAdd = + DAG.getNode(ISD::ADD, SDLoc(N0), VT, + N0.getOperand(0), DAG.getConstant(ADDC, VT)); + CombineTo(N0.getNode(), NewAdd); + // Return N so it doesn't get rechecked! + return SDValue(LocReference, 0); + } + } + } + } + } + } + + return SDValue(); +} + SDValue DAGCombiner::visitAND(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -2716,9 +2833,6 @@ SDValue DAGCombiner::visitAND(SDNode *N) { return N0; } - // fold (and x, undef) -> 0 - if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) - return DAG.getConstant(0, VT); // fold (and c1, c2) -> c1&c2 ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); @@ -2808,9 +2922,13 @@ SDValue DAGCombiner::visitAND(SDNode *N) { SplatBitSize = SplatBitSize * 2) SplatValue |= SplatValue.shl(SplatBitSize); - Constant = APInt::getAllOnesValue(BitWidth); - for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i) - Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth); + // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a + // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value. + if (SplatBitSize % BitWidth == 0) { + Constant = APInt::getAllOnesValue(BitWidth); + for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i) + Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth); + } } } @@ -2863,118 +2981,6 @@ SDValue DAGCombiner::visitAND(SDNode *N) { return SDValue(N, 0); // Return N so it doesn't get rechecked! } } - // fold (and (setcc x), (setcc y)) -> (setcc (and x, y)) - SDValue LL, LR, RL, RR, CC0, CC1; - if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ - ISD::CondCode Op0 = cast(CC0)->get(); - ISD::CondCode Op1 = cast(CC1)->get(); - - if (LR == RR && isa(LR) && Op0 == Op1 && - LL.getValueType().isInteger()) { - // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0) - if (cast(LR)->isNullValue() && Op1 == ISD::SETEQ) { - SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), - LR.getValueType(), LL, RL); - AddToWorklist(ORNode.getNode()); - return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1); - } - // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1) - if (cast(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) { - SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0), - LR.getValueType(), LL, RL); - AddToWorklist(ANDNode.getNode()); - return DAG.getSetCC(SDLoc(N), VT, ANDNode, LR, Op1); - } - // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1) - if (cast(LR)->isAllOnesValue() && Op1 == ISD::SETGT) { - SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), - LR.getValueType(), LL, RL); - AddToWorklist(ORNode.getNode()); - return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1); - } - } - // Simplify (and (setne X, 0), (setne X, -1)) -> (setuge (add X, 1), 2) - if (LL == RL && isa(LR) && isa(RR) && - Op0 == Op1 && LL.getValueType().isInteger() && - Op0 == ISD::SETNE && ((cast(LR)->isNullValue() && - cast(RR)->isAllOnesValue()) || - (cast(LR)->isAllOnesValue() && - cast(RR)->isNullValue()))) { - SDValue ADDNode = DAG.getNode(ISD::ADD, SDLoc(N0), LL.getValueType(), - LL, DAG.getConstant(1, LL.getValueType())); - AddToWorklist(ADDNode.getNode()); - return DAG.getSetCC(SDLoc(N), VT, ADDNode, - DAG.getConstant(2, LL.getValueType()), ISD::SETUGE); - } - // canonicalize equivalent to ll == rl - if (LL == RR && LR == RL) { - Op1 = ISD::getSetCCSwappedOperands(Op1); - std::swap(RL, RR); - } - if (LL == RL && LR == RR) { - bool isInteger = LL.getValueType().isInteger(); - ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger); - if (Result != ISD::SETCC_INVALID && - (!LegalOperations || - (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && - TLI.isOperationLegal(ISD::SETCC, - getSetCCResultType(N0.getSimpleValueType()))))) - return DAG.getSetCC(SDLoc(N), N0.getValueType(), - LL, LR, Result); - } - } - - // Simplify: (and (op x...), (op y...)) -> (op (and x, y)) - if (N0.getOpcode() == N1.getOpcode()) { - SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); - if (Tmp.getNode()) return Tmp; - } - - // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) - // fold (and (sra)) -> (and (srl)) when possible. - if (!VT.isVector() && - SimplifyDemandedBits(SDValue(N, 0))) - return SDValue(N, 0); - - // fold (zext_inreg (extload x)) -> (zextload x) - if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { - LoadSDNode *LN0 = cast(N0); - EVT MemVT = LN0->getMemoryVT(); - // If we zero all the possible extended bits, then we can turn this into - // a zextload if we are running before legalize or the operation is legal. - unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits(); - if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, - BitWidth - MemVT.getScalarType().getSizeInBits())) && - ((!LegalOperations && !LN0->isVolatile()) || - TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { - SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, - LN0->getChain(), LN0->getBasePtr(), - MemVT, LN0->getMemOperand()); - AddToWorklist(N); - CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); - return SDValue(N, 0); // Return N so it doesn't get rechecked! - } - } - // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use - if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && - N0.hasOneUse()) { - LoadSDNode *LN0 = cast(N0); - EVT MemVT = LN0->getMemoryVT(); - // If we zero all the possible extended bits, then we can turn this into - // a zextload if we are running before legalize or the operation is legal. - unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits(); - if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, - BitWidth - MemVT.getScalarType().getSizeInBits())) && - ((!LegalOperations && !LN0->isVolatile()) || - TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { - SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, - LN0->getChain(), LN0->getBasePtr(), - MemVT, LN0->getMemOperand()); - AddToWorklist(N); - CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); - return SDValue(N, 0); // Return N so it doesn't get rechecked! - } - } // fold (and (load x), 255) -> (zextload x, i8) // fold (and (extload x, i16), 255) -> (zextload x, i8) @@ -3046,33 +3052,60 @@ SDValue DAGCombiner::visitAND(SDNode *N) { } } - if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && - VT.getSizeInBits() <= 64) { - if (ConstantSDNode *ADDI = dyn_cast(N0.getOperand(1))) { - APInt ADDC = ADDI->getAPIntValue(); - if (!TLI.isLegalAddImmediate(ADDC.getSExtValue())) { - // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal - // immediate for an add, but it is legal if its top c2 bits are set, - // transform the ADD so the immediate doesn't need to be materialized - // in a register. - if (ConstantSDNode *SRLI = dyn_cast(N1.getOperand(1))) { - APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), - SRLI->getZExtValue()); - if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) { - ADDC |= Mask; - if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) { - SDValue NewAdd = - DAG.getNode(ISD::ADD, SDLoc(N0), VT, - N0.getOperand(0), DAG.getConstant(ADDC, VT)); - CombineTo(N0.getNode(), NewAdd); - return SDValue(N, 0); // Return N so it doesn't get rechecked! - } - } - } - } - } + if (SDValue Combined = visitANDLike(N0, N1, N)) + return Combined; + + // Simplify: (and (op x...), (op y...)) -> (op (and x, y)) + if (N0.getOpcode() == N1.getOpcode()) { + SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); + if (Tmp.getNode()) return Tmp; } + // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) + // fold (and (sra)) -> (and (srl)) when possible. + if (!VT.isVector() && + SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); + + // fold (zext_inreg (extload x)) -> (zextload x) + if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { + LoadSDNode *LN0 = cast(N0); + EVT MemVT = LN0->getMemoryVT(); + // If we zero all the possible extended bits, then we can turn this into + // a zextload if we are running before legalize or the operation is legal. + unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits(); + if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, + BitWidth - MemVT.getScalarType().getSizeInBits())) && + ((!LegalOperations && !LN0->isVolatile()) || + TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { + SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, + LN0->getChain(), LN0->getBasePtr(), + MemVT, LN0->getMemOperand()); + AddToWorklist(N); + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! + } + } + // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use + if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && + N0.hasOneUse()) { + LoadSDNode *LN0 = cast(N0); + EVT MemVT = LN0->getMemoryVT(); + // If we zero all the possible extended bits, then we can turn this into + // a zextload if we are running before legalize or the operation is legal. + unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits(); + if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, + BitWidth - MemVT.getScalarType().getSizeInBits())) && + ((!LegalOperations && !LN0->isVolatile()) || + TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { + SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, + LN0->getChain(), LN0->getBasePtr(), + MemVT, LN0->getMemOperand()); + AddToWorklist(N); + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! + } + } // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const) if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), @@ -3287,55 +3320,147 @@ SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) { SDValue N01 = N0.getOperand(1); SDNode *Parts[4] = {}; - if (N1.getOpcode() == ISD::OR && - N00.getNumOperands() == 2 && N01.getNumOperands() == 2) { - // (or (or (and), (and)), (or (and), (and))) - SDValue N000 = N00.getOperand(0); - if (!isBSwapHWordElement(N000, Parts)) - return SDValue(); + if (N1.getOpcode() == ISD::OR && + N00.getNumOperands() == 2 && N01.getNumOperands() == 2) { + // (or (or (and), (and)), (or (and), (and))) + SDValue N000 = N00.getOperand(0); + if (!isBSwapHWordElement(N000, Parts)) + return SDValue(); + + SDValue N001 = N00.getOperand(1); + if (!isBSwapHWordElement(N001, Parts)) + return SDValue(); + SDValue N010 = N01.getOperand(0); + if (!isBSwapHWordElement(N010, Parts)) + return SDValue(); + SDValue N011 = N01.getOperand(1); + if (!isBSwapHWordElement(N011, Parts)) + return SDValue(); + } else { + // (or (or (or (and), (and)), (and)), (and)) + if (!isBSwapHWordElement(N1, Parts)) + return SDValue(); + if (!isBSwapHWordElement(N01, Parts)) + return SDValue(); + if (N00.getOpcode() != ISD::OR) + return SDValue(); + SDValue N000 = N00.getOperand(0); + if (!isBSwapHWordElement(N000, Parts)) + return SDValue(); + SDValue N001 = N00.getOperand(1); + if (!isBSwapHWordElement(N001, Parts)) + return SDValue(); + } + + // Make sure the parts are all coming from the same node. + if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3]) + return SDValue(); + + SDValue BSwap = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, + SDValue(Parts[0],0)); + + // Result of the bswap should be rotated by 16. If it's not legal, then + // do (x << 16) | (x >> 16). + SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT)); + if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) + return DAG.getNode(ISD::ROTL, SDLoc(N), VT, BSwap, ShAmt); + if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) + return DAG.getNode(ISD::ROTR, SDLoc(N), VT, BSwap, ShAmt); + return DAG.getNode(ISD::OR, SDLoc(N), VT, + DAG.getNode(ISD::SHL, SDLoc(N), VT, BSwap, ShAmt), + DAG.getNode(ISD::SRL, SDLoc(N), VT, BSwap, ShAmt)); +} + +/// This contains all DAGCombine rules which reduce two values combined by +/// an Or operation to a single value \see visitANDLike(). +SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *LocReference) { + EVT VT = N1.getValueType(); + // fold (or x, undef) -> -1 + if (!LegalOperations && + (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) { + EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT; + return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); + } + // fold (or (setcc x), (setcc y)) -> (setcc (or x, y)) + SDValue LL, LR, RL, RR, CC0, CC1; + if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ + ISD::CondCode Op0 = cast(CC0)->get(); + ISD::CondCode Op1 = cast(CC1)->get(); + + if (LR == RR && isa(LR) && Op0 == Op1 && + LL.getValueType().isInteger()) { + // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0) + // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0) + if (cast(LR)->isNullValue() && + (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) { + SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR), + LR.getValueType(), LL, RL); + AddToWorklist(ORNode.getNode()); + return DAG.getSetCC(SDLoc(LocReference), VT, ORNode, LR, Op1); + } + // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1) + // fold (or (setgt X, -1), (setgt Y -1)) -> (setgt (and X, Y), -1) + if (cast(LR)->isAllOnesValue() && + (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) { + SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR), + LR.getValueType(), LL, RL); + AddToWorklist(ANDNode.getNode()); + return DAG.getSetCC(SDLoc(LocReference), VT, ANDNode, LR, Op1); + } + } + // canonicalize equivalent to ll == rl + if (LL == RR && LR == RL) { + Op1 = ISD::getSetCCSwappedOperands(Op1); + std::swap(RL, RR); + } + if (LL == RL && LR == RR) { + bool isInteger = LL.getValueType().isInteger(); + ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger); + if (Result != ISD::SETCC_INVALID && + (!LegalOperations || + (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && + TLI.isOperationLegal(ISD::SETCC, + getSetCCResultType(N0.getValueType()))))) + return DAG.getSetCC(SDLoc(LocReference), N0.getValueType(), + LL, LR, Result); + } + } + + // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible. + if (N0.getOpcode() == ISD::AND && + N1.getOpcode() == ISD::AND && + N0.getOperand(1).getOpcode() == ISD::Constant && + N1.getOperand(1).getOpcode() == ISD::Constant && + // Don't increase # computations. + (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { + // We can only do this xform if we know that bits from X that are set in C2 + // but not in C1 are already zero. Likewise for Y. + const APInt &LHSMask = + cast(N0.getOperand(1))->getAPIntValue(); + const APInt &RHSMask = + cast(N1.getOperand(1))->getAPIntValue(); - SDValue N001 = N00.getOperand(1); - if (!isBSwapHWordElement(N001, Parts)) - return SDValue(); - SDValue N010 = N01.getOperand(0); - if (!isBSwapHWordElement(N010, Parts)) - return SDValue(); - SDValue N011 = N01.getOperand(1); - if (!isBSwapHWordElement(N011, Parts)) - return SDValue(); - } else { - // (or (or (or (and), (and)), (and)), (and)) - if (!isBSwapHWordElement(N1, Parts)) - return SDValue(); - if (!isBSwapHWordElement(N01, Parts)) - return SDValue(); - if (N00.getOpcode() != ISD::OR) - return SDValue(); - SDValue N000 = N00.getOperand(0); - if (!isBSwapHWordElement(N000, Parts)) - return SDValue(); - SDValue N001 = N00.getOperand(1); - if (!isBSwapHWordElement(N001, Parts)) - return SDValue(); + if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && + DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) { + SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, + N0.getOperand(0), N1.getOperand(0)); + return DAG.getNode(ISD::AND, SDLoc(LocReference), VT, X, + DAG.getConstant(LHSMask | RHSMask, VT)); + } } - // Make sure the parts are all coming from the same node. - if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3]) - return SDValue(); - - SDValue BSwap = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, - SDValue(Parts[0],0)); + // (or (and X, M), (and X, N)) -> (and X, (or M, N)) + if (N0.getOpcode() == ISD::AND && + N1.getOpcode() == ISD::AND && + N0.getOperand(0) == N1.getOperand(0) && + // Don't increase # computations. + (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { + SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, + N0.getOperand(1), N1.getOperand(1)); + return DAG.getNode(ISD::AND, SDLoc(LocReference), VT, N0.getOperand(0), X); + } - // Result of the bswap should be rotated by 16. If it's not legal, then - // do (x << 16) | (x >> 16). - SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT)); - if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) - return DAG.getNode(ISD::ROTL, SDLoc(N), VT, BSwap, ShAmt); - if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) - return DAG.getNode(ISD::ROTR, SDLoc(N), VT, BSwap, ShAmt); - return DAG.getNode(ISD::OR, SDLoc(N), VT, - DAG.getNode(ISD::SHL, SDLoc(N), VT, BSwap, ShAmt), - DAG.getNode(ISD::SRL, SDLoc(N), VT, BSwap, ShAmt)); + return SDValue(); } SDValue DAGCombiner::visitOR(SDNode *N) { @@ -3425,12 +3550,6 @@ SDValue DAGCombiner::visitOR(SDNode *N) { } } - // fold (or x, undef) -> -1 - if (!LegalOperations && - (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) { - EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT; - return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); - } // fold (or c1, c2) -> c1|c2 ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); @@ -3449,6 +3568,9 @@ SDValue DAGCombiner::visitOR(SDNode *N) { if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue())) return N1; + if (SDValue Combined = visitORLike(N0, N1, N)) + return Combined; + // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16) SDValue BSwap = MatchBSwapHWord(N, N0, N1); if (BSwap.getNode()) @@ -3474,91 +3596,12 @@ SDValue DAGCombiner::visitOR(SDNode *N) { return SDValue(); } } - // fold (or (setcc x), (setcc y)) -> (setcc (or x, y)) - SDValue LL, LR, RL, RR, CC0, CC1; - if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ - ISD::CondCode Op0 = cast(CC0)->get(); - ISD::CondCode Op1 = cast(CC1)->get(); - - if (LR == RR && isa(LR) && Op0 == Op1 && - LL.getValueType().isInteger()) { - // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0) - // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0) - if (cast(LR)->isNullValue() && - (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) { - SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR), - LR.getValueType(), LL, RL); - AddToWorklist(ORNode.getNode()); - return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1); - } - // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1) - // fold (or (setgt X, -1), (setgt Y -1)) -> (setgt (and X, Y), -1) - if (cast(LR)->isAllOnesValue() && - (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) { - SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR), - LR.getValueType(), LL, RL); - AddToWorklist(ANDNode.getNode()); - return DAG.getSetCC(SDLoc(N), VT, ANDNode, LR, Op1); - } - } - // canonicalize equivalent to ll == rl - if (LL == RR && LR == RL) { - Op1 = ISD::getSetCCSwappedOperands(Op1); - std::swap(RL, RR); - } - if (LL == RL && LR == RR) { - bool isInteger = LL.getValueType().isInteger(); - ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger); - if (Result != ISD::SETCC_INVALID && - (!LegalOperations || - (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && - TLI.isOperationLegal(ISD::SETCC, - getSetCCResultType(N0.getValueType()))))) - return DAG.getSetCC(SDLoc(N), N0.getValueType(), - LL, LR, Result); - } - } - // Simplify: (or (op x...), (op y...)) -> (op (or x, y)) if (N0.getOpcode() == N1.getOpcode()) { SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); if (Tmp.getNode()) return Tmp; } - // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible. - if (N0.getOpcode() == ISD::AND && - N1.getOpcode() == ISD::AND && - N0.getOperand(1).getOpcode() == ISD::Constant && - N1.getOperand(1).getOpcode() == ISD::Constant && - // Don't increase # computations. - (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { - // We can only do this xform if we know that bits from X that are set in C2 - // but not in C1 are already zero. Likewise for Y. - const APInt &LHSMask = - cast(N0.getOperand(1))->getAPIntValue(); - const APInt &RHSMask = - cast(N1.getOperand(1))->getAPIntValue(); - - if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && - DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) { - SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, - N0.getOperand(0), N1.getOperand(0)); - return DAG.getNode(ISD::AND, SDLoc(N), VT, X, - DAG.getConstant(LHSMask | RHSMask, VT)); - } - } - - // (or (and X, M), (and X, N)) -> (and X, (or M, N)) - if (N0.getOpcode() == ISD::AND && - N1.getOpcode() == ISD::AND && - N0.getOperand(0) == N1.getOperand(0) && - // Don't increase # computations. - (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { - SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, - N0.getOperand(1), N1.getOperand(1)); - return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0), X); - } - // See if this is some rotate idiom. if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N))) return SDValue(Rot, 0); @@ -3947,6 +3990,32 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { if (N0 == N1) return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes); + // fold (xor (shl 1, x), -1) -> (rotl ~1, x) + // Here is a concrete example of this equivalence: + // i16 x == 14 + // i16 shl == 1 << 14 == 16384 == 0b0100000000000000 + // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111 + // + // => + // + // i16 ~1 == 0b1111111111111110 + // i16 rol(~1, 14) == 0b1011111111111111 + // + // Some additional tips to help conceptualize this transform: + // - Try to see the operation as placing a single zero in a value of all ones. + // - There exists no value for x which would allow the result to contain zero. + // - Values of x larger than the bitwidth are undefined and do not require a + // consistent result. + // - Pushing the zero left requires shifting one bits in from the right. + // A rotate left of ~1 is a nice way of achieving the desired result. + if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) + if (auto *N1C = dyn_cast(N1.getNode())) + if (N0.getOpcode() == ISD::SHL) + if (auto *ShlLHS = dyn_cast(N0.getOperand(0))) + if (N1C->isAllOnesValue() && ShlLHS->isOne()) + return DAG.getNode(ISD::ROTL, SDLoc(N), VT, DAG.getConstant(~1, VT), + N0.getOperand(1)); + // Simplify: xor (op x...), (op y...) -> (op (xor x, y)) if (N0.getOpcode() == N1.getOpcode()) { SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); @@ -4792,6 +4861,69 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { return SimplifySelect(SDLoc(N), N0, N1, N2); } + if (VT0 == MVT::i1) { + if (TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) { + // select (and Cond0, Cond1), X, Y + // -> select Cond0, (select Cond1, X, Y), Y + if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { + SDValue Cond0 = N0->getOperand(0); + SDValue Cond1 = N0->getOperand(1); + SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N), + N1.getValueType(), Cond1, N1, N2); + return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, + InnerSelect, N2); + } + // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y) + if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { + SDValue Cond0 = N0->getOperand(0); + SDValue Cond1 = N0->getOperand(1); + SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N), + N1.getValueType(), Cond1, N1, N2); + return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, N1, + InnerSelect); + } + } + + // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y + if (N1->getOpcode() == ISD::SELECT) { + SDValue N1_0 = N1->getOperand(0); + SDValue N1_1 = N1->getOperand(1); + SDValue N1_2 = N1->getOperand(2); + if (N1_2 == N2) { + // Create the actual and node if we can generate good code for it. + if (!TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) { + SDValue And = DAG.getNode(ISD::AND, SDLoc(N), N0.getValueType(), + N0, N1_0); + return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), And, + N1_1, N2); + } + // Otherwise see if we can optimize the "and" to a better pattern. + if (SDValue Combined = visitANDLike(N0, N1_0, N)) + return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined, + N1_1, N2); + } + } + // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y + if (N2->getOpcode() == ISD::SELECT) { + SDValue N2_0 = N2->getOperand(0); + SDValue N2_1 = N2->getOperand(1); + SDValue N2_2 = N2->getOperand(2); + if (N2_1 == N1) { + // Create the actual or node if we can generate good code for it. + if (!TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) { + SDValue Or = DAG.getNode(ISD::OR, SDLoc(N), N0.getValueType(), + N0, N2_0); + return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Or, + N1, N2_2); + } + // Otherwise see if we can optimize to a better pattern. + if (SDValue Combined = visitORLike(N0, N2_0, N)) + return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined, + N1, N2_2); + } + } + } + return SDValue(); } @@ -6440,7 +6572,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { if (N0.getValueType() == N->getValueType(0)) return N0; // fold (truncate c1) -> c1 - if (isa(N0)) + if (isConstantIntBuildVectorOrConstantInt(N0)) return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); // fold (truncate (truncate x)) -> (truncate x) if (N0.getOpcode() == ISD::TRUNCATE) @@ -7453,14 +7585,23 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { // Fold scalars or any vector constants (not just splats). // This fold is done in general by InstCombine, but extra fmul insts // may have been generated during lowering. + SDValue N00 = N0.getOperand(0); SDValue N01 = N0.getOperand(1); auto *BV1 = dyn_cast(N1); + auto *BV00 = dyn_cast(N00); auto *BV01 = dyn_cast(N01); - if ((N1CFP && isConstOrConstSplatFP(N01)) || - (BV1 && BV01 && BV1->isConstant() && BV01->isConstant())) { - SDLoc SL(N); - SDValue MulConsts = DAG.getNode(ISD::FMUL, SL, VT, N01, N1); - return DAG.getNode(ISD::FMUL, SL, VT, N0.getOperand(0), MulConsts); + + // Check 1: Make sure that the first operand of the inner multiply is NOT + // a constant. Otherwise, we may induce infinite looping. + if (!(isConstOrConstSplatFP(N00) || (BV00 && BV00->isConstant()))) { + // Check 2: Make sure that the second operand of the inner multiply and + // the second operand of the outer multiply are constants. + if ((N1CFP && isConstOrConstSplatFP(N01)) || + (BV1 && BV01 && BV1->isConstant() && BV01->isConstant())) { + SDLoc SL(N); + SDValue MulConsts = DAG.getNode(ISD::FMUL, SL, VT, N01, N1); + return DAG.getNode(ISD::FMUL, SL, VT, N00, MulConsts); + } } } @@ -7821,8 +7962,7 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) { EVT OpVT = N0.getValueType(); // fold (sint_to_fp c1) -> c1fp - ConstantSDNode *N0C = dyn_cast(N0); - if (N0C && + if (isConstantIntBuildVectorOrConstantInt(N0) && // ...but only if the target supports immediate floating-point values (!LegalOperations || TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) @@ -7874,8 +8014,7 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) { EVT OpVT = N0.getValueType(); // fold (uint_to_fp c1) -> c1fp - ConstantSDNode *N0C = dyn_cast(N0); - if (N0C && + if (isConstantIntBuildVectorOrConstantInt(N0) && // ...but only if the target supports immediate floating-point values (!LegalOperations || TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) @@ -8033,7 +8172,6 @@ SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) { SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) { SDValue N0 = N->getOperand(0); - ConstantFPSDNode *N0CFP = dyn_cast(N0); EVT VT = N->getValueType(0); // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded. @@ -8042,7 +8180,7 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) { return SDValue(); // fold (fp_extend c1fp) -> c1fp - if (N0CFP) + if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the @@ -8117,14 +8255,9 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVUnaryOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } - // Constant fold FNEG. - if (isa(N0)) - return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N->getOperand(0)); + if (isConstantFPBuildVectorOrConstantFP(N0)) + return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(), &DAG.getTarget().Options)) @@ -8219,13 +8352,8 @@ SDValue DAGCombiner::visitFABS(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVUnaryOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } - // fold (fabs c1) -> fabs(c1) - if (isa(N0)) + if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); // fold (fabs (fabs x)) -> (fabs x) @@ -8941,7 +9069,8 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) { LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), LD->isInvariant(), Align, LD->getAAInfo()); - return CombineTo(N, NewLoad, SDValue(NewLoad.getNode(), 1), true); + if (NewLoad.getNode() != N) + return CombineTo(N, NewLoad, SDValue(NewLoad.getNode(), 1), true); } } } @@ -9106,9 +9235,6 @@ struct LoadedSlice { unsigned Shift = 0, SelectionDAG *DAG = nullptr) : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {} - LoadedSlice(const LoadedSlice &LS) - : Inst(LS.Inst), Origin(LS.Origin), Shift(LS.Shift), DAG(LS.DAG) {} - /// \brief Get the bits used in a chunk of bits \p BitWidth large. /// \return Result is \p BitWidth and has used bits set to 1 and /// not used bits set to 0. @@ -9855,6 +9981,7 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { return SDValue(); } +namespace { /// Helper struct to parse and store a memory address as base + index + offset. /// We ignore sign extensions when it is safe to do so. /// The following two expressions are not equivalent. To differentiate we need @@ -9942,6 +10069,7 @@ struct BaseIndexOffset { return BaseIndexOffset(Base, Index, Off, IsIndexSignExt); } }; +} // namespace bool DAGCombiner::MergeStoresOfConstantsOrVecElts( SmallVectorImpl &StoreNodes, EVT MemVT, @@ -10575,11 +10703,15 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { // Try to infer better alignment information than the store already has. if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) { if (unsigned Align = DAG.InferPtrAlignment(Ptr)) { - if (Align > ST->getAlignment()) - return DAG.getTruncStore(Chain, SDLoc(N), Value, + if (Align > ST->getAlignment()) { + SDValue NewStore = + DAG.getTruncStore(Chain, SDLoc(N), Value, Ptr, ST->getPointerInfo(), ST->getMemoryVT(), ST->isVolatile(), ST->isNonTemporal(), Align, ST->getAAInfo()); + if (NewStore.getNode() != N) + return CombineTo(ST, NewStore, true); + } } } @@ -11226,12 +11358,10 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { if (ISD::allOperandsUndef(N)) return DAG.getUNDEF(VT); - SDValue V = reduceBuildVecExtToExtBuildVec(N); - if (V.getNode()) + if (SDValue V = reduceBuildVecExtToExtBuildVec(N)) return V; - V = reduceBuildVecConvertToConvertBuildVec(N); - if (V.getNode()) + if (SDValue V = reduceBuildVecConvertToConvertBuildVec(N)) return V; // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT @@ -11352,7 +11482,9 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { } else if (VecInT.getSizeInBits() == VT.getSizeInBits() * 2) { // If the input vector is too large, try to split it. // We don't support having two input vectors that are too large. - if (VecIn2.getNode()) + // If the zero vector was used, we can not split the vector, + // since we'd need 3 inputs. + if (UsesZeroVector || VecIn2.getNode()) return SDValue(); if (!TLI.isExtractSubvectorCheap(VT, VT.getVectorNumElements())) @@ -11364,7 +11496,6 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { DAG.getConstant(VT.getVectorNumElements(), TLI.getVectorIdxTy())); VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1, DAG.getConstant(0, TLI.getVectorIdxTy())); - UsesZeroVector = false; } else return SDValue(); } @@ -11465,14 +11596,12 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { unsigned NumElts = OpVT.getVectorNumElements(); if (ISD::UNDEF == Op.getOpcode()) - for (unsigned i = 0; i != NumElts; ++i) - Opnds.push_back(DAG.getUNDEF(MinVT)); + Opnds.append(NumElts, DAG.getUNDEF(MinVT)); if (ISD::BUILD_VECTOR == Op.getOpcode()) { if (SVT.isFloatingPoint()) { assert(SVT == OpVT.getScalarType() && "Concat vector type mismatch"); - for (unsigned i = 0; i != NumElts; ++i) - Opnds.push_back(Op.getOperand(i)); + Opnds.append(Op->op_begin(), Op->op_begin() + NumElts); } else { for (unsigned i = 0; i != NumElts; ++i) Opnds.push_back( @@ -11850,7 +11979,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { // We may have jumped through bitcasts, so the type of the // BUILD_VECTOR may not match the type of the shuffle. if (V->getValueType(0) != VT) - NewBV = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, NewBV); + NewBV = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, NewBV); return NewBV; } } @@ -11872,6 +12001,81 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { return V; } + // If this shuffle only has a single input that is a bitcasted shuffle, + // attempt to merge the 2 shuffles and suitably bitcast the inputs/output + // back to their original types. + if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && + N1.getOpcode() == ISD::UNDEF && Level < AfterLegalizeVectorOps && + TLI.isTypeLegal(VT)) { + + // Peek through the bitcast only if there is one user. + SDValue BC0 = N0; + while (BC0.getOpcode() == ISD::BITCAST) { + if (!BC0.hasOneUse()) + break; + BC0 = BC0.getOperand(0); + } + + auto ScaleShuffleMask = [](ArrayRef Mask, int Scale) { + if (Scale == 1) + return SmallVector(Mask.begin(), Mask.end()); + + SmallVector NewMask; + for (int M : Mask) + for (int s = 0; s != Scale; ++s) + NewMask.push_back(M < 0 ? -1 : Scale * M + s); + return NewMask; + }; + + if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) { + EVT SVT = VT.getScalarType(); + EVT InnerVT = BC0->getValueType(0); + EVT InnerSVT = InnerVT.getScalarType(); + + // Determine which shuffle works with the smaller scalar type. + EVT ScaleVT = SVT.bitsLT(InnerSVT) ? VT : InnerVT; + EVT ScaleSVT = ScaleVT.getScalarType(); + + if (TLI.isTypeLegal(ScaleVT) && + 0 == (InnerSVT.getSizeInBits() % ScaleSVT.getSizeInBits()) && + 0 == (SVT.getSizeInBits() % ScaleSVT.getSizeInBits())) { + + int InnerScale = InnerSVT.getSizeInBits() / ScaleSVT.getSizeInBits(); + int OuterScale = SVT.getSizeInBits() / ScaleSVT.getSizeInBits(); + + // Scale the shuffle masks to the smaller scalar type. + ShuffleVectorSDNode *InnerSVN = cast(BC0); + SmallVector InnerMask = + ScaleShuffleMask(InnerSVN->getMask(), InnerScale); + SmallVector OuterMask = + ScaleShuffleMask(SVN->getMask(), OuterScale); + + // Merge the shuffle masks. + SmallVector NewMask; + for (int M : OuterMask) + NewMask.push_back(M < 0 ? -1 : InnerMask[M]); + + // Test for shuffle mask legality over both commutations. + SDValue SV0 = BC0->getOperand(0); + SDValue SV1 = BC0->getOperand(1); + bool LegalMask = TLI.isShuffleMaskLegal(NewMask, ScaleVT); + if (!LegalMask) { + std::swap(SV0, SV1); + ShuffleVectorSDNode::commuteMask(NewMask); + LegalMask = TLI.isShuffleMaskLegal(NewMask, ScaleVT); + } + + if (LegalMask) { + SV0 = DAG.getNode(ISD::BITCAST, SDLoc(N), ScaleVT, SV0); + SV1 = DAG.getNode(ISD::BITCAST, SDLoc(N), ScaleVT, SV1); + return DAG.getNode( + ISD::BITCAST, SDLoc(N), VT, + DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask)); + } + } + } + } + // Canonicalize shuffles according to rules: // shuffle(A, shuffle(A, B)) -> shuffle(shuffle(A,B), A) // shuffle(B, shuffle(A, B)) -> shuffle(shuffle(A,B), B) @@ -11981,16 +12185,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { // Avoid introducing shuffles with illegal mask. if (!TLI.isShuffleMaskLegal(Mask, VT)) { - // Compute the commuted shuffle mask and test again. - for (unsigned i = 0; i != NumElts; ++i) { - int idx = Mask[i]; - if (idx < 0) - continue; - else if (idx < (int)NumElts) - Mask[i] = idx + NumElts; - else - Mask[i] = idx - NumElts; - } + ShuffleVectorSDNode::commuteMask(Mask); if (!TLI.isShuffleMaskLegal(Mask, VT)) return SDValue(); @@ -12010,6 +12205,34 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitSCALAR_TO_VECTOR(SDNode *N) { + SDValue InVal = N->getOperand(0); + EVT VT = N->getValueType(0); + + // Replace a SCALAR_TO_VECTOR(EXTRACT_VECTOR_ELT(V,C0)) pattern + // with a VECTOR_SHUFFLE. + if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { + SDValue InVec = InVal->getOperand(0); + SDValue EltNo = InVal->getOperand(1); + + // FIXME: We could support implicit truncation if the shuffle can be + // scaled to a smaller vector scalar type. + ConstantSDNode *C0 = dyn_cast(EltNo); + if (C0 && VT == InVec.getValueType() && + VT.getScalarType() == InVal.getValueType()) { + SmallVector NewMask(VT.getVectorNumElements(), -1); + int Elt = C0->getZExtValue(); + NewMask[0] = Elt; + + if (TLI.isShuffleMaskLegal(NewMask, VT)) + return DAG.getVectorShuffle(VT, SDLoc(N), InVec, DAG.getUNDEF(VT), + NewMask); + } + } + + return SDValue(); +} + SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N2 = N->getOperand(2); @@ -12043,44 +12266,51 @@ SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) { /// vector_shuffle V, Zero, <0, 4, 2, 4> SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { EVT VT = N->getValueType(0); - SDLoc dl(N); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); - if (N->getOpcode() == ISD::AND) { - if (RHS.getOpcode() == ISD::BITCAST) - RHS = RHS.getOperand(0); - if (RHS.getOpcode() == ISD::BUILD_VECTOR) { - SmallVector Indices; - unsigned NumElts = RHS.getNumOperands(); - for (unsigned i = 0; i != NumElts; ++i) { - SDValue Elt = RHS.getOperand(i); - if (!isa(Elt)) - return SDValue(); + SDLoc dl(N); - if (cast(Elt)->isAllOnesValue()) - Indices.push_back(i); - else if (cast(Elt)->isNullValue()) - Indices.push_back(NumElts+i); - else - return SDValue(); - } + // Make sure we're not running after operation legalization where it + // may have custom lowered the vector shuffles. + if (LegalOperations) + return SDValue(); + + if (N->getOpcode() != ISD::AND) + return SDValue(); + + if (RHS.getOpcode() == ISD::BITCAST) + RHS = RHS.getOperand(0); - // Let's see if the target supports this vector_shuffle and make sure - // we're not running after operation legalization where it may have - // custom lowered the vector shuffles. - EVT RVT = RHS.getValueType(); - if (LegalOperations || !TLI.isVectorClearMaskLegal(Indices, RVT)) + if (RHS.getOpcode() == ISD::BUILD_VECTOR) { + SmallVector Indices; + unsigned NumElts = RHS.getNumOperands(); + + for (unsigned i = 0; i != NumElts; ++i) { + SDValue Elt = RHS.getOperand(i); + if (!isa(Elt)) return SDValue(); - // Return the new VECTOR_SHUFFLE node. - EVT EltVT = RVT.getVectorElementType(); - SmallVector ZeroOps(RVT.getVectorNumElements(), - DAG.getConstant(0, EltVT)); - SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), RVT, ZeroOps); - LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS); - SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]); - return DAG.getNode(ISD::BITCAST, dl, VT, Shuf); + if (cast(Elt)->isAllOnesValue()) + Indices.push_back(i); + else if (cast(Elt)->isNullValue()) + Indices.push_back(NumElts+i); + else + return SDValue(); } + + // Let's see if the target supports this vector_shuffle. + EVT RVT = RHS.getValueType(); + if (!TLI.isVectorClearMaskLegal(Indices, RVT)) + return SDValue(); + + // Return the new VECTOR_SHUFFLE node. + EVT EltVT = RVT.getVectorElementType(); + SmallVector ZeroOps(RVT.getVectorNumElements(), + DAG.getConstant(0, EltVT)); + SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), RVT, ZeroOps); + LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS); + SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]); + return DAG.getNode(ISD::BITCAST, dl, VT, Shuf); } return SDValue(); @@ -12093,8 +12323,9 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); - SDValue Shuffle = XformToShuffleWithZero(N); - if (Shuffle.getNode()) return Shuffle; + + if (SDValue Shuffle = XformToShuffleWithZero(N)) + return Shuffle; // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold // this operation. @@ -12172,38 +12403,6 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { return SDValue(); } -/// Visit a binary vector operation, like FABS/FNEG. -SDValue DAGCombiner::SimplifyVUnaryOp(SDNode *N) { - assert(N->getValueType(0).isVector() && - "SimplifyVUnaryOp only works on vectors!"); - - SDValue N0 = N->getOperand(0); - - if (N0.getOpcode() != ISD::BUILD_VECTOR) - return SDValue(); - - // Operand is a BUILD_VECTOR node, see if we can constant fold it. - SmallVector Ops; - for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) { - SDValue Op = N0.getOperand(i); - if (Op.getOpcode() != ISD::UNDEF && - Op.getOpcode() != ISD::ConstantFP) - break; - EVT EltVT = Op.getValueType(); - SDValue FoldOp = DAG.getNode(N->getOpcode(), SDLoc(N0), EltVT, Op); - if (FoldOp.getOpcode() != ISD::UNDEF && - FoldOp.getOpcode() != ISD::ConstantFP) - break; - Ops.push_back(FoldOp); - AddToWorklist(FoldOp.getNode()); - } - - if (Ops.size() != N0.getNumOperands()) - return SDValue(); - - return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N0.getValueType(), Ops); -} - SDValue DAGCombiner::SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2){ assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!"); diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 1df4a1d..223a149 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -62,6 +62,7 @@ #include "llvm/IR/Operator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" @@ -497,7 +498,7 @@ bool FastISel::selectGetElementPtr(const User *I) { OI != E; ++OI) { const Value *Idx = *OI; if (auto *StTy = dyn_cast(Ty)) { - unsigned Field = cast(Idx)->getZExtValue(); + uint64_t Field = cast(Idx)->getZExtValue(); if (Field) { // N = N + Offset TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field); @@ -518,8 +519,8 @@ bool FastISel::selectGetElementPtr(const User *I) { if (CI->isZero()) continue; // N = N + Offset - TotalOffs += - DL.getTypeAllocSize(Ty) * cast(CI)->getSExtValue(); + uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue(); + TotalOffs += DL.getTypeAllocSize(Ty) * IdxN; if (TotalOffs >= MaxOffs) { N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT); if (!N) // Unhandled operand. Halt "fast" selection and bail. @@ -801,7 +802,8 @@ bool FastISel::selectPatchpoint(const CallInst *I) { return false; // Push the register mask info. - Ops.push_back(MachineOperand::CreateRegMask(TRI.getCallPreservedMask(CC))); + Ops.push_back(MachineOperand::CreateRegMask( + TRI.getCallPreservedMask(*FuncInfo.MF, CC))); // Add scratch registers as implicit def and early clobber. const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC); diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index 7e72dc6..291b583 100644 --- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -31,6 +31,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 61c0a6f..ece38f3 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1442,13 +1442,27 @@ SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); + SDValue NewLoad; + if (Op.getValueType().isVector()) - return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), - false, false, false, 0); - return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, - MachinePointerInfo(), - Vec.getValueType().getVectorElementType(), - false, false, false, 0); + NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, + MachinePointerInfo(), false, false, false, 0); + else + NewLoad = DAG.getExtLoad( + ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, MachinePointerInfo(), + Vec.getValueType().getVectorElementType(), false, false, false, 0); + + // Replace the chain going out of the store, by the one out of the load. + DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); + + // We introduced a cycle though, so update the loads operands, making sure + // to use the original store's chain as an incoming chain. + SmallVector NewLoadOperands(NewLoad->op_begin(), + NewLoad->op_end()); + NewLoadOperands[0] = Ch; + NewLoad = + SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); + return NewLoad; } SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { @@ -2817,132 +2831,8 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, std::pair SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { unsigned Opc = Node->getOpcode(); MVT VT = cast(Node)->getMemoryVT().getSimpleVT(); - RTLIB::Libcall LC; - - switch (Opc) { - default: - llvm_unreachable("Unhandled atomic intrinsic Expand!"); - case ISD::ATOMIC_SWAP: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; - case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; - case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; - case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; - case MVT::i128:LC = RTLIB::SYNC_LOCK_TEST_AND_SET_16;break; - } - break; - case ISD::ATOMIC_CMP_SWAP: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; - case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; - case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; - case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; - case MVT::i128:LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16;break; - } - break; - case ISD::ATOMIC_LOAD_ADD: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_ADD_16;break; - } - break; - case ISD::ATOMIC_LOAD_SUB: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_SUB_16;break; - } - break; - case ISD::ATOMIC_LOAD_AND: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_AND_16;break; - } - break; - case ISD::ATOMIC_LOAD_OR: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_OR_16;break; - } - break; - case ISD::ATOMIC_LOAD_XOR: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_XOR_16;break; - } - break; - case ISD::ATOMIC_LOAD_NAND: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_NAND_16;break; - } - break; - case ISD::ATOMIC_LOAD_MAX: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MAX_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MAX_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MAX_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MAX_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MAX_16;break; - } - break; - case ISD::ATOMIC_LOAD_UMAX: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMAX_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMAX_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMAX_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMAX_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMAX_16;break; - } - break; - case ISD::ATOMIC_LOAD_MIN: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MIN_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MIN_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MIN_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MIN_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MIN_16;break; - } - break; - case ISD::ATOMIC_LOAD_UMIN: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMIN_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMIN_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMIN_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMIN_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMIN_16;break; - } - break; - } + RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); return ExpandChainLibCall(LC, Node, false); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 5507c70..25e80b9 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1116,7 +1116,6 @@ SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo){ - assert(OpNo == 2 && "Only know how to promote the mask!"); SDValue DataOp = N->getValue(); EVT DataVT = DataOp.getValueType(); SDValue Mask = N->getMask(); @@ -1127,7 +1126,8 @@ SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpN if (!TLI.isTypeLegal(DataVT)) { if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) { DataOp = GetPromotedInteger(DataOp); - Mask = PromoteTargetBoolean(Mask, DataOp.getValueType()); + if (!TLI.isTypeLegal(MaskVT)) + Mask = PromoteTargetBoolean(Mask, DataOp.getValueType()); TruncateStore = true; } else { @@ -1323,92 +1323,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { std::pair DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { unsigned Opc = Node->getOpcode(); MVT VT = cast(Node)->getMemoryVT().getSimpleVT(); - RTLIB::Libcall LC; - - switch (Opc) { - default: - llvm_unreachable("Unhandled atomic intrinsic Expand!"); - case ISD::ATOMIC_SWAP: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; - case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; - case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; - case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; - case MVT::i128:LC = RTLIB::SYNC_LOCK_TEST_AND_SET_16;break; - } - break; - case ISD::ATOMIC_CMP_SWAP: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; - case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; - case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; - case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; - case MVT::i128:LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16;break; - } - break; - case ISD::ATOMIC_LOAD_ADD: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_ADD_16;break; - } - break; - case ISD::ATOMIC_LOAD_SUB: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_SUB_16;break; - } - break; - case ISD::ATOMIC_LOAD_AND: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_AND_16;break; - } - break; - case ISD::ATOMIC_LOAD_OR: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_OR_16;break; - } - break; - case ISD::ATOMIC_LOAD_XOR: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_XOR_16;break; - } - break; - case ISD::ATOMIC_LOAD_NAND: - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected value type for atomic!"); - case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; - case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; - case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; - case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; - case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_NAND_16;break; - } - break; - } + RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); return ExpandChainLibCall(LC, Node, false); } @@ -1417,12 +1333,19 @@ std::pair DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { /// and the shift amount is a constant 'Amt'. Expand the operation. void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, SDValue &Lo, SDValue &Hi) { - assert(Amt && "Expected zero shifts to be already optimized away."); SDLoc DL(N); // Expand the incoming operand to be shifted, so that we have its parts SDValue InL, InH; GetExpandedInteger(N->getOperand(0), InL, InH); + // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization + // splitted a vector shift, like this: SHL <0, 2>. + if (!Amt) { + Lo = InL; + Hi = InH; + return; + } + EVT NVT = InL.getValueType(); unsigned VTBits = N->getValueType(0).getSizeInBits(); unsigned NVTBits = NVT.getSizeInBits(); diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 63671f7..f7e4557 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -2553,6 +2553,16 @@ SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { assert(InVT.isVector() && "can not widen non-vector type"); EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); + + // The input and output types often differ here, and it could be that while + // we'd prefer to widen the result type, the input operands have been split. + // In this case, we also need to split the result of this node as well. + if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { + SDValue SplitVSetCC = SplitVecOp_VSETCC(N); + SDValue Res = ModifyToType(SplitVSetCC, WidenVT); + return Res; + } + InOp1 = GetWidenedVector(InOp1); SDValue InOp2 = GetWidenedVector(N->getOperand(1)); diff --git a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp index db38b76..6303422 100644 --- a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp +++ b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp @@ -47,7 +47,7 @@ ResourcePriorityQueue::ResourcePriorityQueue(SelectionDAGISel *IS) TRI = STI.getRegisterInfo(); TLI = IS->TLI; TII = STI.getInstrInfo(); - ResourcesModel = TII->CreateTargetScheduleState(STI); + ResourcesModel.reset(TII->CreateTargetScheduleState(STI)); // This hard requirement could be relaxed, but for now // do not let it procede. assert(ResourcesModel && "Unimplemented CreateTargetScheduleState."); @@ -637,17 +637,3 @@ void ResourcePriorityQueue::remove(SUnit *SU) { Queue.pop_back(); } - - -#ifdef NDEBUG -void ResourcePriorityQueue::dump(ScheduleDAG *DAG) const {} -#else -void ResourcePriorityQueue::dump(ScheduleDAG *DAG) const { - ResourcePriorityQueue q = *this; - while (!q.empty()) { - SUnit *su = q.pop(); - dbgs() << "Height " << su->getHeight() << ": "; - su->dump(DAG); - } -} -#endif diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 9466f4d..b52f648 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -196,6 +196,22 @@ bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { return true; } +/// \brief Return true if the specified node is a BUILD_VECTOR node of +/// all ConstantFPSDNode or undef. +bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { + if (N->getOpcode() != ISD::BUILD_VECTOR) + return false; + + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + SDValue Op = N->getOperand(i); + if (Op.getOpcode() == ISD::UNDEF) + continue; + if (!isa(Op)) + return false; + } + return true; +} + /// isScalarToVector - Return true if the specified node is a /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low /// element is not an undef. @@ -1446,13 +1462,7 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { // N2 to point at N1. static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl &M) { std::swap(N1, N2); - int NElts = M.size(); - for (int i = 0; i != NElts; ++i) { - if (M[i] >= NElts) - M[i] -= NElts; - else if (M[i] >= 0) - M[i] += NElts; - } + ShuffleVectorSDNode::commuteMask(M); } SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, @@ -1625,19 +1635,8 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { MVT VT = SV.getSimpleValueType(0); - unsigned NumElems = VT.getVectorNumElements(); - SmallVector MaskVec; - - for (unsigned i = 0; i != NumElems; ++i) { - int Idx = SV.getMaskElt(i); - if (Idx >= 0) { - if (Idx < (int)NumElems) - Idx += NumElems; - else - Idx -= NumElems; - } - MaskVec.push_back(Idx); - } + SmallVector MaskVec(SV.getMask().begin(), SV.getMask().end()); + ShuffleVectorSDNode::commuteMask(MaskVec); SDValue Op0 = SV.getOperand(0); SDValue Op1 = SV.getOperand(1); @@ -2844,7 +2843,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, } } - // Constant fold unary operations with a vector integer operand. + // Constant fold unary operations with a vector integer or float operand. if (BuildVectorSDNode *BV = dyn_cast(Operand.getNode())) { if (BV->isConstant()) { switch (Opcode) { @@ -2852,18 +2851,25 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, // FIXME: Entirely reasonable to perform folding of other unary // operations here as the need arises. break; + case ISD::FNEG: + case ISD::FABS: + case ISD::FP_EXTEND: + case ISD::TRUNCATE: case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: { + // Let the above scalar folding handle the folding of each element. SmallVector Ops; for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { SDValue OpN = BV->getOperand(i); - // Let the above scalar folding handle the conversion of each - // element. - OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(), - OpN); + OpN = getNode(Opcode, DL, VT.getVectorElementType(), OpN); + if (OpN.getOpcode() != ISD::UNDEF && + OpN.getOpcode() != ISD::Constant && + OpN.getOpcode() != ISD::ConstantFP) + break; Ops.push_back(OpN); } - return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); + if (Ops.size() == VT.getVectorNumElements()) + return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); } } } @@ -5418,17 +5424,9 @@ UpdateNodeOperands(SDNode *N, ArrayRef Ops) { assert(N->getNumOperands() == NumOps && "Update with wrong number of operands"); - // Check to see if there is no change. - bool AnyChange = false; - for (unsigned i = 0; i != NumOps; ++i) { - if (Ops[i] != N->getOperand(i)) { - AnyChange = true; - break; - } - } - - // No operands changed, just return the input node. - if (!AnyChange) return N; + // If no operands changed just return the input node. + if (Ops.empty() || std::equal(Ops.begin(), Ops.end(), N->op_begin())) + return N; // See if the modified node already exists. void *InsertPos = nullptr; @@ -6673,8 +6671,8 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType()); APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); - llvm::computeKnownBits(const_cast(GV), KnownZero, KnownOne, - TLI->getDataLayout()); + llvm::computeKnownBits(const_cast(GV), KnownZero, KnownOne, + *TLI->getDataLayout()); unsigned AlignBits = KnownZero.countTrailingOnes(); unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; if (Align) diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 097b618..6c14e79 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1016,6 +1016,24 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, } } +/// getCopyFromRegs - If there was virtual register allocated for the value V +/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. +SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { + DenseMap::iterator It = FuncInfo.ValueMap.find(V); + SDValue res; + + if (It != FuncInfo.ValueMap.end()) { + unsigned InReg = It->second; + RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg, + Ty); + SDValue Chain = DAG.getEntryNode(); + res = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); + resolveDanglingDebugInfo(V, res); + } + + return res; +} + /// getValue - Return an SDValue for the given Value. SDValue SelectionDAGBuilder::getValue(const Value *V) { // If we already have an SDValue for this value, use it. It's important @@ -1026,15 +1044,9 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { // If there's a virtual register allocated and initialized for this // value, use it. - DenseMap::iterator It = FuncInfo.ValueMap.find(V); - if (It != FuncInfo.ValueMap.end()) { - unsigned InReg = It->second; - RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg, - V->getType()); - SDValue Chain = DAG.getEntryNode(); - N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); - resolveDanglingDebugInfo(V, N); - return N; + SDValue copyFromReg = getCopyFromRegs(V, V->getType()); + if (copyFromReg.getNode()) { + return copyFromReg; } // Otherwise create a new SDValue and remember it. @@ -1573,19 +1585,13 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) { // Update machine-CFG edges. MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; - // Figure out which block is immediately after the current one. - MachineBasicBlock *NextBlock = nullptr; - MachineFunction::iterator BBI = BrMBB; - if (++BBI != FuncInfo.MF->end()) - NextBlock = BBI; - if (I.isUnconditional()) { // Update machine-CFG edges. BrMBB->addSuccessor(Succ0MBB); // If this is not a fall-through branch or optimizations are switched off, // emit the branch. - if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None) + if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None) DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), DAG.getBasicBlock(Succ0MBB))); @@ -1682,7 +1688,7 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); const APInt& Low = cast(CB.CmpLHS)->getValue(); - const APInt& High = cast(CB.CmpRHS)->getValue(); + const APInt& High = cast(CB.CmpRHS)->getValue(); SDValue CmpOp = getValue(CB.CmpMHS); EVT VT = CmpOp.getValueType(); @@ -1705,16 +1711,9 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, if (CB.TrueBB != CB.FalseBB) addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight); - // Set NextBlock to be the MBB immediately after the current one, if any. - // This is used to avoid emitting unnecessary branches to the next block. - MachineBasicBlock *NextBlock = nullptr; - MachineFunction::iterator BBI = SwitchBB; - if (++BBI != FuncInfo.MF->end()) - NextBlock = BBI; - // If the lhs block is the next block, invert the condition so that we can // fall through to the lhs instead of the rhs block. - if (CB.TrueBB == NextBlock) { + if (CB.TrueBB == NextBlock(SwitchBB)) { std::swap(CB.TrueBB, CB.FalseBB); SDValue True = DAG.getConstant(1, Cond.getValueType()); Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); @@ -1781,19 +1780,12 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, Sub.getValueType()), Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT); - // Set NextBlock to be the MBB immediately after the current one, if any. - // This is used to avoid emitting unnecessary branches to the next block. - MachineBasicBlock *NextBlock = nullptr; - MachineFunction::iterator BBI = SwitchBB; - - if (++BBI != FuncInfo.MF->end()) - NextBlock = BBI; - SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), MVT::Other, CopyTo, CMP, DAG.getBasicBlock(JT.Default)); - if (JT.MBB != NextBlock) + // Avoid emitting unnecessary branches to the next block. + if (JT.MBB != NextBlock(SwitchBB)) BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond, DAG.getBasicBlock(JT.MBB)); @@ -1922,13 +1914,6 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), B.Reg, Sub); - // Set NextBlock to be the MBB immediately after the current one, if any. - // This is used to avoid emitting unnecessary branches to the next block. - MachineBasicBlock *NextBlock = nullptr; - MachineFunction::iterator BBI = SwitchBB; - if (++BBI != FuncInfo.MF->end()) - NextBlock = BBI; - MachineBasicBlock* MBB = B.Cases[0].ThisBB; addSuccessorWithWeight(SwitchBB, B.Default); @@ -1938,7 +1923,8 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, MVT::Other, CopyTo, RangeCmp, DAG.getBasicBlock(B.Default)); - if (MBB != NextBlock) + // Avoid emitting unnecessary branches to the next block. + if (MBB != NextBlock(SwitchBB)) BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo, DAG.getBasicBlock(MBB)); @@ -1991,14 +1977,8 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, MVT::Other, getControlRoot(), Cmp, DAG.getBasicBlock(B.TargetBB)); - // Set NextBlock to be the MBB immediately after the current one, if any. - // This is used to avoid emitting unnecessary branches to the next block. - MachineBasicBlock *NextBlock = nullptr; - MachineFunction::iterator BBI = SwitchBB; - if (++BBI != FuncInfo.MF->end()) - NextBlock = BBI; - - if (NextMBB != NextBlock) + // Avoid emitting unnecessary branches to the next block. + if (NextMBB != NextBlock(SwitchBB)) BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd, DAG.getBasicBlock(NextMBB)); @@ -2027,13 +2007,20 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { case Intrinsic::experimental_patchpoint_i64: visitPatchpoint(&I, LandingPad); break; + case Intrinsic::experimental_gc_statepoint: + LowerStatepoint(ImmutableStatepoint(&I), LandingPad); + break; } } else LowerCallTo(&I, getValue(Callee), false, LandingPad); // If the value of the invoke is used outside of its defining block, make it // available as a virtual register. - CopyToExportRegsIfNeeded(&I); + // We already took care of the exported value for the statepoint instruction + // during call to the LowerStatepoint. + if (!isStatepoint(I)) { + CopyToExportRegsIfNeeded(&I); + } // Update successor info addSuccessorWithWeight(InvokeMBB, Return); @@ -2128,11 +2115,10 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, MachineFunction *CurMF = FuncInfo.MF; // Figure out which block is immediately after the current one. - MachineBasicBlock *NextBlock = nullptr; + MachineBasicBlock *NextMBB = nullptr; MachineFunction::iterator BBI = CR.CaseBB; - if (++BBI != FuncInfo.MF->end()) - NextBlock = BBI; + NextMBB = BBI; BranchProbabilityInfo *BPI = FuncInfo.BPI; // If any two of the cases has the same destination, and if one value @@ -2146,8 +2132,8 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, Case &Big = *(CR.Range.second-1); if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) { - const APInt& SmallValue = cast(Small.Low)->getValue(); - const APInt& BigValue = cast(Big.Low)->getValue(); + const APInt& SmallValue = Small.Low->getValue(); + const APInt& BigValue = Big.Low->getValue(); // Check that there is only one bit different. if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 && @@ -2205,13 +2191,12 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, } // Rearrange the case blocks so that the last one falls through if possible. Case &BackCase = *(CR.Range.second-1); - if (Size > 1 && - NextBlock && Default != NextBlock && BackCase.BB != NextBlock) { - // The last case block won't fall through into 'NextBlock' if we emit the + if (Size > 1 && NextMBB && Default != NextMBB && BackCase.BB != NextMBB) { + // The last case block won't fall through into 'NextMBB' if we emit the // branches in this order. See if rearranging a case value would help. // We start at the bottom as it's the case with the least weight. for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I) - if (I->BB == NextBlock) { + if (I->BB == NextMBB) { std::swap(*I, BackCase); break; } @@ -2287,8 +2272,8 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); - const APInt &First = cast(FrontCase.Low)->getValue(); - const APInt &Last = cast(BackCase.High)->getValue(); + const APInt &First = FrontCase.Low->getValue(); + const APInt &Last = BackCase.High->getValue(); APInt TSize(First.getBitWidth(), 0); for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) @@ -2338,8 +2323,8 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, std::vector DestBBs; APInt TEI = First; for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) { - const APInt &Low = cast(I->Low)->getValue(); - const APInt &High = cast(I->High)->getValue(); + const APInt &Low = I->Low->getValue(); + const APInt &High = I->High->getValue(); if (Low.sle(TEI) && TEI.sle(High)) { DestBBs.push_back(I->BB); @@ -2352,26 +2337,19 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, // Calculate weight for each unique destination in CR. DenseMap DestWeights; - if (FuncInfo.BPI) - for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { - DenseMap::iterator Itr = - DestWeights.find(I->BB); - if (Itr != DestWeights.end()) - Itr->second += I->ExtraWeight; - else - DestWeights[I->BB] = I->ExtraWeight; - } + if (FuncInfo.BPI) { + for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) + DestWeights[I->BB] += I->ExtraWeight; + } // Update successor info. Add one edge to each unique successor. BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs()); - for (std::vector::iterator I = DestBBs.begin(), - E = DestBBs.end(); I != E; ++I) { - if (!SuccsHandled[(*I)->getNumber()]) { - SuccsHandled[(*I)->getNumber()] = true; - DenseMap::iterator Itr = - DestWeights.find(*I); - addSuccessorWithWeight(JumpTableBB, *I, - Itr != DestWeights.end() ? Itr->second : 0); + for (MachineBasicBlock *DestBB : DestBBs) { + if (!SuccsHandled[DestBB->getNumber()]) { + SuccsHandled[DestBB->getNumber()] = true; + auto I = DestWeights.find(DestBB); + addSuccessorWithWeight(JumpTableBB, DestBB, + I != DestWeights.end() ? I->second : 0); } } @@ -2403,8 +2381,8 @@ bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, // Size is the number of Cases represented by this range. unsigned Size = CR.Range.second - CR.Range.first; - const APInt &First = cast(FrontCase.Low)->getValue(); - const APInt &Last = cast(BackCase.High)->getValue(); + const APInt &First = FrontCase.Low->getValue(); + const APInt &Last = BackCase.High->getValue(); double FMetric = 0; CaseItr Pivot = CR.Range.first + Size/2; @@ -2423,8 +2401,8 @@ bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, const TargetLowering &TLI = DAG.getTargetLoweringInfo(); for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second; J!=E; ++I, ++J) { - const APInt &LEnd = cast(I->High)->getValue(); - const APInt &RBegin = cast(J->Low)->getValue(); + const APInt &LEnd = I->High->getValue(); + const APInt &RBegin = J->Low->getValue(); APInt Range = ComputeRange(LEnd, RBegin); assert((Range - 2ULL).isNonNegative() && "Invalid case distance"); @@ -2479,7 +2457,7 @@ void SelectionDAGBuilder::splitSwitchCase(CaseRec &CR, CaseItr Pivot, CaseRange LHSR(CR.Range.first, Pivot); CaseRange RHSR(Pivot, CR.Range.second); - const Constant *C = Pivot->Low; + const ConstantInt *C = Pivot->Low; MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr; // We know that we branch to the LHS if the Value being switched on is @@ -2489,8 +2467,7 @@ void SelectionDAGBuilder::splitSwitchCase(CaseRec &CR, CaseItr Pivot, // Pivot's Value, then we can branch directly to the LHS's Target, // rather than creating a leaf node for it. if ((LHSR.second - LHSR.first) == 1 && LHSR.first->High == CR.GE && - cast(C)->getValue() == - (cast(CR.GE)->getValue() + 1LL)) { + C->getValue() == (CR.GE->getValue() + 1LL)) { TrueBB = LHSR.first->BB; } else { TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); @@ -2506,8 +2483,7 @@ void SelectionDAGBuilder::splitSwitchCase(CaseRec &CR, CaseItr Pivot, // is CR.LT - 1, then we can branch directly to the target block for // the current Case Value, rather than emitting a RHS leaf node for it. if ((RHSR.second - RHSR.first) == 1 && CR.LT && - cast(RHSR.first->Low)->getValue() == - (cast(CR.LT)->getValue() - 1LL)) { + RHSR.first->Low->getValue() == (CR.LT->getValue() - 1LL)) { FalseBB = RHSR.first->BB; } else { FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); @@ -2571,8 +2547,8 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, << "Total number of comparisons: " << numCmps << '\n'); // Compute span of values. - const APInt& minValue = cast(FrontCase.Low)->getValue(); - const APInt& maxValue = cast(BackCase.High)->getValue(); + const APInt& minValue = FrontCase.Low->getValue(); + const APInt& maxValue = BackCase.High->getValue(); APInt cmpRange = maxValue - minValue; DEBUG(dbgs() << "Compare range: " << cmpRange << '\n' @@ -2612,8 +2588,8 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, count++; } - const APInt& lowValue = cast(I->Low)->getValue(); - const APInt& highValue = cast(I->High)->getValue(); + const APInt& lowValue = I->Low->getValue(); + const APInt& highValue = I->High->getValue(); uint64_t lo = (lowValue - lowBound).getZExtValue(); uint64_t hi = (highValue - lowBound).getZExtValue(); @@ -2663,45 +2639,42 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, return true; } -/// Clusterify - Transform simple list of Cases into list of CaseRange's -void SelectionDAGBuilder::Clusterify(CaseVector& Cases, - const SwitchInst& SI) { +void SelectionDAGBuilder::Clusterify(CaseVector &Cases, const SwitchInst *SI) { BranchProbabilityInfo *BPI = FuncInfo.BPI; - // Start with "simple" cases. - for (SwitchInst::ConstCaseIt i : SI.cases()) { - const BasicBlock *SuccBB = i.getCaseSuccessor(); - MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB]; - - uint32_t ExtraWeight = - BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0; - - Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(), - SMBB, ExtraWeight)); - } - std::sort(Cases.begin(), Cases.end(), CaseCmp()); - - // Merge case into clusters - if (Cases.size() >= 2) - // Must recompute end() each iteration because it may be - // invalidated by erase if we hold on to it - for (CaseItr I = Cases.begin(), J = std::next(Cases.begin()); - J != Cases.end(); ) { - const APInt& nextValue = cast(J->Low)->getValue(); - const APInt& currentValue = cast(I->High)->getValue(); - MachineBasicBlock* nextBB = J->BB; - MachineBasicBlock* currentBB = I->BB; - - // If the two neighboring cases go to the same destination, merge them - // into a single case. - if ((nextValue - currentValue == 1) && (currentBB == nextBB)) { - I->High = J->High; - I->ExtraWeight += J->ExtraWeight; - J = Cases.erase(J); - } else { - I = J++; - } + + // Extract cases from the switch and sort them. + typedef std::pair CasePair; + std::vector Sorted; + Sorted.reserve(SI->getNumCases()); + for (auto I : SI->cases()) + Sorted.push_back(std::make_pair(I.getCaseValue(), I.getSuccessorIndex())); + std::sort(Sorted.begin(), Sorted.end(), [](CasePair a, CasePair b) { + return a.first->getValue().slt(b.first->getValue()); + }); + + // Merge adjacent cases with the same destination, build Cases vector. + assert(Cases.empty() && "Cases should be empty before Clusterify;"); + Cases.reserve(SI->getNumCases()); + MachineBasicBlock *PreviousSucc = nullptr; + for (CasePair &CP : Sorted) { + const ConstantInt *CaseVal = CP.first; + unsigned SuccIndex = CP.second; + MachineBasicBlock *Succ = FuncInfo.MBBMap[SI->getSuccessor(SuccIndex)]; + uint32_t Weight = BPI ? BPI->getEdgeWeight(SI->getParent(), SuccIndex) : 0; + + if (PreviousSucc == Succ && + (CaseVal->getValue() - Cases.back().High->getValue()) == 1) { + // If this case has the same successor and is a neighbour, merge it into + // the previous cluster. + Cases.back().High = CaseVal; + Cases.back().ExtraWeight += Weight; + } else { + Cases.push_back(Case(CaseVal, CaseVal, Succ, Weight)); } + PreviousSucc = Succ; + } + DEBUG({ size_t numCmps = 0; for (auto &I : Cases) @@ -2729,16 +2702,10 @@ void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { MachineBasicBlock *SwitchMBB = FuncInfo.MBB; - // Figure out which block is immediately after the current one. - MachineBasicBlock *NextBlock = nullptr; - if (SwitchMBB + 1 != FuncInfo.MF->end()) - NextBlock = SwitchMBB + 1; - - // Create a vector of Cases, sorted so that we can efficiently create a binary // search tree from them. CaseVector Cases; - Clusterify(Cases, SI); + Clusterify(Cases, &SI); // Get the default destination MBB. MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()]; @@ -2775,7 +2742,7 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { SwitchMBB->addSuccessor(Default); // If this is not a fall-through branch, emit the branch. - if (Default != NextBlock) { + if (Default != NextBlock(SwitchMBB)) { DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), DAG.getBasicBlock(Default))); } @@ -3429,30 +3396,21 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { Ty = StTy->getElementType(Field); } else { Ty = cast(Ty)->getElementType(); + MVT PtrTy = DAG.getTargetLoweringInfo().getPointerTy(AS); + unsigned PtrSize = PtrTy.getSizeInBits(); + APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty)); // If this is a constant subscript, handle it quickly. - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - if (const ConstantInt *CI = dyn_cast(Idx)) { - if (CI->isZero()) continue; - uint64_t Offs = - DL->getTypeAllocSize(Ty)*cast(CI)->getSExtValue(); - SDValue OffsVal; - EVT PTy = TLI.getPointerTy(AS); - unsigned PtrBits = PTy.getSizeInBits(); - if (PtrBits < 64) - OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy, - DAG.getConstant(Offs, MVT::i64)); - else - OffsVal = DAG.getConstant(Offs, PTy); - - N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, - OffsVal); + if (const auto *CI = dyn_cast(Idx)) { + if (CI->isZero()) + continue; + APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize); + SDValue OffsVal = DAG.getConstant(Offs, PtrTy); + N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, OffsVal); continue; } // N = N + Idx * ElementSize; - APInt ElementSize = - APInt(TLI.getPointerSizeInBits(AS), DL->getTypeAllocSize(Ty)); SDValue IdxN = getValue(Idx); // If the index is smaller or larger than intptr_t, truncate or extend @@ -3988,6 +3946,93 @@ getF32Constant(SelectionDAG &DAG, unsigned Flt) { MVT::f32); } +static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl, + SelectionDAG &DAG) { + // IntegerPartOfX = ((int32_t)(t0); + SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); + + // FractionalPartOfX = t0 - (float)IntegerPartOfX; + SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); + SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); + + // IntegerPartOfX <<= 23; + IntegerPartOfX = DAG.getNode( + ISD::SHL, dl, MVT::i32, IntegerPartOfX, + DAG.getConstant(23, DAG.getTargetLoweringInfo().getPointerTy())); + + SDValue TwoToFractionalPartOfX; + if (LimitFloatPrecision <= 6) { + // For floating-point precision of 6: + // + // TwoToFractionalPartOfX = + // 0.997535578f + + // (0.735607626f + 0.252464424f * x) * x; + // + // error 0.0144103317, which is 6 bits + SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, + getF32Constant(DAG, 0x3e814304)); + SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, + getF32Constant(DAG, 0x3f3c50c8)); + SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3f7f5e7e)); + } else if (LimitFloatPrecision <= 12) { + // For floating-point precision of 12: + // + // TwoToFractionalPartOfX = + // 0.999892986f + + // (0.696457318f + + // (0.224338339f + 0.792043434e-1f * x) * x) * x; + // + // error 0.000107046256, which is 13 to 14 bits + SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, + getF32Constant(DAG, 0x3da235e3)); + SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, + getF32Constant(DAG, 0x3e65b8f3)); + SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); + SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3f324b07)); + SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, + getF32Constant(DAG, 0x3f7ff8fd)); + } else { // LimitFloatPrecision <= 18 + // For floating-point precision of 18: + // + // TwoToFractionalPartOfX = + // 0.999999982f + + // (0.693148872f + + // (0.240227044f + + // (0.554906021e-1f + + // (0.961591928e-2f + + // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; + // error 2.47208000*10^(-7), which is better than 18 bits + SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, + getF32Constant(DAG, 0x3924b03e)); + SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, + getF32Constant(DAG, 0x3ab24b87)); + SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); + SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3c1d8c17)); + SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); + SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, + getF32Constant(DAG, 0x3d634a1d)); + SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); + SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, + getF32Constant(DAG, 0x3e75fe14)); + SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); + SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, + getF32Constant(DAG, 0x3f317234)); + SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, + getF32Constant(DAG, 0x3f800000)); + } + + // Add the exponent into the result in integer domain. + SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX); + return DAG.getNode(ISD::BITCAST, dl, MVT::f32, + DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX)); +} + /// expandExp - Lower an exp intrinsic. Handles the special sequences for /// limited-precision mode. static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG, @@ -3999,92 +4044,10 @@ static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG, // final result: // // #define LOG2OFe 1.4426950f - // IntegerPartOfX = ((int32_t)(X * LOG2OFe)); + // t0 = Op * LOG2OFe SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, getF32Constant(DAG, 0x3fb8aa3b)); - SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); - - // FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX; - SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); - SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); - - // IntegerPartOfX <<= 23; - IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, - DAG.getConstant(23, TLI.getPointerTy())); - - SDValue TwoToFracPartOfX; - if (LimitFloatPrecision <= 6) { - // For floating-point precision of 6: - // - // TwoToFractionalPartOfX = - // 0.997535578f + - // (0.735607626f + 0.252464424f * x) * x; - // - // error 0.0144103317, which is 6 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3e814304)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3f3c50c8)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f7f5e7e)); - } else if (LimitFloatPrecision <= 12) { - // For floating-point precision of 12: - // - // TwoToFractionalPartOfX = - // 0.999892986f + - // (0.696457318f + - // (0.224338339f + 0.792043434e-1f * x) * x) * x; - // - // 0.000107046256 error, which is 13 to 14 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3da235e3)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3e65b8f3)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f324b07)); - SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3f7ff8fd)); - } else { // LimitFloatPrecision <= 18 - // For floating-point precision of 18: - // - // TwoToFractionalPartOfX = - // 0.999999982f + - // (0.693148872f + - // (0.240227044f + - // (0.554906021e-1f + - // (0.961591928e-2f + - // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; - // - // error 2.47208000*10^(-7), which is better than 18 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3924b03e)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3ab24b87)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3c1d8c17)); - SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3d634a1d)); - SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); - SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, - getF32Constant(DAG, 0x3e75fe14)); - SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); - SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, - getF32Constant(DAG, 0x3f317234)); - SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); - TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, - getF32Constant(DAG, 0x3f800000)); - } - - // Add the exponent into the result in integer domain. - SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX); - return DAG.getNode(ISD::BITCAST, dl, MVT::f32, - DAG.getNode(ISD::ADD, dl, MVT::i32, - t13, IntegerPartOfX)); + return getLimitedPrecisionExp2(t0, dl, DAG); } // No special expansion. @@ -4375,91 +4338,8 @@ static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG, static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI) { if (Op.getValueType() == MVT::f32 && - LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); - - // FractionalPartOfX = x - (float)IntegerPartOfX; - SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); - SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1); - - // IntegerPartOfX <<= 23; - IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, - DAG.getConstant(23, TLI.getPointerTy())); - - SDValue TwoToFractionalPartOfX; - if (LimitFloatPrecision <= 6) { - // For floating-point precision of 6: - // - // TwoToFractionalPartOfX = - // 0.997535578f + - // (0.735607626f + 0.252464424f * x) * x; - // - // error 0.0144103317, which is 6 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3e814304)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3f3c50c8)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f7f5e7e)); - } else if (LimitFloatPrecision <= 12) { - // For floating-point precision of 12: - // - // TwoToFractionalPartOfX = - // 0.999892986f + - // (0.696457318f + - // (0.224338339f + 0.792043434e-1f * x) * x) * x; - // - // error 0.000107046256, which is 13 to 14 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3da235e3)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3e65b8f3)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f324b07)); - SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3f7ff8fd)); - } else { // LimitFloatPrecision <= 18 - // For floating-point precision of 18: - // - // TwoToFractionalPartOfX = - // 0.999999982f + - // (0.693148872f + - // (0.240227044f + - // (0.554906021e-1f + - // (0.961591928e-2f + - // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; - // error 2.47208000*10^(-7), which is better than 18 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3924b03e)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3ab24b87)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3c1d8c17)); - SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3d634a1d)); - SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); - SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, - getF32Constant(DAG, 0x3e75fe14)); - SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); - SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, - getF32Constant(DAG, 0x3f317234)); - SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); - TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, - getF32Constant(DAG, 0x3f800000)); - } - - // Add the exponent into the result in integer domain. - SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, - TwoToFractionalPartOfX); - return DAG.getNode(ISD::BITCAST, dl, MVT::f32, - DAG.getNode(ISD::ADD, dl, MVT::i32, - t13, IntegerPartOfX)); - } + LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) + return getLimitedPrecisionExp2(Op, dl, DAG); // No special expansion. return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op); @@ -4483,90 +4363,10 @@ static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS, // final result: // // #define LOG2OF10 3.3219281f - // IntegerPartOfX = (int32_t)(x * LOG2OF10); + // t0 = Op * LOG2OF10; SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, getF32Constant(DAG, 0x40549a78)); - SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); - - // FractionalPartOfX = x - (float)IntegerPartOfX; - SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); - SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); - - // IntegerPartOfX <<= 23; - IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, - DAG.getConstant(23, TLI.getPointerTy())); - - SDValue TwoToFractionalPartOfX; - if (LimitFloatPrecision <= 6) { - // For floating-point precision of 6: - // - // twoToFractionalPartOfX = - // 0.997535578f + - // (0.735607626f + 0.252464424f * x) * x; - // - // error 0.0144103317, which is 6 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3e814304)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3f3c50c8)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f7f5e7e)); - } else if (LimitFloatPrecision <= 12) { - // For floating-point precision of 12: - // - // TwoToFractionalPartOfX = - // 0.999892986f + - // (0.696457318f + - // (0.224338339f + 0.792043434e-1f * x) * x) * x; - // - // error 0.000107046256, which is 13 to 14 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3da235e3)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3e65b8f3)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f324b07)); - SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3f7ff8fd)); - } else { // LimitFloatPrecision <= 18 - // For floating-point precision of 18: - // - // TwoToFractionalPartOfX = - // 0.999999982f + - // (0.693148872f + - // (0.240227044f + - // (0.554906021e-1f + - // (0.961591928e-2f + - // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; - // error 2.47208000*10^(-7), which is better than 18 bits - SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, - getF32Constant(DAG, 0x3924b03e)); - SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3ab24b87)); - SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3c1d8c17)); - SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3d634a1d)); - SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); - SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, - getF32Constant(DAG, 0x3e75fe14)); - SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); - SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, - getF32Constant(DAG, 0x3f317234)); - SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); - TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, - getF32Constant(DAG, 0x3f800000)); - } - - SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX); - return DAG.getNode(ISD::BITCAST, dl, MVT::f32, - DAG.getNode(ISD::ADD, dl, MVT::i32, - t13, IntegerPartOfX)); + return getLimitedPrecisionExp2(t0, dl, DAG); } // No special expansion. @@ -5114,34 +4914,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { setValue(&I, Res); return nullptr; } - case Intrinsic::x86_avx_vinsertf128_pd_256: - case Intrinsic::x86_avx_vinsertf128_ps_256: - case Intrinsic::x86_avx_vinsertf128_si_256: - case Intrinsic::x86_avx2_vinserti128: { - EVT DestVT = TLI.getValueType(I.getType()); - EVT ElVT = TLI.getValueType(I.getArgOperand(1)->getType()); - uint64_t Idx = (cast(I.getArgOperand(2))->getZExtValue() & 1) * - ElVT.getVectorNumElements(); - Res = - DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT, - getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), - DAG.getConstant(Idx, TLI.getVectorIdxTy())); - setValue(&I, Res); - return nullptr; - } - case Intrinsic::x86_avx_vextractf128_pd_256: - case Intrinsic::x86_avx_vextractf128_ps_256: - case Intrinsic::x86_avx_vextractf128_si_256: - case Intrinsic::x86_avx2_vextracti128: { - EVT DestVT = TLI.getValueType(I.getType()); - uint64_t Idx = (cast(I.getArgOperand(1))->getZExtValue() & 1) * - DestVT.getVectorNumElements(); - Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT, - getValue(I.getArgOperand(0)), - DAG.getConstant(Idx, TLI.getVectorIdxTy())); - setValue(&I, Res); - return nullptr; - } case Intrinsic::convertff: case Intrinsic::convertfsi: case Intrinsic::convertfui: @@ -5539,7 +5311,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { return nullptr; SmallVector Allocas; - GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL); + GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL); for (SmallVectorImpl::iterator Object = Allocas.begin(), E = Allocas.end(); Object != E; ++Object) { @@ -5618,45 +5390,47 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::instrprof_increment: llvm_unreachable("instrprof failed to lower an increment"); - case Intrinsic::frameallocate: { + case Intrinsic::frameescape: { MachineFunction &MF = DAG.getMachineFunction(); const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); - // Do the allocation and map it as a normal value. - // FIXME: Maybe we should add this to the alloca map so that we don't have - // to register allocate it? - uint64_t Size = cast(I.getArgOperand(0))->getZExtValue(); - int Alloc = MF.getFrameInfo()->CreateFrameAllocation(Size); - MVT PtrVT = TLI.getPointerTy(0); - SDValue FIVal = DAG.getFrameIndex(Alloc, PtrVT); - setValue(&I, FIVal); - - // Directly emit a FRAME_ALLOC machine instr. Label assignment emission is - // the same on all targets. - MCSymbol *FrameAllocSym = - MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName()); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, - TII->get(TargetOpcode::FRAME_ALLOC)) - .addSym(FrameAllocSym) - .addFrameIndex(Alloc); + // Directly emit some FRAME_ALLOC machine instrs. Label assignment emission + // is the same on all targets. + for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) { + AllocaInst *Slot = + cast(I.getArgOperand(Idx)->stripPointerCasts()); + assert(FuncInfo.StaticAllocaMap.count(Slot) && + "can only escape static allocas"); + int FI = FuncInfo.StaticAllocaMap[Slot]; + MCSymbol *FrameAllocSym = + MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName(), + Idx); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, + TII->get(TargetOpcode::FRAME_ALLOC)) + .addSym(FrameAllocSym) + .addFrameIndex(FI); + } return nullptr; } case Intrinsic::framerecover: { - // i8* @llvm.framerecover(i8* %fn, i8* %fp) + // i8* @llvm.framerecover(i8* %fn, i8* %fp, i32 %idx) MachineFunction &MF = DAG.getMachineFunction(); MVT PtrVT = TLI.getPointerTy(0); // Get the symbol that defines the frame offset. - Function *Fn = cast(I.getArgOperand(0)->stripPointerCasts()); + auto *Fn = cast(I.getArgOperand(0)->stripPointerCasts()); + auto *Idx = cast(I.getArgOperand(2)); + unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX)); MCSymbol *FrameAllocSym = - MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName()); + MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName(), + IdxVal); // Create a TargetExternalSymbol for the label to avoid any target lowering // that would make this PC relative. StringRef Name = FrameAllocSym->getName(); - assert(Name.size() == strlen(Name.data()) && "not null terminated"); + assert(Name.data()[Name.size()] == '\0' && "not null terminated"); SDValue OffsetSym = DAG.getTargetExternalSymbol(Name.data(), PtrVT); SDValue OffsetVal = DAG.getNode(ISD::FRAME_ALLOC_RECOVER, sdl, PtrVT, OffsetSym); @@ -5672,6 +5446,16 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_begincatch: case Intrinsic::eh_endcatch: llvm_unreachable("begin/end catch intrinsics not lowered in codegen"); + case Intrinsic::eh_unwindhelp: { + AllocaInst *Slot = + cast(I.getArgOperand(0)->stripPointerCasts()); + assert(FuncInfo.StaticAllocaMap.count(Slot) && + "can only use static allocas with llvm.eh.unwindhelp"); + int FI = FuncInfo.StaticAllocaMap[Slot]; + // TODO: Save this in the not-yet-existant WinEHFuncInfo struct. + (void)FI; + return nullptr; + } } } @@ -5805,9 +5589,8 @@ static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, LoadInput = ConstantExpr::getBitCast(const_cast(LoadInput), PointerType::getUnqual(LoadTy)); - if (const Constant *LoadCst = - ConstantFoldLoadFromConstPtr(const_cast(LoadInput), - Builder.DL)) + if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr( + const_cast(LoadInput), *Builder.DL)) return Builder.getValue(LoadCst); } @@ -6748,10 +6531,15 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // Memory output, or 'other' output (e.g. 'X' constraint). assert(OpInfo.isIndirect && "Memory output must be indirect operand"); + unsigned ConstraintID = + TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); + assert(ConstraintID != InlineAsm::Constraint_Unknown && + "Failed to convert memory constraint code to constraint id."); + // Add information to the INLINEASM node to know about this output. unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); - AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, - TLI.getPointerTy())); + OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID); + AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, MVT::i32)); AsmNodeOperands.push_back(OpInfo.CallOperand); break; } @@ -6855,6 +6643,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { "Unexpected number of operands"); // Add information to the INLINEASM node to know about this input. // See InlineAsm.h isUseOperandTiedToDef. + OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag); OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, OpInfo.getMatchedOperand()); AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag, @@ -6894,10 +6683,15 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { assert(InOperandVal.getValueType() == TLI.getPointerTy() && "Memory operands expect pointer values"); + unsigned ConstraintID = + TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); + assert(ConstraintID != InlineAsm::Constraint_Unknown && + "Failed to convert memory constraint code to constraint id."); + // Add information to the INLINEASM node to know about this input. unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); - AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, - TLI.getPointerTy())); + ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID); + AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, MVT::i32)); AsmNodeOperands.push_back(InOperandVal); break; } @@ -7901,8 +7695,8 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { SmallPtrSet SuccsHandled; - // Check successor nodes' PHI nodes that expect a constant to be available - // from this block. + // Check PHI nodes in successors that expect a value to be available from this + // block. for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { const BasicBlock *SuccBB = TI->getSuccessor(succ); if (!isa(SuccBB->begin())) continue; @@ -7989,3 +7783,10 @@ AddSuccessorMBB(const BasicBlock *BB, SuccMBB, BranchProbabilityInfo::getBranchWeightStackProtector(IsLikely)); return SuccMBB; } + +MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) { + MachineFunction::iterator I = MBB; + if (++I == FuncInfo.MF->end()) + return nullptr; + return I; +} diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index ad7411f..30240d8 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -137,19 +137,19 @@ private: /// Case - A struct to record the Value for a switch case, and the /// case's target basic block. struct Case { - const Constant *Low; - const Constant *High; + const ConstantInt *Low; + const ConstantInt *High; MachineBasicBlock* BB; uint32_t ExtraWeight; Case() : Low(nullptr), High(nullptr), BB(nullptr), ExtraWeight(0) { } - Case(const Constant *low, const Constant *high, MachineBasicBlock *bb, + Case(const ConstantInt *low, const ConstantInt *high, MachineBasicBlock *bb, uint32_t extraweight) : Low(low), High(high), BB(bb), ExtraWeight(extraweight) { } APInt size() const { - const APInt &rHigh = cast(High)->getValue(); - const APInt &rLow = cast(Low)->getValue(); + const APInt &rHigh = High->getValue(); + const APInt &rLow = Low->getValue(); return (rHigh - rLow + 1ULL); } }; @@ -173,7 +173,7 @@ private: /// CaseRec - A struct with ctor used in lowering switches to a binary tree /// of conditional branches. struct CaseRec { - CaseRec(MachineBasicBlock *bb, const Constant *lt, const Constant *ge, + CaseRec(MachineBasicBlock *bb, const ConstantInt *lt, const ConstantInt *ge, CaseRange r) : CaseBB(bb), LT(lt), GE(ge), Range(r) {} @@ -181,8 +181,8 @@ private: MachineBasicBlock *CaseBB; /// LT, GE - If nonzero, we know the current case value must be less-than or /// greater-than-or-equal-to these Constants. - const Constant *LT; - const Constant *GE; + const ConstantInt *LT; + const ConstantInt *GE; /// Range - A pair of iterators representing the range of case values to be /// processed at this point in the binary search tree. CaseRange Range; @@ -190,24 +190,15 @@ private: typedef std::vector CaseRecVector; - /// The comparison function for sorting the switch case values in the vector. - /// WARNING: Case ranges should be disjoint! - struct CaseCmp { - bool operator()(const Case &C1, const Case &C2) { - assert(isa(C1.Low) && isa(C2.High)); - const ConstantInt* CI1 = cast(C1.Low); - const ConstantInt* CI2 = cast(C2.High); - return CI1->getValue().slt(CI2->getValue()); - } - }; - struct CaseBitsCmp { bool operator()(const CaseBits &C1, const CaseBits &C2) { return C1.Bits > C2.Bits; } }; - void Clusterify(CaseVector &Cases, const SwitchInst &SI); + /// Populate Cases with the cases in SI, clustering adjacent cases with the + /// same destination together. + void Clusterify(CaseVector &Cases, const SwitchInst *SI); /// CaseBlock - This structure is used to communicate between /// SelectionDAGBuilder and SDISel for the code generation of additional basic @@ -606,6 +597,10 @@ public: void visit(unsigned Opcode, const User &I); + /// getCopyFromRegs - If there was virtual register allocated for the value V + /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. + SDValue getCopyFromRegs(const Value *V, Type *Ty); + // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, // generate the debug data structures now that we've seen its definition. void resolveDanglingDebugInfo(const Value *V, SDValue Val); @@ -622,8 +617,7 @@ public: void removeValue(const Value *V) { // This is to support hack in lowerCallFromStatepoint // Should be removed when hack is resolved - if (NodeMap.count(V)) - NodeMap.erase(V); + NodeMap.erase(V); } void setUnusedArgValue(const Value *V, SDValue NewN) { @@ -662,7 +656,9 @@ public: void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); // This function is responsible for the whole statepoint lowering process. - void LowerStatepoint(ImmutableStatepoint Statepoint); + // It uniformly handles invoke and call statepoints. + void LowerStatepoint(ImmutableStatepoint Statepoint, + MachineBasicBlock *LandingPad = nullptr); private: std::pair lowerInvokable( TargetLowering::CallLoweringInfo &CLI, @@ -830,6 +826,9 @@ private: bool EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable, MDNode *Expr, int64_t Offset, bool IsIndirect, const SDValue &N); + + /// Return the next block after MBB, or nullptr if there is none. + MachineBasicBlock *NextBlock(MachineBasicBlock *MBB); }; } // end namespace llvm diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 17eff94..5898da4 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -95,6 +95,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE"; case ISD::RETURNADDR: return "RETURNADDR"; case ISD::FRAMEADDR: return "FRAMEADDR"; + case ISD::FRAME_ALLOC_RECOVER: return "FRAME_ALLOC_RECOVER"; case ISD::READ_REGISTER: return "READ_REGISTER"; case ISD::WRITE_REGISTER: return "WRITE_REGISTER"; case ISD::FRAME_TO_ARGS_OFFSET: return "FRAME_TO_ARGS_OFFSET"; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 5e867cf..4d2af3f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -168,14 +168,13 @@ static cl::opt EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, cl::desc("Enable verbose messages in the \"fast\" " "instruction selector")); -static cl::opt -EnableFastISelAbort("fast-isel-abort", cl::Hidden, - cl::desc("Enable abort calls when \"fast\" instruction selection " - "fails to lower an instruction")); -static cl::opt -EnableFastISelAbortArgs("fast-isel-abort-args", cl::Hidden, - cl::desc("Enable abort calls when \"fast\" instruction selection " - "fails to lower a formal argument")); +static cl::opt EnableFastISelAbort( + "fast-isel-abort", cl::Hidden, + cl::desc("Enable abort calls when \"fast\" instruction selection " + "fails to lower an instruction: 0 disable the abort, 1 will " + "abort but for args, calls and terminators, 2 will also " + "abort for argument lowering, and 3 will never fallback " + "to SelectionDAG.")); static cl::opt UseMBPI("use-mbpi", @@ -293,7 +292,8 @@ namespace llvm { const TargetLowering *TLI = IS->TLI; const TargetSubtargetInfo &ST = IS->MF->getSubtarget(); - if (OptLevel == CodeGenOpt::None || ST.useMachineScheduler() || + if (OptLevel == CodeGenOpt::None || + (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) || TLI->getSchedulingPreference() == Sched::Source) return createSourceListDAGScheduler(IS, OptLevel); if (TLI->getSchedulingPreference() == Sched::RegPressure) @@ -416,7 +416,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { assert((!EnableFastISelVerbose || TM.Options.EnableFastISel) && "-fast-isel-verbose requires -fast-isel"); assert((!EnableFastISelAbort || TM.Options.EnableFastISel) && - "-fast-isel-abort requires -fast-isel"); + "-fast-isel-abort > 0 requires -fast-isel"); const Function &Fn = *mf.getFunction(); MF = &mf; @@ -595,9 +595,8 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, BasicBlock::const_iterator End, bool &HadTailCall) { - // Lower all of the non-terminator instructions. If a call is emitted - // as a tail call, cease emitting nodes for this block. Terminators - // are handled below. + // Lower the instructions. If a call is emitted as a tail call, cease emitting + // nodes for this block. for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) SDB->visit(*I); @@ -1182,8 +1181,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { if (!FastIS->lowerArguments()) { // Fast isel failed to lower these arguments ++NumFastIselFailLowerArguments; - if (EnableFastISelAbortArgs) - llvm_unreachable("FastISel didn't lower all arguments"); + if (EnableFastISelAbort > 1) + report_fatal_error("FastISel didn't lower all arguments"); // Use SelectionDAG argument lowering LowerArguments(Fn); @@ -1252,6 +1251,10 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { dbgs() << "FastISel missed call: "; Inst->dump(); } + if (EnableFastISelAbort > 2) + // FastISel selector couldn't handle something and bailed. + // For the purpose of debugging, just abort. + report_fatal_error("FastISel didn't select the entire block"); if (!Inst->getType()->isVoidTy() && !Inst->use_empty()) { unsigned &R = FuncInfo->ValueMap[Inst]; @@ -1279,24 +1282,24 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { continue; } - if (isa(Inst) && !isa(Inst)) { - // Don't abort, and use a different message for terminator misses. - NumFastIselFailures += NumFastIselRemaining; - if (EnableFastISelVerbose || EnableFastISelAbort) { + bool ShouldAbort = EnableFastISelAbort; + if (EnableFastISelVerbose || EnableFastISelAbort) { + if (isa(Inst)) { + // Use a different message for terminator misses. dbgs() << "FastISel missed terminator: "; - Inst->dump(); - } - } else { - NumFastIselFailures += NumFastIselRemaining; - if (EnableFastISelVerbose || EnableFastISelAbort) { + // Don't abort unless for terminator unless the level is really high + ShouldAbort = (EnableFastISelAbort > 2); + } else { dbgs() << "FastISel miss: "; - Inst->dump(); } - if (EnableFastISelAbort) - // The "fast" selector couldn't handle something and bailed. - // For the purpose of debugging, just abort. - llvm_unreachable("FastISel didn't select the entire block"); + Inst->dump(); } + if (ShouldAbort) + // FastISel selector couldn't handle something and bailed. + // For the purpose of debugging, just abort. + report_fatal_error("FastISel didn't select the entire block"); + + NumFastIselFailures += NumFastIselRemaining; break; } @@ -1775,9 +1778,23 @@ SelectInlineAsmMemoryOperands(std::vector &Ops) { } else { assert(InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"); + + unsigned TiedToOperand; + if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) { + // We need the constraint ID from the operand this is tied to. + unsigned CurOp = InlineAsm::Op_FirstOperand; + Flags = cast(InOps[CurOp])->getZExtValue(); + for (; TiedToOperand; --TiedToOperand) { + CurOp += InlineAsm::getNumOperandRegisters(Flags)+1; + Flags = cast(InOps[CurOp])->getZExtValue(); + } + } + // Otherwise, this is a memory operand. Ask the target to select it. std::vector SelOps; - if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps)) + if (SelectInlineAsmMemoryOperand(InOps[i+1], + InlineAsm::getMemoryConstraintID(Flags), + SelOps)) report_fatal_error("Could not match memory address. Inline asm" " failure!"); @@ -1933,7 +1950,7 @@ SDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) { std::vector Ops(N->op_begin(), N->op_end()); SelectInlineAsmMemoryOperands(Ops); - EVT VTs[] = { MVT::Other, MVT::Glue }; + const EVT VTs[] = {MVT::Other, MVT::Glue}; SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), VTs, Ops); New->setNodeId(-1); return New.getNode(); diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 1271f6b..3cc7a98 100644 --- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -224,6 +224,7 @@ static void removeDuplicatesGCPtrs(SmallVectorImpl &Bases, /// call node. Also update NodeMap so that getValue(statepoint) will /// reference lowered call result static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite, + MachineBasicBlock *LandingPad, SelectionDAGBuilder &Builder) { ImmutableCallSite CS(StatepointSite.getCallSite()); @@ -245,15 +246,29 @@ static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite, Tmp->setTailCall(CS.isTailCall()); Tmp->setCallingConv(CS.getCallingConv()); Tmp->setAttributes(CS.getAttributes()); - Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false); + Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false, LandingPad); // Handle the return value of the call iff any. const bool HasDef = !Tmp->getType()->isVoidTy(); if (HasDef) { - // The value of the statepoint itself will be the value of call itself. - // We'll replace the actually call node shortly. gc_result will grab - // this value. - Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp)); + if (CS.isInvoke()) { + // Result value will be used in different basic block for invokes + // so we need to export it now. But statepoint call has a different type + // than the actuall call. It means that standart exporting mechanism will + // create register of the wrong type. So instead we need to create + // register with correct type and save value into it manually. + // TODO: To eliminate this problem we can remove gc.result intrinsics + // completelly and make statepoint call to return a tuple. + unsigned reg = Builder.FuncInfo.CreateRegs(Tmp->getType()); + Builder.CopyValueToVirtualRegister(Tmp, reg); + Builder.FuncInfo.ValueMap[CS.getInstruction()] = reg; + } + else { + // The value of the statepoint itself will be the value of call itself. + // We'll replace the actually call node shortly. gc_result will grab + // this value. + Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp)); + } } else { // The token value is never used from here on, just generate a poison value Builder.setValue(CS.getInstruction(), Builder.DAG.getIntPtrConstant(-1)); @@ -267,6 +282,15 @@ static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite, // Search for the call node // The following code is essentially reverse engineering X86's // LowerCallTo. + // We are expecting DAG to have the following form: + // ch = eh_label (only in case of invoke statepoint) + // ch, glue = callseq_start ch + // ch, glue = X86::Call ch, glue + // ch, glue = callseq_end ch, glue + // ch = eh_label ch (only in case of invoke statepoint) + // + // DAG root will be either last eh_label or callseq_end. + SDNode *CallNode = nullptr; // We just emitted a call, so it should be last thing generated @@ -276,8 +300,11 @@ static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite, SDNode *CallEnd = Chain.getNode(); int Sanity = 0; while (CallEnd->getOpcode() != ISD::CALLSEQ_END) { - CallEnd = CallEnd->getGluedNode(); - assert(CallEnd && "Can not find call node"); + assert(CallEnd->getNumOperands() >= 1 && + CallEnd->getOperand(0).getValueType() == MVT::Other); + + CallEnd = CallEnd->getOperand(0).getNode(); + assert(Sanity < 20 && "should have found call end already"); Sanity++; } @@ -506,7 +533,9 @@ void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) { LowerStatepoint(ImmutableStatepoint(&CI)); } -void SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP) { +void +SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, + MachineBasicBlock *LandingPad/*=nullptr*/) { // The basic scheme here is that information about both the original call and // the safepoint is encoded in the CallInst. We create a temporary call and // lower it, then reverse engineer the calling sequence. @@ -542,13 +571,12 @@ void SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP) { } #endif - // Lower statepoint vmstate and gcstate arguments SmallVector LoweredArgs; lowerStatepointMetaArgs(LoweredArgs, ISP, *this); // Get call node, we will replace it later with statepoint - SDNode *CallNode = lowerCallFromStatepoint(ISP, *this); + SDNode *CallNode = lowerCallFromStatepoint(ISP, LandingPad, *this); // Construct the actual STATEPOINT node with all the appropriate arguments // and return values. @@ -634,7 +662,24 @@ void SelectionDAGBuilder::visitGCResult(const CallInst &CI) { assert(isStatepoint(I) && "first argument must be a statepoint token"); - setValue(&CI, getValue(I)); + if (isa(I)) { + // For invokes we should have stored call result in a virtual register. + // We can not use default getValue() functionality to copy value from this + // register because statepoint and actuall call return types can be + // different, and getValue() will use CopyFromReg of the wrong type, + // which is always i32 in our case. + PointerType *CalleeType = cast( + ImmutableStatepoint(I).actualCallee()->getType()); + Type *RetTy = cast( + CalleeType->getElementType())->getReturnType(); + SDValue CopyFromReg = getCopyFromRegs(I, RetTy); + + assert(CopyFromReg.getNode()); + setValue(&CI, CopyFromReg); + } + else { + setValue(&CI, getValue(I)); + } } void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) { diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 0a3c926..ddbf0b2 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -96,18 +96,21 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, for (unsigned i = 0; i != NumOps; ++i) { Entry.Node = Ops[i]; Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); - Entry.isSExt = isSigned; - Entry.isZExt = !isSigned; + Entry.isSExt = shouldSignExtendTypeInLibCall(Ops[i].getValueType(), isSigned); + Entry.isZExt = !shouldSignExtendTypeInLibCall(Ops[i].getValueType(), isSigned); Args.push_back(Entry); } + if (LC == RTLIB::UNKNOWN_LIBCALL) + report_fatal_error("Unsupported library call operation!"); SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC), getPointerTy()); Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); TargetLowering::CallLoweringInfo CLI(DAG); + bool signExtend = shouldSignExtendTypeInLibCall(RetVT, isSigned); CLI.setDebugLoc(dl).setChain(DAG.getEntryNode()) .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) .setNoReturn(doesNotReturn).setDiscardResult(!isReturnValueUsed) - .setSExtResult(isSigned).setZExtResult(!isSigned); + .setSExtResult(signExtend).setZExtResult(!signExtend); return LowerCallTo(CLI); } diff --git a/lib/CodeGen/ShadowStackGCLowering.cpp b/lib/CodeGen/ShadowStackGCLowering.cpp index f6393a5..66a6a3c 100644 --- a/lib/CodeGen/ShadowStackGCLowering.cpp +++ b/lib/CodeGen/ShadowStackGCLowering.cpp @@ -53,10 +53,10 @@ private: Type *GetConcreteStackEntryType(Function &F); void CollectRoots(Function &F); static GetElementPtrInst *CreateGEP(LLVMContext &Context, IRBuilder<> &B, - Value *BasePtr, int Idx1, + Type *Ty, Value *BasePtr, int Idx1, const char *Name); static GetElementPtrInst *CreateGEP(LLVMContext &Context, IRBuilder<> &B, - Value *BasePtr, int Idx1, int Idx2, + Type *Ty, Value *BasePtr, int Idx1, int Idx2, const char *Name); }; } @@ -343,13 +343,14 @@ void ShadowStackGCLowering::CollectRoots(Function &F) { } GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, - IRBuilder<> &B, Value *BasePtr, - int Idx, int Idx2, - const char *Name) { + IRBuilder<> &B, Type *Ty, + Value *BasePtr, int Idx, + int Idx2, + const char *Name) { Value *Indices[] = {ConstantInt::get(Type::getInt32Ty(Context), 0), ConstantInt::get(Type::getInt32Ty(Context), Idx), ConstantInt::get(Type::getInt32Ty(Context), Idx2)}; - Value *Val = B.CreateGEP(BasePtr, Indices, Name); + Value *Val = B.CreateGEP(Ty, BasePtr, Indices, Name); assert(isa(Val) && "Unexpected folded constant"); @@ -357,11 +358,11 @@ GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, } GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, - IRBuilder<> &B, Value *BasePtr, + IRBuilder<> &B, Type *Ty, Value *BasePtr, int Idx, const char *Name) { Value *Indices[] = {ConstantInt::get(Type::getInt32Ty(Context), 0), ConstantInt::get(Type::getInt32Ty(Context), Idx)}; - Value *Val = B.CreateGEP(BasePtr, Indices, Name); + Value *Val = B.CreateGEP(Ty, BasePtr, Indices, Name); assert(isa(Val) && "Unexpected folded constant"); @@ -402,14 +403,15 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { // Initialize the map pointer and load the current head of the shadow stack. Instruction *CurrentHead = AtEntry.CreateLoad(Head, "gc_currhead"); - Instruction *EntryMapPtr = - CreateGEP(Context, AtEntry, StackEntry, 0, 1, "gc_frame.map"); + Instruction *EntryMapPtr = CreateGEP(Context, AtEntry, ConcreteStackEntryTy, + StackEntry, 0, 1, "gc_frame.map"); AtEntry.CreateStore(FrameMap, EntryMapPtr); // After all the allocas... for (unsigned I = 0, E = Roots.size(); I != E; ++I) { // For each root, find the corresponding slot in the aggregate... - Value *SlotPtr = CreateGEP(Context, AtEntry, StackEntry, 1 + I, "gc_root"); + Value *SlotPtr = CreateGEP(Context, AtEntry, ConcreteStackEntryTy, + StackEntry, 1 + I, "gc_root"); // And use it in lieu of the alloca. AllocaInst *OriginalAlloca = Roots[I].second; @@ -426,10 +428,10 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { AtEntry.SetInsertPoint(IP->getParent(), IP); // Push the entry onto the shadow stack. - Instruction *EntryNextPtr = - CreateGEP(Context, AtEntry, StackEntry, 0, 0, "gc_frame.next"); - Instruction *NewHeadVal = - CreateGEP(Context, AtEntry, StackEntry, 0, "gc_newhead"); + Instruction *EntryNextPtr = CreateGEP(Context, AtEntry, ConcreteStackEntryTy, + StackEntry, 0, 0, "gc_frame.next"); + Instruction *NewHeadVal = CreateGEP(Context, AtEntry, ConcreteStackEntryTy, + StackEntry, 0, "gc_newhead"); AtEntry.CreateStore(CurrentHead, EntryNextPtr); AtEntry.CreateStore(NewHeadVal, Head); @@ -439,7 +441,8 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { // Pop the entry from the shadow stack. Don't reuse CurrentHead from // AtEntry, since that would make the value live for the entire function. Instruction *EntryNextPtr2 = - CreateGEP(Context, *AtExit, StackEntry, 0, 0, "gc_frame.next"); + CreateGEP(Context, *AtExit, ConcreteStackEntryTy, StackEntry, 0, 0, + "gc_frame.next"); Value *SavedHead = AtExit->CreateLoad(EntryNextPtr2, "gc_savedhead"); AtExit->CreateStore(SavedHead, Head); } diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp index 35e4292..2335a88 100644 --- a/lib/CodeGen/SjLjEHPrepare.cpp +++ b/lib/CodeGen/SjLjEHPrepare.cpp @@ -128,7 +128,8 @@ void SjLjEHPrepare::insertCallSiteStore(Instruction *I, int Number) { Value *Zero = ConstantInt::get(Int32Ty, 0); Value *One = ConstantInt::get(Int32Ty, 1); Value *Idxs[2] = { Zero, One }; - Value *CallSite = Builder.CreateGEP(FuncCtx, Idxs, "call_site"); + Value *CallSite = + Builder.CreateGEP(FunctionContextTy, FuncCtx, Idxs, "call_site"); // Insert a store of the call-site number ConstantInt *CallSiteNoC = diff --git a/lib/CodeGen/SlotIndexes.cpp b/lib/CodeGen/SlotIndexes.cpp index d46621d..025ae70 100644 --- a/lib/CodeGen/SlotIndexes.cpp +++ b/lib/CodeGen/SlotIndexes.cpp @@ -127,7 +127,7 @@ void SlotIndexes::renumberIndexes() { void SlotIndexes::renumberIndexes(IndexList::iterator curItr) { // Number indexes with half the default spacing so we can catch up quickly. const unsigned Space = SlotIndex::InstrDist/2; - assert((Space & 3) == 0 && "InstrDist must be a multiple of 2*NUM"); + static_assert((Space & 3) == 0, "InstrDist must be a multiple of 2*NUM"); IndexList::iterator startItr = std::prev(curItr); unsigned index = startItr->getIndex(); diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp index faf94b6..7572803 100644 --- a/lib/CodeGen/StackColoring.cpp +++ b/lib/CodeGen/StackColoring.cpp @@ -364,7 +364,7 @@ void StackColoring::calculateLocalLiveness() { } } - BBSet = NextBBSet; + BBSet = std::move(NextBBSet); }// while changed. } diff --git a/lib/CodeGen/StackMapLivenessAnalysis.cpp b/lib/CodeGen/StackMapLivenessAnalysis.cpp index 767f43a..d88be57 100644 --- a/lib/CodeGen/StackMapLivenessAnalysis.cpp +++ b/lib/CodeGen/StackMapLivenessAnalysis.cpp @@ -14,24 +14,24 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionAnalysis.h" +#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/StackMapLivenessAnalysis.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; #define DEBUG_TYPE "stackmaps" -namespace llvm { -cl::opt EnablePatchPointLiveness("enable-patchpoint-liveness", - cl::Hidden, cl::init(true), - cl::desc("Enable PatchPoint Liveness Analysis Pass")); -} +static cl::opt EnablePatchPointLiveness( + "enable-patchpoint-liveness", cl::Hidden, cl::init(true), + cl::desc("Enable PatchPoint Liveness Analysis Pass")); STATISTIC(NumStackMapFuncVisited, "Number of functions visited"); STATISTIC(NumStackMapFuncSkipped, "Number of functions skipped"); @@ -39,6 +39,46 @@ STATISTIC(NumBBsVisited, "Number of basic blocks visited"); STATISTIC(NumBBsHaveNoStackmap, "Number of basic blocks with no stackmap"); STATISTIC(NumStackMaps, "Number of StackMaps visited"); +namespace { +/// \brief This pass calculates the liveness information for each basic block in +/// a function and attaches the register live-out information to a patchpoint +/// intrinsic if present. +/// +/// This pass can be disabled via the -enable-patchpoint-liveness=false flag. +/// The pass skips functions that don't have any patchpoint intrinsics. The +/// information provided by this pass is optional and not required by the +/// aformentioned intrinsic to function. +class StackMapLiveness : public MachineFunctionPass { + MachineFunction *MF; + const TargetRegisterInfo *TRI; + LivePhysRegs LiveRegs; + +public: + static char ID; + + /// \brief Default construct and initialize the pass. + StackMapLiveness(); + + /// \brief Tell the pass manager which passes we depend on and what + /// information we preserve. + void getAnalysisUsage(AnalysisUsage &AU) const override; + + /// \brief Calculate the liveness information for the given machine function. + bool runOnMachineFunction(MachineFunction &MF) override; + +private: + /// \brief Performs the actual liveness calculation for the function. + bool calculateLiveness(); + + /// \brief Add the current register live set to the instruction. + void addLiveOutSetToMI(MachineInstr &MI); + + /// \brief Create a register mask and initialize it with the registers from + /// the register live set. + uint32_t *createRegisterMask() const; +}; +} // namespace + char StackMapLiveness::ID = 0; char &llvm::StackMapLivenessID = StackMapLiveness::ID; INITIALIZE_PASS(StackMapLiveness, "stackmap-liveness", @@ -60,18 +100,18 @@ void StackMapLiveness::getAnalysisUsage(AnalysisUsage &AU) const { } /// Calculate the liveness information for the given machine function. -bool StackMapLiveness::runOnMachineFunction(MachineFunction &_MF) { +bool StackMapLiveness::runOnMachineFunction(MachineFunction &MF) { if (!EnablePatchPointLiveness) return false; - DEBUG(dbgs() << "********** COMPUTING STACKMAP LIVENESS: " - << _MF.getName() << " **********\n"); - MF = &_MF; - TRI = MF->getSubtarget().getRegisterInfo(); + DEBUG(dbgs() << "********** COMPUTING STACKMAP LIVENESS: " << MF.getName() + << " **********\n"); + this->MF = &MF; + TRI = MF.getSubtarget().getRegisterInfo(); ++NumStackMapFuncVisited; // Skip this function if there are no patchpoints to process. - if (!MF->getFrameInfo()->hasPatchPoint()) { + if (!MF.getFrameInfo()->hasPatchPoint()) { ++NumStackMapFuncSkipped; return false; } diff --git a/lib/CodeGen/StackMaps.cpp b/lib/CodeGen/StackMaps.cpp index 5d46419..aa18dea 100644 --- a/lib/CodeGen/StackMaps.cpp +++ b/lib/CodeGen/StackMaps.cpp @@ -19,8 +19,6 @@ #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOpcodes.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -76,10 +74,21 @@ StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) { llvm_unreachable("Unsupported stackmap version!"); } +/// Go up the super-register chain until we hit a valid dwarf register number. +static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) { + int RegNo = TRI->getDwarfRegNum(Reg, false); + for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNo < 0; ++SR) + RegNo = TRI->getDwarfRegNum(*SR, false); + + assert(RegNo >= 0 && "Invalid Dwarf register number."); + return (unsigned) RegNo; +} + MachineInstr::const_mop_iterator StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, MachineInstr::const_mop_iterator MOE, LocationVec &Locs, LiveOutVec &LiveOuts) const { + const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo(); if (MOI->isImm()) { switch (MOI->getImm()) { default: llvm_unreachable("Unrecognized operand type."); @@ -89,7 +98,8 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, Size /= 8; unsigned Reg = (++MOI)->getReg(); int64_t Imm = (++MOI)->getImm(); - Locs.push_back(Location(StackMaps::Location::Direct, Size, Reg, Imm)); + Locs.push_back(Location(StackMaps::Location::Direct, Size, + getDwarfRegNum(Reg, TRI), Imm)); break; } case StackMaps::IndirectMemRefOp: { @@ -97,7 +107,8 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, assert(Size > 0 && "Need a valid size for indirect memory locations."); unsigned Reg = (++MOI)->getReg(); int64_t Imm = (++MOI)->getImm(); - Locs.push_back(Location(StackMaps::Location::Indirect, Size, Reg, Imm)); + Locs.push_back(Location(StackMaps::Location::Indirect, Size, + getDwarfRegNum(Reg, TRI), Imm)); break; } case StackMaps::ConstantOp: { @@ -122,12 +133,18 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, assert(TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) && "Virtreg operands should have been rewritten before now."); - const TargetRegisterClass *RC = - AP.TM.getSubtargetImpl()->getRegisterInfo()->getMinimalPhysRegClass( - MOI->getReg()); + const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(MOI->getReg()); assert(!MOI->getSubReg() && "Physical subreg still around."); + + unsigned Offset = 0; + unsigned RegNo = getDwarfRegNum(MOI->getReg(), TRI); + unsigned LLVMRegNo = TRI->getLLVMRegNum(RegNo, false); + unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNo, MOI->getReg()); + if (SubRegIdx) + Offset = TRI->getSubRegIdxOffset(SubRegIdx); + Locs.push_back( - Location(Location::Register, RC->getSize(), MOI->getReg(), 0)); + Location(Location::Register, RC->getSize(), RegNo, Offset)); return ++MOI; } @@ -137,14 +154,74 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, return ++MOI; } -/// Go up the super-register chain until we hit a valid dwarf register number. -static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) { - int RegNo = TRI->getDwarfRegNum(Reg, false); - for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNo < 0; ++SR) - RegNo = TRI->getDwarfRegNum(*SR, false); +void StackMaps::print(raw_ostream &OS) { + const TargetRegisterInfo *TRI = + AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr; + OS << WSMP << "callsites:\n"; + for (const auto &CSI : CSInfos) { + const LocationVec &CSLocs = CSI.Locations; + const LiveOutVec &LiveOuts = CSI.LiveOuts; - assert(RegNo >= 0 && "Invalid Dwarf register number."); - return (unsigned) RegNo; + OS << WSMP << "callsite " << CSI.ID << "\n"; + OS << WSMP << " has " << CSLocs.size() << " locations\n"; + + unsigned OperIdx = 0; + for (const auto &Loc : CSLocs) { + OS << WSMP << " Loc " << OperIdx << ": "; + switch (Loc.LocType) { + case Location::Unprocessed: + OS << ""; + break; + case Location::Register: + OS << "Register "; + if (TRI) + OS << TRI->getName(Loc.Reg); + else + OS << Loc.Reg; + break; + case Location::Direct: + OS << "Direct "; + if (TRI) + OS << TRI->getName(Loc.Reg); + else + OS << Loc.Reg; + if (Loc.Offset) + OS << " + " << Loc.Offset; + break; + case Location::Indirect: + OS << "Indirect "; + if (TRI) + OS << TRI->getName(Loc.Reg); + else + OS << Loc.Reg; + OS << "+" << Loc.Offset; + break; + case Location::Constant: + OS << "Constant " << Loc.Offset; + break; + case Location::ConstantIndex: + OS << "Constant Index " << Loc.Offset; + break; + } + OS << " [encoding: .byte " << Loc.LocType << ", .byte " << Loc.Size + << ", .short " << Loc.Reg << ", .int " << Loc.Offset << "]\n"; + OperIdx++; + } + + OS << WSMP << " has " << LiveOuts.size() << " live-out registers\n"; + + OperIdx = 0; + for (const auto &LO : LiveOuts) { + OS << WSMP << " LO " << OperIdx << ": "; + if (TRI) + OS << TRI->getName(LO.Reg); + else + OS << LO.Reg; + OS << " [encoding: .short " << LO.RegNo << ", .byte 0, .byte " + << LO.Size << "]\n"; + OperIdx++; + } + } } /// Create a live-out register record for the given register Reg. @@ -160,7 +237,7 @@ StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const { StackMaps::LiveOutVec StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const { assert(Mask && "No register mask specified"); - const TargetRegisterInfo *TRI = AP.TM.getSubtargetImpl()->getRegisterInfo(); + const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo(); LiveOutVec LiveOuts; // Create a LiveOutReg for each bit that is set in the register mask. @@ -383,16 +460,13 @@ void StackMaps::emitConstantPoolEntries(MCStreamer &OS) { /// 0x3, Indirect, [Reg + Offset] (spilled value) /// 0x4, Constant, Offset (small constant) /// 0x5, ConstIndex, Constants[Offset] (large constant) -void StackMaps::emitCallsiteEntries(MCStreamer &OS, - const TargetRegisterInfo *TRI) { +void StackMaps::emitCallsiteEntries(MCStreamer &OS) { + DEBUG(print(dbgs())); // Callsite entries. - DEBUG(dbgs() << WSMP << "callsites:\n"); for (const auto &CSI : CSInfos) { const LocationVec &CSLocs = CSI.Locations; const LiveOutVec &LiveOuts = CSI.LiveOuts; - DEBUG(dbgs() << WSMP << "callsite " << CSI.ID << "\n"); - // Verify stack map entry. It's better to communicate a problem to the // runtime than crash in case of in-process compilation. Currently, we do // simple overflow checks, but we may eventually communicate other @@ -413,83 +487,20 @@ void StackMaps::emitCallsiteEntries(MCStreamer &OS, // Reserved for flags. OS.EmitIntValue(0, 2); - - DEBUG(dbgs() << WSMP << " has " << CSLocs.size() << " locations\n"); - OS.EmitIntValue(CSLocs.size(), 2); - unsigned OperIdx = 0; for (const auto &Loc : CSLocs) { - unsigned RegNo = 0; - int Offset = Loc.Offset; - if(Loc.Reg) { - RegNo = getDwarfRegNum(Loc.Reg, TRI); - - // If this is a register location, put the subregister byte offset in - // the location offset. - if (Loc.LocType == Location::Register) { - assert(!Loc.Offset && "Register location should have zero offset"); - unsigned LLVMRegNo = TRI->getLLVMRegNum(RegNo, false); - unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNo, Loc.Reg); - if (SubRegIdx) - Offset = TRI->getSubRegIdxOffset(SubRegIdx); - } - } - else { - assert(Loc.LocType != Location::Register && - "Missing location register"); - } - - DEBUG(dbgs() << WSMP << " Loc " << OperIdx << ": "; - switch (Loc.LocType) { - case Location::Unprocessed: - dbgs() << ""; - break; - case Location::Register: - dbgs() << "Register " << TRI->getName(Loc.Reg); - break; - case Location::Direct: - dbgs() << "Direct " << TRI->getName(Loc.Reg); - if (Loc.Offset) - dbgs() << " + " << Loc.Offset; - break; - case Location::Indirect: - dbgs() << "Indirect " << TRI->getName(Loc.Reg) - << " + " << Loc.Offset; - break; - case Location::Constant: - dbgs() << "Constant " << Loc.Offset; - break; - case Location::ConstantIndex: - dbgs() << "Constant Index " << Loc.Offset; - break; - } - dbgs() << " [encoding: .byte " << Loc.LocType - << ", .byte " << Loc.Size - << ", .short " << RegNo - << ", .int " << Offset << "]\n"; - ); - OS.EmitIntValue(Loc.LocType, 1); OS.EmitIntValue(Loc.Size, 1); - OS.EmitIntValue(RegNo, 2); - OS.EmitIntValue(Offset, 4); - OperIdx++; + OS.EmitIntValue(Loc.Reg, 2); + OS.EmitIntValue(Loc.Offset, 4); } - DEBUG(dbgs() << WSMP << " has " << LiveOuts.size() - << " live-out registers\n"); - // Num live-out registers and padding to align to 4 byte. OS.EmitIntValue(0, 2); OS.EmitIntValue(LiveOuts.size(), 2); - OperIdx = 0; for (const auto &LO : LiveOuts) { - DEBUG(dbgs() << WSMP << " LO " << OperIdx << ": " - << TRI->getName(LO.Reg) - << " [encoding: .short " << LO.RegNo - << ", .byte 0, .byte " << LO.Size << "]\n"); OS.EmitIntValue(LO.RegNo, 2); OS.EmitIntValue(0, 1); OS.EmitIntValue(LO.Size, 1); @@ -512,7 +523,6 @@ void StackMaps::serializeToStackMapSection() { MCContext &OutContext = AP.OutStreamer.getContext(); MCStreamer &OS = AP.OutStreamer; - const TargetRegisterInfo *TRI = AP.TM.getSubtargetImpl()->getRegisterInfo(); // Create the section. const MCSection *StackMapSection = @@ -527,7 +537,7 @@ void StackMaps::serializeToStackMapSection() { emitStackmapHeader(OS); emitFunctionFrameRecords(OS); emitConstantPoolEntries(OS); - emitCallsiteEntries(OS, TRI); + emitCallsiteEntries(OS); OS.AddBlankLine(); // Clean up. diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp index cc72e5e..a5a175f 100644 --- a/lib/CodeGen/StackSlotColoring.cpp +++ b/lib/CodeGen/StackSlotColoring.cpp @@ -184,10 +184,18 @@ void StackSlotColoring::InitializeSlots() { UsedColors.resize(LastFI); Assignments.resize(LastFI); + typedef std::iterator_traits::value_type Pair; + SmallVector Intervals; + Intervals.reserve(LS->getNumIntervals()); + for (auto &I : *LS) + Intervals.push_back(&I); + std::sort(Intervals.begin(), Intervals.end(), + [](Pair *LHS, Pair *RHS) { return LHS->first < RHS->first; }); + // Gather all spill slots into a list. DEBUG(dbgs() << "Spill slot intervals:\n"); - for (LiveStacks::iterator i = LS->begin(), e = LS->end(); i != e; ++i) { - LiveInterval &li = i->second; + for (auto *I : Intervals) { + LiveInterval &li = I->second; DEBUG(li.dump()); int FI = TargetRegisterInfo::stackSlot2Index(li.reg); if (MFI->isDeadObjectIndex(FI)) diff --git a/lib/CodeGen/TargetInstrInfo.cpp b/lib/CodeGen/TargetInstrInfo.cpp index 2566c1f..38725b5 100644 --- a/lib/CodeGen/TargetInstrInfo.cpp +++ b/lib/CodeGen/TargetInstrInfo.cpp @@ -285,21 +285,20 @@ bool TargetInstrInfo::hasStoreToStackSlot(const MachineInstr *MI, bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC, unsigned SubIdx, unsigned &Size, unsigned &Offset, - const TargetMachine *TM) const { + const MachineFunction &MF) const { if (!SubIdx) { Size = RC->getSize(); Offset = 0; return true; } - unsigned BitSize = - TM->getSubtargetImpl()->getRegisterInfo()->getSubRegIdxSize(SubIdx); + const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); + unsigned BitSize = TRI->getSubRegIdxSize(SubIdx); // Convert bit size to byte size to be consistent with // MCRegisterClass::getSize(). if (BitSize % 8) return false; - int BitOffset = - TM->getSubtargetImpl()->getRegisterInfo()->getSubRegIdxOffset(SubIdx); + int BitOffset = TRI->getSubRegIdxOffset(SubIdx); if (BitOffset < 0 || BitOffset % 8) return false; @@ -308,7 +307,7 @@ bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC, assert(RC->getSize() >= (Offset + Size) && "bad subregister range"); - if (!TM->getDataLayout()->isLittleEndian()) { + if (!MF.getTarget().getDataLayout()->isLittleEndian()) { Offset = RC->getSize() - (Offset + Size); } return true; @@ -377,16 +376,13 @@ void TargetInstrInfo::getNoopForMachoTarget(MCInst &NopInst) const { llvm_unreachable("Not a MachO target"); } -bool TargetInstrInfo:: -canFoldMemoryOperand(const MachineInstr *MI, - const SmallVectorImpl &Ops) const { +bool TargetInstrInfo::canFoldMemoryOperand(const MachineInstr *MI, + ArrayRef Ops) const { return MI->isCopy() && Ops.size() == 1 && canFoldCopy(MI, Ops[0]); } -static MachineInstr* foldPatchpoint(MachineFunction &MF, - MachineInstr *MI, - const SmallVectorImpl &Ops, - int FrameIndex, +static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr *MI, + ArrayRef Ops, int FrameIndex, const TargetInstrInfo &TII) { unsigned StartIdx = 0; switch (MI->getOpcode()) { @@ -405,9 +401,8 @@ static MachineInstr* foldPatchpoint(MachineFunction &MF, // Return false if any operands requested for folding are not foldable (not // part of the stackmap's live values). - for (SmallVectorImpl::const_iterator I = Ops.begin(), E = Ops.end(); - I != E; ++I) { - if (*I < StartIdx) + for (unsigned Op : Ops) { + if (Op < StartIdx) return nullptr; } @@ -427,8 +422,8 @@ static MachineInstr* foldPatchpoint(MachineFunction &MF, // Compute the spill slot size and offset. const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(MO.getReg()); - bool Valid = TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, - SpillOffset, &MF.getTarget()); + bool Valid = + TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF); if (!Valid) report_fatal_error("cannot spill patchpoint subregister operand"); MIB.addImm(StackMaps::IndirectMemRefOp); @@ -448,10 +443,9 @@ static MachineInstr* foldPatchpoint(MachineFunction &MF, /// operand folded, otherwise NULL is returned. The client is responsible for /// removing the old instruction and adding the new one in the instruction /// stream. -MachineInstr* -TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, - const SmallVectorImpl &Ops, - int FI) const { +MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, + ArrayRef Ops, + int FI) const { unsigned Flags = 0; for (unsigned i = 0, e = Ops.size(); i != e; ++i) if (MI->getOperand(Ops[i]).isDef()) @@ -517,10 +511,9 @@ TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, /// foldMemoryOperand - Same as the previous version except it allows folding /// of any load and store from / to any address, not just from a specific /// stack slot. -MachineInstr* -TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, - const SmallVectorImpl &Ops, - MachineInstr* LoadMI) const { +MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, + ArrayRef Ops, + MachineInstr *LoadMI) const { assert(LoadMI->canFoldAsLoad() && "LoadMI isn't foldable!"); #ifndef NDEBUG for (unsigned i = 0, e = Ops.size(); i != e; ++i) diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index 9048a44..58a6d52 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -664,6 +664,44 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { return UNKNOWN_LIBCALL; } +RTLIB::Libcall RTLIB::getATOMIC(unsigned Opc, MVT VT) { +#define OP_TO_LIBCALL(Name, Enum) \ + case Name: \ + switch (VT.SimpleTy) { \ + default: \ + return UNKNOWN_LIBCALL; \ + case MVT::i8: \ + return Enum##_1; \ + case MVT::i16: \ + return Enum##_2; \ + case MVT::i32: \ + return Enum##_4; \ + case MVT::i64: \ + return Enum##_8; \ + case MVT::i128: \ + return Enum##_16; \ + } + + switch (Opc) { + OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET) + OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN) + OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN) + } + +#undef OP_TO_LIBCALL + + return UNKNOWN_LIBCALL; +} + /// InitCmpLibcallCCs - Set default comparison libcall CC. /// static void InitCmpLibcallCCs(ISD::CondCode *CCs) { @@ -695,12 +733,11 @@ static void InitCmpLibcallCCs(ISD::CondCode *CCs) { } /// NOTE: The TargetMachine owns TLOF. -TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) - : TM(tm), DL(TM.getDataLayout()) { +TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) { initActions(); // Perform these initializations only once. - IsLittleEndian = DL->isLittleEndian(); + IsLittleEndian = getDataLayout()->isLittleEndian(); MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove = 8; MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize = MaxStoresPerMemmoveOptSize = 4; @@ -792,58 +829,21 @@ void TargetLoweringBase::initActions() { setOperationAction(ISD::ConstantFP, MVT::f128, Expand); // These library functions default to expand. - setOperationAction(ISD::FLOG , MVT::f16, Expand); - setOperationAction(ISD::FLOG2, MVT::f16, Expand); - setOperationAction(ISD::FLOG10, MVT::f16, Expand); - setOperationAction(ISD::FEXP , MVT::f16, Expand); - setOperationAction(ISD::FEXP2, MVT::f16, Expand); - setOperationAction(ISD::FFLOOR, MVT::f16, Expand); - setOperationAction(ISD::FMINNUM, MVT::f16, Expand); - setOperationAction(ISD::FMAXNUM, MVT::f16, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f16, Expand); - setOperationAction(ISD::FCEIL, MVT::f16, Expand); - setOperationAction(ISD::FRINT, MVT::f16, Expand); - setOperationAction(ISD::FTRUNC, MVT::f16, Expand); - setOperationAction(ISD::FROUND, MVT::f16, Expand); - setOperationAction(ISD::FLOG , MVT::f32, Expand); - setOperationAction(ISD::FLOG2, MVT::f32, Expand); - setOperationAction(ISD::FLOG10, MVT::f32, Expand); - setOperationAction(ISD::FEXP , MVT::f32, Expand); - setOperationAction(ISD::FEXP2, MVT::f32, Expand); - setOperationAction(ISD::FFLOOR, MVT::f32, Expand); - setOperationAction(ISD::FMINNUM, MVT::f32, Expand); - setOperationAction(ISD::FMAXNUM, MVT::f32, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f32, Expand); - setOperationAction(ISD::FCEIL, MVT::f32, Expand); - setOperationAction(ISD::FRINT, MVT::f32, Expand); - setOperationAction(ISD::FTRUNC, MVT::f32, Expand); - setOperationAction(ISD::FROUND, MVT::f32, Expand); - setOperationAction(ISD::FLOG , MVT::f64, Expand); - setOperationAction(ISD::FLOG2, MVT::f64, Expand); - setOperationAction(ISD::FLOG10, MVT::f64, Expand); - setOperationAction(ISD::FEXP , MVT::f64, Expand); - setOperationAction(ISD::FEXP2, MVT::f64, Expand); - setOperationAction(ISD::FFLOOR, MVT::f64, Expand); - setOperationAction(ISD::FMINNUM, MVT::f64, Expand); - setOperationAction(ISD::FMAXNUM, MVT::f64, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); - setOperationAction(ISD::FCEIL, MVT::f64, Expand); - setOperationAction(ISD::FRINT, MVT::f64, Expand); - setOperationAction(ISD::FTRUNC, MVT::f64, Expand); - setOperationAction(ISD::FROUND, MVT::f64, Expand); - setOperationAction(ISD::FLOG , MVT::f128, Expand); - setOperationAction(ISD::FLOG2, MVT::f128, Expand); - setOperationAction(ISD::FLOG10, MVT::f128, Expand); - setOperationAction(ISD::FEXP , MVT::f128, Expand); - setOperationAction(ISD::FEXP2, MVT::f128, Expand); - setOperationAction(ISD::FFLOOR, MVT::f128, Expand); - setOperationAction(ISD::FMINNUM, MVT::f128, Expand); - setOperationAction(ISD::FMAXNUM, MVT::f128, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f128, Expand); - setOperationAction(ISD::FCEIL, MVT::f128, Expand); - setOperationAction(ISD::FRINT, MVT::f128, Expand); - setOperationAction(ISD::FTRUNC, MVT::f128, Expand); - setOperationAction(ISD::FROUND, MVT::f128, Expand); + for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) { + setOperationAction(ISD::FLOG , VT, Expand); + setOperationAction(ISD::FLOG2, VT, Expand); + setOperationAction(ISD::FLOG10, VT, Expand); + setOperationAction(ISD::FEXP , VT, Expand); + setOperationAction(ISD::FEXP2, VT, Expand); + setOperationAction(ISD::FFLOOR, VT, Expand); + setOperationAction(ISD::FMINNUM, VT, Expand); + setOperationAction(ISD::FMAXNUM, VT, Expand); + setOperationAction(ISD::FNEARBYINT, VT, Expand); + setOperationAction(ISD::FCEIL, VT, Expand); + setOperationAction(ISD::FRINT, VT, Expand); + setOperationAction(ISD::FTRUNC, VT, Expand); + setOperationAction(ISD::FROUND, VT, Expand); + } // Default ISD::TRAP to expand (which turns it into abort). setOperationAction(ISD::TRAP, MVT::Other, Expand); @@ -859,7 +859,7 @@ MVT TargetLoweringBase::getPointerTy(uint32_t AS) const { } unsigned TargetLoweringBase::getPointerSizeInBits(uint32_t AS) const { - return DL->getPointerSizeInBits(AS); + return getDataLayout()->getPointerSizeInBits(AS); } unsigned TargetLoweringBase::getPointerTypeSizeInBits(Type *Ty) const { @@ -868,7 +868,7 @@ unsigned TargetLoweringBase::getPointerTypeSizeInBits(Type *Ty) const { } MVT TargetLoweringBase::getScalarShiftAmountTy(EVT LHSTy) const { - return MVT::getIntegerVT(8*DL->getPointerSize(0)); + return MVT::getIntegerVT(8 * getDataLayout()->getPointerSize(0)); } EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy) const { @@ -1144,6 +1144,10 @@ TargetLoweringBase::emitPatchPoint(MachineInstr *MI, /// findRepresentativeClass - Return the largest legal super-reg register class /// of the register class for the specified type and its associated "cost". +// This function is in TargetLowering because it uses RegClassForVT which would +// need to be moved to TargetRegisterInfo and would necessitate moving +// isTypeLegal over as well - a massive change that would just require +// TargetLowering having a TargetRegisterInfo class member that it would use. std::pair TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const { @@ -1498,7 +1502,7 @@ void llvm::GetReturnInfo(Type* ReturnType, AttributeSet attr, /// function arguments in the caller parameter area. This is the actual /// alignment, not its logarithm. unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty) const { - return DL->getABITypeAlignment(Ty); + return getDataLayout()->getABITypeAlignment(Ty); } //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index c1b34f7..bcf2aa7 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -31,6 +31,7 @@ #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCValue.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/ELF.h" #include "llvm/Support/ErrorHandling.h" @@ -244,22 +245,9 @@ static StringRef getSectionPrefixForGlobal(SectionKind Kind) { return ".data.rel.ro"; } -const MCSection *TargetLoweringObjectFileELF:: -SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler &Mang, const TargetMachine &TM) const { - unsigned Flags = getELFSectionFlags(Kind); - - // If we have -ffunction-section or -fdata-section then we should emit the - // global value to a uniqued section specifically for it. - bool EmitUniqueSection = false; - if (!(Flags & ELF::SHF_MERGE) && !Kind.isCommon()) { - if (Kind.isText()) - EmitUniqueSection = TM.getFunctionSections(); - else - EmitUniqueSection = TM.getDataSections(); - } - EmitUniqueSection |= GV->hasComdat(); - +static const MCSectionELF *selectELFSectionForGlobal( + MCContext &Ctx, const GlobalValue *GV, SectionKind Kind, Mangler &Mang, + const TargetMachine &TM, bool EmitUniqueSection, unsigned Flags) { unsigned EntrySize = 0; if (Kind.isMergeableCString()) { if (Kind.isMergeable2ByteCString()) { @@ -309,9 +297,29 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Name.push_back('.'); TM.getNameWithPrefix(Name, GV, Mang, true); } - return getContext().getELFSection(Name, getELFSectionType(Name, Kind), Flags, - EntrySize, Group, - EmitUniqueSection && !UniqueSectionNames); + return Ctx.getELFSection(Name, getELFSectionType(Name, Kind), Flags, + EntrySize, Group, + EmitUniqueSection && !UniqueSectionNames); +} + +const MCSection *TargetLoweringObjectFileELF::SelectSectionForGlobal( + const GlobalValue *GV, SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const { + unsigned Flags = getELFSectionFlags(Kind); + + // If we have -ffunction-section or -fdata-section then we should emit the + // global value to a uniqued section specifically for it. + bool EmitUniqueSection = false; + if (!(Flags & ELF::SHF_MERGE) && !Kind.isCommon()) { + if (Kind.isText()) + EmitUniqueSection = TM.getFunctionSections(); + else + EmitUniqueSection = TM.getDataSections(); + } + EmitUniqueSection |= GV->hasComdat(); + + return selectELFSectionForGlobal(getContext(), GV, Kind, Mang, TM, + EmitUniqueSection, Flags); } const MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable( @@ -323,7 +331,8 @@ const MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable( if (!EmitUniqueSection) return ReadOnlySection; - return SelectSectionForGlobal(&F, SectionKind::getReadOnly(), Mang, TM); + return selectELFSectionForGlobal(getContext(), &F, SectionKind::getReadOnly(), + Mang, TM, EmitUniqueSection, ELF::SHF_ALLOC); } bool TargetLoweringObjectFileELF::shouldPutJumpTableInFunctionSection( @@ -423,6 +432,11 @@ TargetLoweringObjectFileELF::InitializeELF(bool UseInitArray_) { // MachO //===----------------------------------------------------------------------===// +TargetLoweringObjectFileMachO::TargetLoweringObjectFileMachO() + : TargetLoweringObjectFile() { + SupportIndirectSymViaGOTPCRel = true; +} + /// getDepLibFromLinkerOpt - Extract the dependent library name from a linker /// option string. Returns StringRef() if the option does not specify a library. StringRef TargetLoweringObjectFileMachO:: @@ -697,6 +711,66 @@ MCSymbol *TargetLoweringObjectFileMachO::getCFIPersonalitySymbol( return SSym; } +const MCExpr *TargetLoweringObjectFileMachO::getIndirectSymViaGOTPCRel( + const MCSymbol *Sym, const MCValue &MV, int64_t Offset, + MachineModuleInfo *MMI, MCStreamer &Streamer) const { + // Although MachO 32-bit targets do not explictly have a GOTPCREL relocation + // as 64-bit do, we replace the GOT equivalent by accessing the final symbol + // through a non_lazy_ptr stub instead. One advantage is that it allows the + // computation of deltas to final external symbols. Example: + // + // _extgotequiv: + // .long _extfoo + // + // _delta: + // .long _extgotequiv-_delta + // + // is transformed to: + // + // _delta: + // .long L_extfoo$non_lazy_ptr-(_delta+0) + // + // .section __IMPORT,__pointers,non_lazy_symbol_pointers + // L_extfoo$non_lazy_ptr: + // .indirect_symbol _extfoo + // .long 0 + // + MachineModuleInfoMachO &MachOMMI = + MMI->getObjFileInfo(); + MCContext &Ctx = getContext(); + + // The offset must consider the original displacement from the base symbol + // since 32-bit targets don't have a GOTPCREL to fold the PC displacement. + Offset = -MV.getConstant(); + const MCSymbol *BaseSym = &MV.getSymB()->getSymbol(); + + // Access the final symbol via sym$non_lazy_ptr and generate the appropriated + // non_lazy_ptr stubs. + SmallString<128> Name; + StringRef Suffix = "$non_lazy_ptr"; + Name += DL->getPrivateGlobalPrefix(); + Name += Sym->getName(); + Name += Suffix; + MCSymbol *Stub = Ctx.GetOrCreateSymbol(Name); + + MachineModuleInfoImpl::StubValueTy &StubSym = MachOMMI.getGVStubEntry(Stub); + if (!StubSym.getPointer()) + StubSym = MachineModuleInfoImpl:: + StubValueTy(const_cast(Sym), true /* access indirectly */); + + const MCExpr *BSymExpr = + MCSymbolRefExpr::Create(BaseSym, MCSymbolRefExpr::VK_None, Ctx); + const MCExpr *LHS = + MCSymbolRefExpr::Create(Stub, MCSymbolRefExpr::VK_None, Ctx); + + if (!Offset) + return MCBinaryExpr::CreateSub(LHS, BSymExpr, Ctx); + + const MCExpr *RHS = + MCBinaryExpr::CreateAdd(BSymExpr, MCConstantExpr::Create(Offset, Ctx), Ctx); + return MCBinaryExpr::CreateSub(LHS, RHS, Ctx); +} + //===----------------------------------------------------------------------===// // COFF //===----------------------------------------------------------------------===// @@ -853,6 +927,11 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, StringRef COMDATSymName = Sym->getName(); return getContext().getCOFFSection(Name, Characteristics, Kind, COMDATSymName, Selection); + } else { + SmallString<256> TmpData; + getNameWithPrefix(TmpData, GV, /*CannotUsePrivateLabel=*/true, Mang, TM); + return getContext().getCOFFSection(Name, Characteristics, Kind, TmpData, + Selection); } } @@ -874,6 +953,42 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, return DataSection; } +void TargetLoweringObjectFileCOFF::getNameWithPrefix( + SmallVectorImpl &OutName, const GlobalValue *GV, + bool CannotUsePrivateLabel, Mangler &Mang, const TargetMachine &TM) const { + if (GV->hasPrivateLinkage() && + ((isa(GV) && TM.getFunctionSections()) || + (isa(GV) && TM.getDataSections()))) + CannotUsePrivateLabel = true; + + Mang.getNameWithPrefix(OutName, GV, CannotUsePrivateLabel); +} + +const MCSection *TargetLoweringObjectFileCOFF::getSectionForJumpTable( + const Function &F, Mangler &Mang, const TargetMachine &TM) const { + // If the function can be removed, produce a unique section so that + // the table doesn't prevent the removal. + const Comdat *C = F.getComdat(); + bool EmitUniqueSection = TM.getFunctionSections() || C; + if (!EmitUniqueSection) + return ReadOnlySection; + + // FIXME: we should produce a symbol for F instead. + if (F.hasPrivateLinkage()) + return ReadOnlySection; + + MCSymbol *Sym = TM.getSymbol(&F, Mang); + StringRef COMDATSymName = Sym->getName(); + + SectionKind Kind = SectionKind::getReadOnly(); + const char *Name = getCOFFSectionNameForUniqueGlobal(Kind); + unsigned Characteristics = getCOFFSectionFlags(Kind); + Characteristics |= COFF::IMAGE_SCN_LNK_COMDAT; + + return getContext().getCOFFSection(Name, Characteristics, Kind, COMDATSymName, + COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE); +} + StringRef TargetLoweringObjectFileCOFF:: getDepLibFromLinkerOpt(StringRef LinkerOption) const { const char *LibCmd = "/DEFAULTLIB:"; diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index 1bbe6e1..57daeab 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -45,6 +45,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -102,6 +103,8 @@ class TwoAddressInstructionPass : public MachineFunctionPass { bool sink3AddrInstruction(MachineInstr *MI, unsigned Reg, MachineBasicBlock::iterator OldPos); + bool isRevCopyChain(unsigned FromReg, unsigned ToReg, int Maxlen); + bool noUseAfterLastDef(unsigned Reg, unsigned Dist, unsigned &LastDef); bool isProfitableToCommute(unsigned regA, unsigned regB, unsigned regC, @@ -309,6 +312,45 @@ sink3AddrInstruction(MachineInstr *MI, unsigned SavedReg, return true; } +/// getSingleDef -- return the MachineInstr* if it is the single def of the Reg +/// in current BB. +static MachineInstr *getSingleDef(unsigned Reg, MachineBasicBlock *BB, + const MachineRegisterInfo *MRI) { + MachineInstr *Ret = nullptr; + for (MachineInstr &DefMI : MRI->def_instructions(Reg)) { + if (DefMI.getParent() != BB || DefMI.isDebugValue()) + continue; + if (!Ret) + Ret = &DefMI; + else if (Ret != &DefMI) + return nullptr; + } + return Ret; +} + +/// Check if there is a reversed copy chain from FromReg to ToReg: +/// %Tmp1 = copy %Tmp2; +/// %FromReg = copy %Tmp1; +/// %ToReg = add %FromReg ... +/// %Tmp2 = copy %ToReg; +/// MaxLen specifies the maximum length of the copy chain the func +/// can walk through. +bool TwoAddressInstructionPass::isRevCopyChain(unsigned FromReg, unsigned ToReg, + int Maxlen) { + unsigned TmpReg = FromReg; + for (int i = 0; i < Maxlen; i++) { + MachineInstr *Def = getSingleDef(TmpReg, MBB, MRI); + if (!Def || !Def->isCopy()) + return false; + + TmpReg = Def->getOperand(1).getReg(); + + if (TmpReg == ToReg) + return true; + } + return false; +} + /// noUseAfterLastDef - Return true if there are no intervening uses between the /// last instruction in the MBB that defines the specified register and the /// two-address instruction which is being processed. It also returns the last @@ -574,6 +616,27 @@ isProfitableToCommute(unsigned regA, unsigned regB, unsigned regC, if (!noUseAfterLastDef(regB, Dist, LastDefB)) return true; + // Look for situation like this: + // %reg101 = MOV %reg100 + // %reg102 = ... + // %reg103 = ADD %reg102, %reg101 + // ... = %reg103 ... + // %reg100 = MOV %reg103 + // If there is a reversed copy chain from reg101 to reg103, commute the ADD + // to eliminate an otherwise unavoidable copy. + // FIXME: + // We can extend the logic further: If an pair of operands in an insn has + // been merged, the insn could be regarded as a virtual copy, and the virtual + // copy could also be used to construct a copy chain. + // To more generally minimize register copies, ideally the logic of two addr + // instruction pass should be integrated with register allocation pass where + // interference graph is available. + if (isRevCopyChain(regC, regA, 3)) + return true; + + if (isRevCopyChain(regB, regA, 3)) + return false; + // Since there are no intervening uses for both registers, then commute // if the def of regC is closer. Its live interval is shorter. return LastDefB && LastDefC && LastDefC > LastDefB; diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp index 7d3b0ce..d9adfdf 100644 --- a/lib/CodeGen/VirtRegMap.cpp +++ b/lib/CodeGen/VirtRegMap.cpp @@ -286,7 +286,7 @@ void VirtRegRewriter::addMBBLiveIns() { } void VirtRegRewriter::rewrite() { - bool NoSubRegLiveness = !MRI->tracksSubRegLiveness(); + bool NoSubRegLiveness = !MRI->subRegLivenessEnabled(); SmallVector SuperDeads; SmallVector SuperDefs; SmallVector SuperKills; diff --git a/lib/CodeGen/WinEHPrepare.cpp b/lib/CodeGen/WinEHPrepare.cpp index 6f712a9..ab0f96e 100644 --- a/lib/CodeGen/WinEHPrepare.cpp +++ b/lib/CodeGen/WinEHPrepare.cpp @@ -16,6 +16,8 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/LibCallSemantics.h" #include "llvm/IR/Function.h" @@ -25,6 +27,10 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PatternMatch.h" #include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Local.h" #include @@ -36,25 +42,31 @@ using namespace llvm::PatternMatch; namespace { -struct HandlerAllocas { - TinyPtrVector Allocas; - int ParentFrameAllocationIndex; -}; - // This map is used to model frame variable usage during outlining, to // construct a structure type to hold the frame variables in a frame // allocation block, and to remap the frame variable allocas (including // spill locations as needed) to GEPs that get the variable from the // frame allocation structure. -typedef MapVector FrameVarInfoMap; +typedef MapVector> FrameVarInfoMap; -class WinEHPrepare : public FunctionPass { - std::unique_ptr DwarfPrepare; +typedef SmallSet VisitedBlockSet; + +enum ActionType { Catch, Cleanup }; + +class LandingPadActions; +class ActionHandler; +class CatchHandler; +class CleanupHandler; +class LandingPadMap; +typedef DenseMap CatchHandlerMapTy; +typedef DenseMap CleanupHandlerMapTy; + +class WinEHPrepare : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid. WinEHPrepare(const TargetMachine *TM = nullptr) - : FunctionPass(ID), DwarfPrepare(createDwarfEHPass(TM)) {} + : FunctionPass(ID) {} bool runOnFunction(Function &Fn) override; @@ -67,11 +79,24 @@ public: } private: - bool prepareCPPEHHandlers(Function &F, - SmallVectorImpl &LPads); - bool outlineCatchHandler(Function *SrcFn, Constant *SelectorType, - LandingPadInst *LPad, CallInst *&EHAlloc, - AllocaInst *&EHObjPtr, FrameVarInfoMap &VarInfo); + bool prepareExceptionHandlers(Function &F, + SmallVectorImpl &LPads); + bool outlineHandler(ActionHandler *Action, Function *SrcFn, + LandingPadInst *LPad, BasicBlock *StartBB, + FrameVarInfoMap &VarInfo); + + void mapLandingPadBlocks(LandingPadInst *LPad, LandingPadActions &Actions); + CatchHandler *findCatchHandler(BasicBlock *BB, BasicBlock *&NextBB, + VisitedBlockSet &VisitedBlocks); + CleanupHandler *findCleanupHandler(BasicBlock *StartBB, BasicBlock *EndBB); + + void processSEHCatchHandler(CatchHandler *Handler, BasicBlock *StartBB); + + // All fields are reset by runOnFunction. + EHPersonality Personality; + CatchHandlerMapTy CatchHandlerMap; + CleanupHandlerMapTy CleanupHandlerMap; + DenseMap LPadMaps; }; class WinEHFrameVariableMaterializer : public ValueMaterializer { @@ -87,34 +112,218 @@ private: IRBuilder<> Builder; }; -class WinEHCatchDirector : public CloningDirector { +class LandingPadMap { +public: + LandingPadMap() : OriginLPad(nullptr) {} + void mapLandingPad(const LandingPadInst *LPad); + + bool isInitialized() { return OriginLPad != nullptr; } + + bool mapIfEHPtrLoad(const LoadInst *Load) { + return mapIfEHLoad(Load, EHPtrStores, EHPtrStoreAddrs); + } + bool mapIfSelectorLoad(const LoadInst *Load) { + return mapIfEHLoad(Load, SelectorStores, SelectorStoreAddrs); + } + + bool isLandingPadSpecificInst(const Instruction *Inst) const; + + void remapSelector(ValueToValueMapTy &VMap, Value *MappedValue) const; + +private: + bool mapIfEHLoad(const LoadInst *Load, + SmallVectorImpl &Stores, + SmallVectorImpl &StoreAddrs); + + const LandingPadInst *OriginLPad; + // We will normally only see one of each of these instructions, but + // if more than one occurs for some reason we can handle that. + TinyPtrVector ExtractedEHPtrs; + TinyPtrVector ExtractedSelectors; + + // In optimized code, there will typically be at most one instance of + // each of the following, but in unoptimized IR it is not uncommon + // for the values to be stored, loaded and then stored again. In that + // case we will create a second entry for each store and store address. + SmallVector EHPtrStores; + SmallVector SelectorStores; + SmallVector EHPtrStoreAddrs; + SmallVector SelectorStoreAddrs; +}; + +class WinEHCloningDirectorBase : public CloningDirector { public: - WinEHCatchDirector(LandingPadInst *LPI, Function *CatchFn, Value *Selector, - Value *EHObj, FrameVarInfoMap &VarInfo) - : LPI(LPI), CurrentSelector(Selector->stripPointerCasts()), EHObj(EHObj), - Materializer(CatchFn, VarInfo), - SelectorIDType(Type::getInt32Ty(LPI->getContext())), - Int8PtrType(Type::getInt8PtrTy(LPI->getContext())) {} + WinEHCloningDirectorBase(Function *HandlerFn, + FrameVarInfoMap &VarInfo, + LandingPadMap &LPadMap) + : Materializer(HandlerFn, VarInfo), + SelectorIDType(Type::getInt32Ty(HandlerFn->getContext())), + Int8PtrType(Type::getInt8PtrTy(HandlerFn->getContext())), + LPadMap(LPadMap) {} CloningAction handleInstruction(ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) override; + virtual CloningAction handleBeginCatch(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) = 0; + virtual CloningAction handleEndCatch(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) = 0; + virtual CloningAction handleTypeIdFor(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) = 0; + virtual CloningAction handleInvoke(ValueToValueMapTy &VMap, + const InvokeInst *Invoke, + BasicBlock *NewBB) = 0; + virtual CloningAction handleResume(ValueToValueMapTy &VMap, + const ResumeInst *Resume, + BasicBlock *NewBB) = 0; + ValueMaterializer *getValueMaterializer() override { return &Materializer; } -private: - LandingPadInst *LPI; - Value *CurrentSelector; - Value *EHObj; +protected: WinEHFrameVariableMaterializer Materializer; Type *SelectorIDType; Type *Int8PtrType; + LandingPadMap &LPadMap; +}; + +class WinEHCatchDirector : public WinEHCloningDirectorBase { +public: + WinEHCatchDirector(Function *CatchFn, Value *Selector, + FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap) + : WinEHCloningDirectorBase(CatchFn, VarInfo, LPadMap), + CurrentSelector(Selector->stripPointerCasts()), + ExceptionObjectVar(nullptr) {} + + CloningAction handleBeginCatch(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) override; + CloningAction handleEndCatch(ValueToValueMapTy &VMap, const Instruction *Inst, + BasicBlock *NewBB) override; + CloningAction handleTypeIdFor(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) override; + CloningAction handleInvoke(ValueToValueMapTy &VMap, const InvokeInst *Invoke, + BasicBlock *NewBB) override; + CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume, + BasicBlock *NewBB) override; + + const Value *getExceptionVar() { return ExceptionObjectVar; } + TinyPtrVector &getReturnTargets() { return ReturnTargets; } + +private: + Value *CurrentSelector; - const Value *ExtractedEHPtr; - const Value *ExtractedSelector; - const Value *EHPtrStoreAddr; - const Value *SelectorStoreAddr; + const Value *ExceptionObjectVar; + TinyPtrVector ReturnTargets; }; + +class WinEHCleanupDirector : public WinEHCloningDirectorBase { +public: + WinEHCleanupDirector(Function *CleanupFn, + FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap) + : WinEHCloningDirectorBase(CleanupFn, VarInfo, LPadMap) {} + + CloningAction handleBeginCatch(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) override; + CloningAction handleEndCatch(ValueToValueMapTy &VMap, const Instruction *Inst, + BasicBlock *NewBB) override; + CloningAction handleTypeIdFor(ValueToValueMapTy &VMap, + const Instruction *Inst, + BasicBlock *NewBB) override; + CloningAction handleInvoke(ValueToValueMapTy &VMap, const InvokeInst *Invoke, + BasicBlock *NewBB) override; + CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume, + BasicBlock *NewBB) override; +}; + +class ActionHandler { +public: + ActionHandler(BasicBlock *BB, ActionType Type) + : StartBB(BB), Type(Type), HandlerBlockOrFunc(nullptr) {} + + ActionType getType() const { return Type; } + BasicBlock *getStartBlock() const { return StartBB; } + + bool hasBeenProcessed() { return HandlerBlockOrFunc != nullptr; } + + void setHandlerBlockOrFunc(Constant *F) { HandlerBlockOrFunc = F; } + Constant *getHandlerBlockOrFunc() { return HandlerBlockOrFunc; } + +private: + BasicBlock *StartBB; + ActionType Type; + + // Can be either a BlockAddress or a Function depending on the EH personality. + Constant *HandlerBlockOrFunc; +}; + +class CatchHandler : public ActionHandler { +public: + CatchHandler(BasicBlock *BB, Constant *Selector, BasicBlock *NextBB) + : ActionHandler(BB, ActionType::Catch), Selector(Selector), + NextBB(NextBB), ExceptionObjectVar(nullptr) {} + + // Method for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const ActionHandler *H) { + return H->getType() == ActionType::Catch; + } + + Constant *getSelector() const { return Selector; } + BasicBlock *getNextBB() const { return NextBB; } + + const Value *getExceptionVar() { return ExceptionObjectVar; } + TinyPtrVector &getReturnTargets() { return ReturnTargets; } + + void setExceptionVar(const Value *Val) { ExceptionObjectVar = Val; } + void setReturnTargets(TinyPtrVector &Targets) { + ReturnTargets = Targets; + } + +private: + Constant *Selector; + BasicBlock *NextBB; + const Value *ExceptionObjectVar; + TinyPtrVector ReturnTargets; +}; + +class CleanupHandler : public ActionHandler { +public: + CleanupHandler(BasicBlock *BB) : ActionHandler(BB, ActionType::Cleanup) {} + + // Method for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const ActionHandler *H) { + return H->getType() == ActionType::Cleanup; + } +}; + +class LandingPadActions { +public: + LandingPadActions() : HasCleanupHandlers(false) {} + + void insertCatchHandler(CatchHandler *Action) { Actions.push_back(Action); } + void insertCleanupHandler(CleanupHandler *Action) { + Actions.push_back(Action); + HasCleanupHandlers = true; + } + + bool includesCleanup() const { return HasCleanupHandlers; } + + SmallVectorImpl::iterator begin() { return Actions.begin(); } + SmallVectorImpl::iterator end() { return Actions.end(); } + +private: + // Note that this class does not own the ActionHandler objects in this vector. + // The ActionHandlers are owned by the CatchHandlerMap and CleanupHandlerMap + // in the WinEHPrepare class. + SmallVector Actions; + bool HasCleanupHandlers; +}; + } // end anonymous namespace char WinEHPrepare::ID = 0; @@ -125,10 +334,10 @@ FunctionPass *llvm::createWinEHPass(const TargetMachine *TM) { return new WinEHPrepare(TM); } -static bool isMSVCPersonality(EHPersonality Pers) { - return Pers == EHPersonality::MSVC_Win64SEH || - Pers == EHPersonality::MSVC_CXX; -} +// FIXME: Remove this once the backend can handle the prepared IR. +static cl::opt +SEHPrepare("sehprepare", cl::Hidden, + cl::desc("Prepare functions with SEH personalities")); bool WinEHPrepare::runOnFunction(Function &Fn) { SmallVector LPads; @@ -145,60 +354,67 @@ bool WinEHPrepare::runOnFunction(Function &Fn) { return false; // Classify the personality to see what kind of preparation we need. - EHPersonality Pers = classifyEHPersonality(LPads.back()->getPersonalityFn()); - - // Delegate through to the DWARF pass if this is unrecognized. - if (!isMSVCPersonality(Pers)) - return DwarfPrepare->runOnFunction(Fn); + Personality = classifyEHPersonality(LPads.back()->getPersonalityFn()); - // FIXME: This only returns true if the C++ EH handlers were outlined. - // When that code is complete, it should always return whatever - // prepareCPPEHHandlers returns. - if (Pers == EHPersonality::MSVC_CXX && prepareCPPEHHandlers(Fn, LPads)) - return true; - - // FIXME: SEH Cleanups are unimplemented. Replace them with unreachable. - if (Resumes.empty()) + // Do nothing if this is not an MSVC personality. + if (!isMSVCEHPersonality(Personality)) return false; - for (ResumeInst *Resume : Resumes) { - IRBuilder<>(Resume).CreateUnreachable(); - Resume->eraseFromParent(); + if (isAsynchronousEHPersonality(Personality) && !SEHPrepare) { + // Replace all resume instructions with unreachable. + // FIXME: Remove this once the backend can handle the prepared IR. + for (ResumeInst *Resume : Resumes) { + IRBuilder<>(Resume).CreateUnreachable(); + Resume->eraseFromParent(); + } + return true; } + // If there were any landing pads, prepareExceptionHandlers will make changes. + prepareExceptionHandlers(Fn, LPads); return true; } bool WinEHPrepare::doFinalization(Module &M) { - return DwarfPrepare->doFinalization(M); + return false; } -void WinEHPrepare::getAnalysisUsage(AnalysisUsage &AU) const { - DwarfPrepare->getAnalysisUsage(AU); -} +void WinEHPrepare::getAnalysisUsage(AnalysisUsage &AU) const {} -bool WinEHPrepare::prepareCPPEHHandlers( +bool WinEHPrepare::prepareExceptionHandlers( Function &F, SmallVectorImpl &LPads) { // These containers are used to re-map frame variables that are used in // outlined catch and cleanup handlers. They will be populated as the // handlers are outlined. FrameVarInfoMap FrameVarInfo; - SmallVector HandlerAllocs; - SmallVector HandlerEHObjPtrs; bool HandlersOutlined = false; + Module *M = F.getParent(); + LLVMContext &Context = M->getContext(); + + // Create a new function to receive the handler contents. + PointerType *Int8PtrType = Type::getInt8PtrTy(Context); + Type *Int32Type = Type::getInt32Ty(Context); + Function *ActionIntrin = Intrinsic::getDeclaration(M, Intrinsic::eh_actions); + for (LandingPadInst *LPad : LPads) { // Look for evidence that this landingpad has already been processed. bool LPadHasActionList = false; BasicBlock *LPadBB = LPad->getParent(); - for (Instruction &Inst : LPadBB->getInstList()) { - // FIXME: Make this an intrinsic. - if (auto *Call = dyn_cast(&Inst)) - if (Call->getCalledFunction()->getName() == "llvm.eh.actions") { + for (Instruction &Inst : *LPadBB) { + if (auto *IntrinCall = dyn_cast(&Inst)) { + if (IntrinCall->getIntrinsicID() == Intrinsic::eh_actions) { LPadHasActionList = true; break; } + } + // FIXME: This is here to help with the development of nested landing pad + // outlining. It should be removed when that is finished. + if (isa(Inst)) { + LPadHasActionList = true; + break; + } } // If we've already outlined the handlers for this landingpad, @@ -206,177 +422,244 @@ bool WinEHPrepare::prepareCPPEHHandlers( if (LPadHasActionList) continue; - for (unsigned Idx = 0, NumClauses = LPad->getNumClauses(); Idx < NumClauses; - ++Idx) { - if (LPad->isCatch(Idx)) { - // Create a new instance of the handler data structure in the - // HandlerData vector. - CallInst *EHAlloc = nullptr; - AllocaInst *EHObjPtr = nullptr; - bool Outlined = outlineCatchHandler(&F, LPad->getClause(Idx), LPad, - EHAlloc, EHObjPtr, FrameVarInfo); - if (Outlined) { + LandingPadActions Actions; + mapLandingPadBlocks(LPad, Actions); + + for (ActionHandler *Action : Actions) { + if (Action->hasBeenProcessed()) + continue; + BasicBlock *StartBB = Action->getStartBlock(); + + // SEH doesn't do any outlining for catches. Instead, pass the handler + // basic block addr to llvm.eh.actions and list the block as a return + // target. + if (isAsynchronousEHPersonality(Personality)) { + if (auto *CatchAction = dyn_cast(Action)) { + processSEHCatchHandler(CatchAction, StartBB); HandlersOutlined = true; - // These values must be resolved after all handlers have been - // outlined. - if (EHAlloc) - HandlerAllocs.push_back(EHAlloc); - if (EHObjPtr) - HandlerEHObjPtrs.push_back(EHObjPtr); + continue; } - } // End if (isCatch) - } // End for each clause - } // End for each landingpad + } + + if (outlineHandler(Action, &F, LPad, StartBB, FrameVarInfo)) { + HandlersOutlined = true; + } + } // End for each Action + + // FIXME: We need a guard against partially outlined functions. + if (!HandlersOutlined) + continue; + + // Replace the landing pad with a new llvm.eh.action based landing pad. + BasicBlock *NewLPadBB = BasicBlock::Create(Context, "lpad", &F, LPadBB); + assert(!isa(LPadBB->begin())); + Instruction *NewLPad = LPad->clone(); + NewLPadBB->getInstList().push_back(NewLPad); + while (!pred_empty(LPadBB)) { + auto *pred = *pred_begin(LPadBB); + InvokeInst *Invoke = cast(pred->getTerminator()); + Invoke->setUnwindDest(NewLPadBB); + } + + // Replace uses of the old lpad in phis with this block and delete the old + // block. + LPadBB->replaceSuccessorsPhiUsesWith(NewLPadBB); + LPadBB->getTerminator()->eraseFromParent(); + new UnreachableInst(LPadBB->getContext(), LPadBB); + + // Add a call to describe the actions for this landing pad. + std::vector ActionArgs; + for (ActionHandler *Action : Actions) { + // Action codes from docs are: 0 cleanup, 1 catch. + if (auto *CatchAction = dyn_cast(Action)) { + ActionArgs.push_back(ConstantInt::get(Int32Type, 1)); + ActionArgs.push_back(CatchAction->getSelector()); + Value *EHObj = const_cast(CatchAction->getExceptionVar()); + if (EHObj) + ActionArgs.push_back(EHObj); + else + ActionArgs.push_back(ConstantPointerNull::get(Int8PtrType)); + } else { + ActionArgs.push_back(ConstantInt::get(Int32Type, 0)); + } + ActionArgs.push_back(Action->getHandlerBlockOrFunc()); + } + CallInst *Recover = + CallInst::Create(ActionIntrin, ActionArgs, "recover", NewLPadBB); + + // Add an indirect branch listing possible successors of the catch handlers. + IndirectBrInst *Branch = IndirectBrInst::Create(Recover, 0, NewLPadBB); + for (ActionHandler *Action : Actions) { + if (auto *CatchAction = dyn_cast(Action)) { + for (auto *Target : CatchAction->getReturnTargets()) { + Branch->addDestination(Target); + } + } + } + } // End for each landingpad // If nothing got outlined, there is no more processing to be done. if (!HandlersOutlined) return false; - // FIXME: We will replace the landingpad bodies with llvm.eh.actions - // calls and indirect branches here and then delete blocks - // which are no longer reachable. That will get rid of the - // handlers that we have outlined. There is code below - // that looks for allocas with no uses in the parent function. - // That will only happen after the pruning is implemented. - - // Remap the frame variables. - SmallVector StructTys; - StructTys.push_back(Type::getInt32Ty(F.getContext())); // EH state - StructTys.push_back(Type::getInt8PtrTy(F.getContext())); // EH object - - // Start the index at two since we always have the above fields at 0 and 1. - int Idx = 2; - - // FIXME: Sort the FrameVarInfo vector by the ParentAlloca size and alignment - // and add padding as necessary to provide the proper alignment. - - // Map the alloca instructions to the corresponding index in the - // frame allocation structure. If any alloca is used only in a single - // handler and is not used in the parent frame after outlining, it will - // be assigned an index of -1, meaning the handler can keep its - // "temporary" alloca and the original alloca can be erased from the - // parent function. If we later encounter this alloca in a second - // handler, we will assign it a place in the frame allocation structure - // at that time. Since the instruction replacement doesn't happen until - // all the entries in the HandlerData have been processed this isn't a - // problem. - for (auto &VarInfoEntry : FrameVarInfo) { - AllocaInst *ParentAlloca = VarInfoEntry.first; - HandlerAllocas &AllocaInfo = VarInfoEntry.second; - - // If the instruction still has uses in the parent function or if it is - // referenced by more than one handler, add it to the frame allocation - // structure. - if (ParentAlloca->getNumUses() != 0 || AllocaInfo.Allocas.size() > 1) { - Type *VarTy = ParentAlloca->getAllocatedType(); - StructTys.push_back(VarTy); - AllocaInfo.ParentFrameAllocationIndex = Idx++; - } else { - // If the variable is not used in the parent frame and it is only used - // in one handler, the alloca can be removed from the parent frame - // and the handler will keep its "temporary" alloca to define the value. - // An element index of -1 is used to indicate this condition. - AllocaInfo.ParentFrameAllocationIndex = -1; - } - } + // Delete any blocks that were only used by handlers that were outlined above. + removeUnreachableBlocks(F); - // Having filled the StructTys vector and assigned an index to each element, - // we can now create the structure. - StructType *EHDataStructTy = StructType::create( - F.getContext(), StructTys, "struct." + F.getName().str() + ".ehdata"); - IRBuilder<> Builder(F.getParent()->getContext()); - - // Create a frame allocation. - Module *M = F.getParent(); - LLVMContext &Context = M->getContext(); BasicBlock *Entry = &F.getEntryBlock(); + IRBuilder<> Builder(F.getParent()->getContext()); Builder.SetInsertPoint(Entry->getFirstInsertionPt()); - Function *FrameAllocFn = - Intrinsic::getDeclaration(M, Intrinsic::frameallocate); - uint64_t EHAllocSize = M->getDataLayout()->getTypeAllocSize(EHDataStructTy); - Value *FrameAllocArgs[] = { - ConstantInt::get(Type::getInt32Ty(Context), EHAllocSize)}; - CallInst *FrameAlloc = - Builder.CreateCall(FrameAllocFn, FrameAllocArgs, "frame.alloc"); - - Value *FrameEHData = Builder.CreateBitCast( - FrameAlloc, EHDataStructTy->getPointerTo(), "eh.data"); - - // Now visit each handler that is using the structure and bitcast its EHAlloc - // value to be a pointer to the frame alloc structure. - DenseMap EHDataMap; - for (CallInst *EHAlloc : HandlerAllocs) { - // The EHAlloc has no uses at this time, so we need to just insert the - // cast before the next instruction. There is always a next instruction. - BasicBlock::iterator II = EHAlloc; - ++II; - Builder.SetInsertPoint(cast(II)); - Value *EHData = Builder.CreateBitCast( - EHAlloc, EHDataStructTy->getPointerTo(), "eh.data"); - EHDataMap[EHAlloc->getParent()->getParent()] = EHData; - } - // Next, replace the place-holder EHObjPtr allocas with GEP instructions - // that pull the EHObjPtr from the frame alloc structure - for (AllocaInst *EHObjPtr : HandlerEHObjPtrs) { - Value *EHData = EHDataMap[EHObjPtr->getParent()->getParent()]; - Builder.SetInsertPoint(EHObjPtr); - Value *ElementPtr = Builder.CreateConstInBoundsGEP2_32(EHData, 0, 1); - EHObjPtr->replaceAllUsesWith(ElementPtr); - EHObjPtr->removeFromParent(); - ElementPtr->takeName(EHObjPtr); - delete EHObjPtr; - } + Function *FrameEscapeFn = + Intrinsic::getDeclaration(M, Intrinsic::frameescape); + Function *RecoverFrameFn = + Intrinsic::getDeclaration(M, Intrinsic::framerecover); // Finally, replace all of the temporary allocas for frame variables used in - // the outlined handlers and the original frame allocas with GEP instructions - // that get the equivalent pointer from the frame allocation struct. + // the outlined handlers with calls to llvm.framerecover. + BasicBlock::iterator II = Entry->getFirstInsertionPt(); + Instruction *AllocaInsertPt = II; + SmallVector AllocasToEscape; for (auto &VarInfoEntry : FrameVarInfo) { - AllocaInst *ParentAlloca = VarInfoEntry.first; - HandlerAllocas &AllocaInfo = VarInfoEntry.second; - int Idx = AllocaInfo.ParentFrameAllocationIndex; - - // If we have an index of -1 for this instruction, it means it isn't used - // outside of this handler. In that case, we just keep the "temporary" - // alloca in the handler and erase the original alloca from the parent. - if (Idx == -1) { + Value *ParentVal = VarInfoEntry.first; + TinyPtrVector &Allocas = VarInfoEntry.second; + + // If the mapped value isn't already an alloca, we need to spill it if it + // is a computed value or copy it if it is an argument. + AllocaInst *ParentAlloca = dyn_cast(ParentVal); + if (!ParentAlloca) { + if (auto *Arg = dyn_cast(ParentVal)) { + // Lower this argument to a copy and then demote that to the stack. + // We can't just use the argument location because the handler needs + // it to be in the frame allocation block. + // Use 'select i8 true, %arg, undef' to simulate a 'no-op' instruction. + Value *TrueValue = ConstantInt::getTrue(Context); + Value *UndefValue = UndefValue::get(Arg->getType()); + Instruction *SI = + SelectInst::Create(TrueValue, Arg, UndefValue, + Arg->getName() + ".tmp", AllocaInsertPt); + Arg->replaceAllUsesWith(SI); + // Reset the select operand, because it was clobbered by the RAUW above. + SI->setOperand(1, Arg); + ParentAlloca = DemoteRegToStack(*SI, true, SI); + } else if (auto *PN = dyn_cast(ParentVal)) { + ParentAlloca = DemotePHIToStack(PN, AllocaInsertPt); + } else { + Instruction *ParentInst = cast(ParentVal); + // FIXME: This is a work-around to temporarily handle the case where an + // instruction that is only used in handlers is not sunk. + // Without uses, DemoteRegToStack would just eliminate the value. + // This will fail if ParentInst is an invoke. + if (ParentInst->getNumUses() == 0) { + BasicBlock::iterator InsertPt = ParentInst; + ++InsertPt; + ParentAlloca = + new AllocaInst(ParentInst->getType(), nullptr, + ParentInst->getName() + ".reg2mem", InsertPt); + new StoreInst(ParentInst, ParentAlloca, InsertPt); + } else { + ParentAlloca = DemoteRegToStack(*ParentInst, true, ParentInst); + } + } + } + + // If the parent alloca is no longer used and only one of the handlers used + // it, erase the parent and leave the copy in the outlined handler. + if (ParentAlloca->getNumUses() == 0 && Allocas.size() == 1) { ParentAlloca->eraseFromParent(); - } else { - // Otherwise, we replace the parent alloca and all outlined allocas - // which map to it with GEP instructions. - - // First replace the original alloca. - Builder.SetInsertPoint(ParentAlloca); - Builder.SetCurrentDebugLocation(ParentAlloca->getDebugLoc()); - Value *ElementPtr = - Builder.CreateConstInBoundsGEP2_32(FrameEHData, 0, Idx); - ParentAlloca->replaceAllUsesWith(ElementPtr); - ParentAlloca->removeFromParent(); - ElementPtr->takeName(ParentAlloca); - delete ParentAlloca; - - // Next replace all outlined allocas that are mapped to it. - for (AllocaInst *TempAlloca : AllocaInfo.Allocas) { - Value *EHData = EHDataMap[TempAlloca->getParent()->getParent()]; - // FIXME: Sink this GEP into the blocks where it is used. - Builder.SetInsertPoint(TempAlloca); - Builder.SetCurrentDebugLocation(TempAlloca->getDebugLoc()); - ElementPtr = Builder.CreateConstInBoundsGEP2_32(EHData, 0, Idx); - TempAlloca->replaceAllUsesWith(ElementPtr); - TempAlloca->removeFromParent(); - ElementPtr->takeName(TempAlloca); - delete TempAlloca; + continue; + } + + // Add this alloca to the list of things to escape. + AllocasToEscape.push_back(ParentAlloca); + + // Next replace all outlined allocas that are mapped to it. + for (AllocaInst *TempAlloca : Allocas) { + Function *HandlerFn = TempAlloca->getParent()->getParent(); + // FIXME: Sink this GEP into the blocks where it is used. + Builder.SetInsertPoint(TempAlloca); + Builder.SetCurrentDebugLocation(TempAlloca->getDebugLoc()); + Value *RecoverArgs[] = { + Builder.CreateBitCast(&F, Int8PtrType, ""), + &(HandlerFn->getArgumentList().back()), + llvm::ConstantInt::get(Int32Type, AllocasToEscape.size() - 1)}; + Value *RecoveredAlloca = Builder.CreateCall(RecoverFrameFn, RecoverArgs); + // Add a pointer bitcast if the alloca wasn't an i8. + if (RecoveredAlloca->getType() != TempAlloca->getType()) { + RecoveredAlloca->setName(Twine(TempAlloca->getName()) + ".i8"); + RecoveredAlloca = + Builder.CreateBitCast(RecoveredAlloca, TempAlloca->getType()); } - } // end else of if (Idx == -1) - } // End for each FrameVarInfo entry. + TempAlloca->replaceAllUsesWith(RecoveredAlloca); + TempAlloca->removeFromParent(); + RecoveredAlloca->takeName(TempAlloca); + delete TempAlloca; + } + } // End for each FrameVarInfo entry. + + // Insert 'call void (...)* @llvm.frameescape(...)' at the end of the entry + // block. + Builder.SetInsertPoint(&F.getEntryBlock().back()); + Builder.CreateCall(FrameEscapeFn, AllocasToEscape); + + // Insert an alloca for the EH state in the entry block. On x86, we will also + // insert stores to update the EH state, but on other ISAs, the runtime does + // it for us. + // FIXME: This record is different on x86. + Type *UnwindHelpTy = Type::getInt64Ty(Context); + AllocaInst *UnwindHelp = + new AllocaInst(UnwindHelpTy, "unwindhelp", &F.getEntryBlock().front()); + Builder.CreateStore(llvm::ConstantInt::get(UnwindHelpTy, -2), UnwindHelp); + Function *UnwindHelpFn = + Intrinsic::getDeclaration(M, Intrinsic::eh_unwindhelp); + Builder.CreateCall(UnwindHelpFn, + Builder.CreateBitCast(UnwindHelp, Int8PtrType)); + + // Clean up the handler action maps we created for this function + DeleteContainerSeconds(CatchHandlerMap); + CatchHandlerMap.clear(); + DeleteContainerSeconds(CleanupHandlerMap); + CleanupHandlerMap.clear(); return HandlersOutlined; } -bool WinEHPrepare::outlineCatchHandler(Function *SrcFn, Constant *SelectorType, - LandingPadInst *LPad, CallInst *&EHAlloc, - AllocaInst *&EHObjPtr, - FrameVarInfoMap &VarInfo) { +// This function examines a block to determine whether the block ends with a +// conditional branch to a catch handler based on a selector comparison. +// This function is used both by the WinEHPrepare::findSelectorComparison() and +// WinEHCleanupDirector::handleTypeIdFor(). +static bool isSelectorDispatch(BasicBlock *BB, BasicBlock *&CatchHandler, + Constant *&Selector, BasicBlock *&NextBB) { + ICmpInst::Predicate Pred; + BasicBlock *TBB, *FBB; + Value *LHS, *RHS; + + if (!match(BB->getTerminator(), + m_Br(m_ICmp(Pred, m_Value(LHS), m_Value(RHS)), TBB, FBB))) + return false; + + if (!match(LHS, + m_Intrinsic(m_Constant(Selector))) && + !match(RHS, m_Intrinsic(m_Constant(Selector)))) + return false; + + if (Pred == CmpInst::ICMP_EQ) { + CatchHandler = TBB; + NextBB = FBB; + return true; + } + + if (Pred == CmpInst::ICMP_NE) { + CatchHandler = FBB; + NextBB = TBB; + return true; + } + + return false; +} + +bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn, + LandingPadInst *LPad, BasicBlock *StartBB, + FrameVarInfoMap &VarInfo) { Module *M = SrcFn->getParent(); LLVMContext &Context = M->getContext(); @@ -385,133 +668,241 @@ bool WinEHPrepare::outlineCatchHandler(Function *SrcFn, Constant *SelectorType, std::vector ArgTys; ArgTys.push_back(Int8PtrType); ArgTys.push_back(Int8PtrType); - FunctionType *FnType = FunctionType::get(Int8PtrType, ArgTys, false); - Function *CatchHandler = Function::Create( - FnType, GlobalVariable::ExternalLinkage, SrcFn->getName() + ".catch", M); + Function *Handler; + if (Action->getType() == Catch) { + FunctionType *FnType = FunctionType::get(Int8PtrType, ArgTys, false); + Handler = Function::Create(FnType, GlobalVariable::InternalLinkage, + SrcFn->getName() + ".catch", M); + } else { + FunctionType *FnType = + FunctionType::get(Type::getVoidTy(Context), ArgTys, false); + Handler = Function::Create(FnType, GlobalVariable::InternalLinkage, + SrcFn->getName() + ".cleanup", M); + } // Generate a standard prolog to setup the frame recovery structure. IRBuilder<> Builder(Context); - BasicBlock *Entry = BasicBlock::Create(Context, "catch.entry"); - CatchHandler->getBasicBlockList().push_front(Entry); + BasicBlock *Entry = BasicBlock::Create(Context, "entry"); + Handler->getBasicBlockList().push_front(Entry); Builder.SetInsertPoint(Entry); Builder.SetCurrentDebugLocation(LPad->getDebugLoc()); - // The outlined handler will be called with the parent's frame pointer as - // its second argument. To enable the handler to access variables from - // the parent frame, we use that pointer to get locate a special block - // of memory that was allocated using llvm.eh.allocateframe for this - // purpose. During the outlining process we will determine which frame - // variables are used in handlers and create a structure that maps these - // variables into the frame allocation block. - // - // The frame allocation block also contains an exception state variable - // used by the runtime and a pointer to the exception object pointer - // which will be filled in by the runtime for use in the handler. - Function *RecoverFrameFn = - Intrinsic::getDeclaration(M, Intrinsic::framerecover); - Value *RecoverArgs[] = {Builder.CreateBitCast(SrcFn, Int8PtrType, ""), - &(CatchHandler->getArgumentList().back())}; - EHAlloc = Builder.CreateCall(RecoverFrameFn, RecoverArgs, "eh.alloc"); - - // This alloca is only temporary. We'll be replacing it once we know all the - // frame variables that need to go in the frame allocation structure. - EHObjPtr = Builder.CreateAlloca(Int8PtrType, 0, "eh.obj.ptr"); - - // This will give us a raw pointer to the exception object, which - // corresponds to the formal parameter of the catch statement. If the - // handler uses this object, we will generate code during the outlining - // process to cast the pointer to the appropriate type and deference it - // as necessary. The un-outlined landing pad code represents the - // exception object as the result of the llvm.eh.begincatch call. - Value *EHObj = Builder.CreateLoad(EHObjPtr, false, "eh.obj"); + std::unique_ptr Director; ValueToValueMapTy VMap; - // FIXME: Map other values referenced in the filter handler. - - WinEHCatchDirector Director(LPad, CatchHandler, SelectorType, EHObj, VarInfo); + LandingPadMap &LPadMap = LPadMaps[LPad]; + if (!LPadMap.isInitialized()) + LPadMap.mapLandingPad(LPad); + if (auto *CatchAction = dyn_cast(Action)) { + Constant *Sel = CatchAction->getSelector(); + Director.reset(new WinEHCatchDirector(Handler, Sel, VarInfo, LPadMap)); + LPadMap.remapSelector(VMap, ConstantInt::get(Type::getInt32Ty(Context), 1)); + } else { + Director.reset(new WinEHCleanupDirector(Handler, VarInfo, LPadMap)); + } SmallVector Returns; - ClonedCodeInfo InlinedFunctionInfo; + ClonedCodeInfo OutlinedFunctionInfo; + + // If the start block contains PHI nodes, we need to map them. + BasicBlock::iterator II = StartBB->begin(); + while (auto *PN = dyn_cast(II)) { + bool Mapped = false; + // Look for PHI values that we have already mapped (such as the selector). + for (Value *Val : PN->incoming_values()) { + if (VMap.count(Val)) { + VMap[PN] = VMap[Val]; + Mapped = true; + } + } + // If we didn't find a match for this value, map it as an undef. + if (!Mapped) { + VMap[PN] = UndefValue::get(PN->getType()); + } + ++II; + } - BasicBlock::iterator II = LPad; + // Skip over PHIs and, if applicable, landingpad instructions. + II = StartBB->getFirstInsertionPt(); - CloneAndPruneIntoFromInst(CatchHandler, SrcFn, ++II, VMap, + CloneAndPruneIntoFromInst(Handler, SrcFn, II, VMap, /*ModuleLevelChanges=*/false, Returns, "", - &InlinedFunctionInfo, - SrcFn->getParent()->getDataLayout(), &Director); + &OutlinedFunctionInfo, Director.get()); // Move all the instructions in the first cloned block into our entry block. BasicBlock *FirstClonedBB = std::next(Function::iterator(Entry)); Entry->getInstList().splice(Entry->end(), FirstClonedBB->getInstList()); FirstClonedBB->eraseFromParent(); + if (auto *CatchAction = dyn_cast(Action)) { + WinEHCatchDirector *CatchDirector = + reinterpret_cast(Director.get()); + CatchAction->setExceptionVar(CatchDirector->getExceptionVar()); + CatchAction->setReturnTargets(CatchDirector->getReturnTargets()); + } + + Action->setHandlerBlockOrFunc(Handler); + return true; } -CloningDirector::CloningAction WinEHCatchDirector::handleInstruction( - ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { - // Intercept instructions which extract values from the landing pad aggregate. - if (auto *Extract = dyn_cast(Inst)) { - if (Extract->getAggregateOperand() == LPI) { - assert(Extract->getNumIndices() == 1 && - "Unexpected operation: extracting both landing pad values"); - assert((*(Extract->idx_begin()) == 0 || *(Extract->idx_begin()) == 1) && - "Unexpected operation: extracting an unknown landing pad element"); - - if (*(Extract->idx_begin()) == 0) { - // Element 0 doesn't directly corresponds to anything in the WinEH - // scheme. - // It will be stored to a memory location, then later loaded and finally - // the loaded value will be used as the argument to an - // llvm.eh.begincatch - // call. We're tracking it here so that we can skip the store and load. - ExtractedEHPtr = Inst; - } else { - // Element 1 corresponds to the filter selector. We'll map it to 1 for - // matching purposes, but it will also probably be stored to memory and - // reloaded, so we need to track the instuction so that we can map the - // loaded value too. - VMap[Inst] = ConstantInt::get(SelectorIDType, 1); - ExtractedSelector = Inst; - } - - // Tell the caller not to clone this instruction. - return CloningDirector::SkipInstruction; - } - // Other extract value instructions just get cloned. - return CloningDirector::CloneInstruction; +/// This BB must end in a selector dispatch. All we need to do is pass the +/// handler block to llvm.eh.actions and list it as a possible indirectbr +/// target. +void WinEHPrepare::processSEHCatchHandler(CatchHandler *CatchAction, + BasicBlock *StartBB) { + BasicBlock *HandlerBB; + BasicBlock *NextBB; + Constant *Selector; + bool Res = isSelectorDispatch(StartBB, HandlerBB, Selector, NextBB); + if (Res) { + // If this was EH dispatch, this must be a conditional branch to the handler + // block. + // FIXME: Handle instructions in the dispatch block. Currently we drop them, + // leading to crashes if some optimization hoists stuff here. + assert(CatchAction->getSelector() && HandlerBB && + "expected catch EH dispatch"); + } else { + // This must be a catch-all. Split the block after the landingpad. + assert(CatchAction->getSelector()->isNullValue() && "expected catch-all"); + HandlerBB = + StartBB->splitBasicBlock(StartBB->getFirstInsertionPt(), "catch.all"); } + CatchAction->setHandlerBlockOrFunc(BlockAddress::get(HandlerBB)); + TinyPtrVector Targets(HandlerBB); + CatchAction->setReturnTargets(Targets); +} - if (auto *Store = dyn_cast(Inst)) { - // Look for and suppress stores of the extracted landingpad values. - const Value *StoredValue = Store->getValueOperand(); - if (StoredValue == ExtractedEHPtr) { - EHPtrStoreAddr = Store->getPointerOperand(); - return CloningDirector::SkipInstruction; +void LandingPadMap::mapLandingPad(const LandingPadInst *LPad) { + // Each instance of this class should only ever be used to map a single + // landing pad. + assert(OriginLPad == nullptr || OriginLPad == LPad); + + // If the landing pad has already been mapped, there's nothing more to do. + if (OriginLPad == LPad) + return; + + OriginLPad = LPad; + + // The landingpad instruction returns an aggregate value. Typically, its + // value will be passed to a pair of extract value instructions and the + // results of those extracts are often passed to store instructions. + // In unoptimized code the stored value will often be loaded and then stored + // again. + for (auto *U : LPad->users()) { + const ExtractValueInst *Extract = dyn_cast(U); + if (!Extract) + continue; + assert(Extract->getNumIndices() == 1 && + "Unexpected operation: extracting both landing pad values"); + unsigned int Idx = *(Extract->idx_begin()); + assert((Idx == 0 || Idx == 1) && + "Unexpected operation: extracting an unknown landing pad element"); + if (Idx == 0) { + // Element 0 doesn't directly corresponds to anything in the WinEH + // scheme. + // It will be stored to a memory location, then later loaded and finally + // the loaded value will be used as the argument to an + // llvm.eh.begincatch + // call. We're tracking it here so that we can skip the store and load. + ExtractedEHPtrs.push_back(Extract); + } else if (Idx == 1) { + // Element 1 corresponds to the filter selector. We'll map it to 1 for + // matching purposes, but it will also probably be stored to memory and + // reloaded, so we need to track the instuction so that we can map the + // loaded value too. + ExtractedSelectors.push_back(Extract); } - if (StoredValue == ExtractedSelector) { - SelectorStoreAddr = Store->getPointerOperand(); - return CloningDirector::SkipInstruction; + + // Look for stores of the extracted values. + for (auto *EU : Extract->users()) { + if (auto *Store = dyn_cast(EU)) { + if (Idx == 1) { + SelectorStores.push_back(Store); + SelectorStoreAddrs.push_back(Store->getPointerOperand()); + } else { + EHPtrStores.push_back(Store); + EHPtrStoreAddrs.push_back(Store->getPointerOperand()); + } + } } + } +} - // Any other store just gets cloned. - return CloningDirector::CloneInstruction; +bool LandingPadMap::isLandingPadSpecificInst(const Instruction *Inst) const { + if (Inst == OriginLPad) + return true; + for (auto *Extract : ExtractedEHPtrs) { + if (Inst == Extract) + return true; + } + for (auto *Extract : ExtractedSelectors) { + if (Inst == Extract) + return true; + } + for (auto *Store : EHPtrStores) { + if (Inst == Store) + return true; + } + for (auto *Store : SelectorStores) { + if (Inst == Store) + return true; + } + + return false; +} + +void LandingPadMap::remapSelector(ValueToValueMapTy &VMap, + Value *MappedValue) const { + // Remap all selector extract instructions to the specified value. + for (auto *Extract : ExtractedSelectors) + VMap[Extract] = MappedValue; +} + +bool LandingPadMap::mapIfEHLoad(const LoadInst *Load, + SmallVectorImpl &Stores, + SmallVectorImpl &StoreAddrs) { + // This makes the assumption that a store we've previously seen dominates + // this load instruction. That might seem like a rather huge assumption, + // but given the way that landingpads are constructed its fairly safe. + // FIXME: Add debug/assert code that verifies this. + const Value *LoadAddr = Load->getPointerOperand(); + for (auto *StoreAddr : StoreAddrs) { + if (LoadAddr == StoreAddr) { + // Handle the common debug scenario where this loaded value is stored + // to a different location. + for (auto *U : Load->users()) { + if (auto *Store = dyn_cast(U)) { + Stores.push_back(Store); + StoreAddrs.push_back(Store->getPointerOperand()); + } + } + return true; + } } + return false; +} + +CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction( + ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { + // If this is one of the boilerplate landing pad instructions, skip it. + // The instruction will have already been remapped in VMap. + if (LPadMap.isLandingPadSpecificInst(Inst)) + return CloningDirector::SkipInstruction; if (auto *Load = dyn_cast(Inst)) { // Look for loads of (previously suppressed) landingpad values. - // The EHPtr load can be ignored (it should only be used as - // an argument to llvm.eh.begincatch), but the selector value - // needs to be mapped to a constant value of 1 to be used to - // simplify the branching to always flow to the current handler. - const Value *LoadAddr = Load->getPointerOperand(); - if (LoadAddr == EHPtrStoreAddr) { - VMap[Inst] = UndefValue::get(Int8PtrType); + // The EHPtr load can be mapped to an undef value as it should only be used + // as an argument to llvm.eh.begincatch, but the selector value needs to be + // mapped to a constant value of 1. This value will be used to simplify the + // branching to always flow to the current handler. + if (LPadMap.mapIfSelectorLoad(Load)) { + VMap[Inst] = ConstantInt::get(SelectorIDType, 1); return CloningDirector::SkipInstruction; } - if (LoadAddr == SelectorStoreAddr) { - VMap[Inst] = ConstantInt::get(SelectorIDType, 1); + if (LPadMap.mapIfEHPtrLoad(Load)) { + VMap[Inst] = UndefValue::get(Int8PtrType); return CloningDirector::SkipInstruction; } @@ -519,108 +910,576 @@ CloningDirector::CloningAction WinEHCatchDirector::handleInstruction( return CloningDirector::CloneInstruction; } - if (match(Inst, m_Intrinsic())) { - // The argument to the call is some form of the first element of the - // landingpad aggregate value, but that doesn't matter. It isn't used - // here. - // The return value of this instruction, however, is used to access the - // EH object pointer. We have generated an instruction to get that value - // from the EH alloc block, so we can just map to that here. - VMap[Inst] = EHObj; - return CloningDirector::SkipInstruction; - } - if (match(Inst, m_Intrinsic())) { - auto *IntrinCall = dyn_cast(Inst); - // It might be interesting to track whether or not we are inside a catch - // function, but that might make the algorithm more brittle than it needs - // to be. - - // The end catch call can occur in one of two places: either in a - // landingpad - // block that is part of the catch handlers exception mechanism, or at the - // end of the catch block. If it occurs in a landing pad, we must skip it - // and continue so that the landing pad gets cloned. - // FIXME: This case isn't fully supported yet and shouldn't turn up in any - // of the test cases until it is. - if (IntrinCall->getParent()->isLandingPad()) - return CloningDirector::SkipInstruction; - - // If an end catch occurs anywhere else the next instruction should be an - // unconditional branch instruction that we want to replace with a return - // to the the address of the branch target. - const BasicBlock *EndCatchBB = IntrinCall->getParent(); - const TerminatorInst *Terminator = EndCatchBB->getTerminator(); - const BranchInst *Branch = dyn_cast(Terminator); - assert(Branch && Branch->isUnconditional()); - assert(std::next(BasicBlock::const_iterator(IntrinCall)) == - BasicBlock::const_iterator(Branch)); - - ReturnInst::Create(NewBB->getContext(), - BlockAddress::get(Branch->getSuccessor(0)), NewBB); - - // We just added a terminator to the cloned block. - // Tell the caller to stop processing the current basic block so that - // the branch instruction will be skipped. + // Nested landing pads will be cloned as stubs, with just the + // landingpad instruction and an unreachable instruction. When + // all landingpads have been outlined, we'll replace this with the + // llvm.eh.actions call and indirect branch created when the + // landing pad was outlined. + if (auto *NestedLPad = dyn_cast(Inst)) { + Instruction *NewInst = NestedLPad->clone(); + if (NestedLPad->hasName()) + NewInst->setName(NestedLPad->getName()); + // FIXME: Store this mapping somewhere else also. + VMap[NestedLPad] = NewInst; + BasicBlock::InstListType &InstList = NewBB->getInstList(); + InstList.push_back(NewInst); + InstList.push_back(new UnreachableInst(NewBB->getContext())); return CloningDirector::StopCloningBB; } - if (match(Inst, m_Intrinsic())) { - auto *IntrinCall = dyn_cast(Inst); - Value *Selector = IntrinCall->getArgOperand(0)->stripPointerCasts(); - // This causes a replacement that will collapse the landing pad CFG based - // on the filter function we intend to match. - if (Selector == CurrentSelector) - VMap[Inst] = ConstantInt::get(SelectorIDType, 1); - else - VMap[Inst] = ConstantInt::get(SelectorIDType, 0); - // Tell the caller not to clone this instruction. - return CloningDirector::SkipInstruction; - } + + if (auto *Invoke = dyn_cast(Inst)) + return handleInvoke(VMap, Invoke, NewBB); + + if (auto *Resume = dyn_cast(Inst)) + return handleResume(VMap, Resume, NewBB); + + if (match(Inst, m_Intrinsic())) + return handleBeginCatch(VMap, Inst, NewBB); + if (match(Inst, m_Intrinsic())) + return handleEndCatch(VMap, Inst, NewBB); + if (match(Inst, m_Intrinsic())) + return handleTypeIdFor(VMap, Inst, NewBB); // Continue with the default cloning behavior. return CloningDirector::CloneInstruction; } +CloningDirector::CloningAction WinEHCatchDirector::handleBeginCatch( + ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { + // The argument to the call is some form of the first element of the + // landingpad aggregate value, but that doesn't matter. It isn't used + // here. + // The second argument is an outparameter where the exception object will be + // stored. Typically the exception object is a scalar, but it can be an + // aggregate when catching by value. + // FIXME: Leave something behind to indicate where the exception object lives + // for this handler. Should it be part of llvm.eh.actions? + assert(ExceptionObjectVar == nullptr && "Multiple calls to " + "llvm.eh.begincatch found while " + "outlining catch handler."); + ExceptionObjectVar = Inst->getOperand(1)->stripPointerCasts(); + return CloningDirector::SkipInstruction; +} + +CloningDirector::CloningAction +WinEHCatchDirector::handleEndCatch(ValueToValueMapTy &VMap, + const Instruction *Inst, BasicBlock *NewBB) { + auto *IntrinCall = dyn_cast(Inst); + // It might be interesting to track whether or not we are inside a catch + // function, but that might make the algorithm more brittle than it needs + // to be. + + // The end catch call can occur in one of two places: either in a + // landingpad block that is part of the catch handlers exception mechanism, + // or at the end of the catch block. If it occurs in a landing pad, we must + // skip it and continue so that the landing pad gets cloned. + // FIXME: This case isn't fully supported yet and shouldn't turn up in any + // of the test cases until it is. + if (IntrinCall->getParent()->isLandingPad()) + return CloningDirector::SkipInstruction; + + // If an end catch occurs anywhere else the next instruction should be an + // unconditional branch instruction that we want to replace with a return + // to the the address of the branch target. + const BasicBlock *EndCatchBB = IntrinCall->getParent(); + const TerminatorInst *Terminator = EndCatchBB->getTerminator(); + const BranchInst *Branch = dyn_cast(Terminator); + assert(Branch && Branch->isUnconditional()); + assert(std::next(BasicBlock::const_iterator(IntrinCall)) == + BasicBlock::const_iterator(Branch)); + + BasicBlock *ContinueLabel = Branch->getSuccessor(0); + ReturnInst::Create(NewBB->getContext(), BlockAddress::get(ContinueLabel), + NewBB); + ReturnTargets.push_back(ContinueLabel); + + // We just added a terminator to the cloned block. + // Tell the caller to stop processing the current basic block so that + // the branch instruction will be skipped. + return CloningDirector::StopCloningBB; +} + +CloningDirector::CloningAction WinEHCatchDirector::handleTypeIdFor( + ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { + auto *IntrinCall = dyn_cast(Inst); + Value *Selector = IntrinCall->getArgOperand(0)->stripPointerCasts(); + // This causes a replacement that will collapse the landing pad CFG based + // on the filter function we intend to match. + if (Selector == CurrentSelector) + VMap[Inst] = ConstantInt::get(SelectorIDType, 1); + else + VMap[Inst] = ConstantInt::get(SelectorIDType, 0); + // Tell the caller not to clone this instruction. + return CloningDirector::SkipInstruction; +} + +CloningDirector::CloningAction +WinEHCatchDirector::handleInvoke(ValueToValueMapTy &VMap, + const InvokeInst *Invoke, BasicBlock *NewBB) { + return CloningDirector::CloneInstruction; +} + +CloningDirector::CloningAction +WinEHCatchDirector::handleResume(ValueToValueMapTy &VMap, + const ResumeInst *Resume, BasicBlock *NewBB) { + // Resume instructions shouldn't be reachable from catch handlers. + // We still need to handle it, but it will be pruned. + BasicBlock::InstListType &InstList = NewBB->getInstList(); + InstList.push_back(new UnreachableInst(NewBB->getContext())); + return CloningDirector::StopCloningBB; +} + +CloningDirector::CloningAction WinEHCleanupDirector::handleBeginCatch( + ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { + // Catch blocks within cleanup handlers will always be unreachable. + // We'll insert an unreachable instruction now, but it will be pruned + // before the cloning process is complete. + BasicBlock::InstListType &InstList = NewBB->getInstList(); + InstList.push_back(new UnreachableInst(NewBB->getContext())); + return CloningDirector::StopCloningBB; +} + +CloningDirector::CloningAction WinEHCleanupDirector::handleEndCatch( + ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { + // Catch blocks within cleanup handlers will always be unreachable. + // We'll insert an unreachable instruction now, but it will be pruned + // before the cloning process is complete. + BasicBlock::InstListType &InstList = NewBB->getInstList(); + InstList.push_back(new UnreachableInst(NewBB->getContext())); + return CloningDirector::StopCloningBB; +} + +CloningDirector::CloningAction WinEHCleanupDirector::handleTypeIdFor( + ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { + // If we encounter a selector comparison while cloning a cleanup handler, + // we want to stop cloning immediately. Anything after the dispatch + // will be outlined into a different handler. + BasicBlock *CatchHandler; + Constant *Selector; + BasicBlock *NextBB; + if (isSelectorDispatch(const_cast(Inst->getParent()), + CatchHandler, Selector, NextBB)) { + ReturnInst::Create(NewBB->getContext(), nullptr, NewBB); + return CloningDirector::StopCloningBB; + } + // If eg.typeid.for is called for any other reason, it can be ignored. + VMap[Inst] = ConstantInt::get(SelectorIDType, 0); + return CloningDirector::SkipInstruction; +} + +CloningDirector::CloningAction WinEHCleanupDirector::handleInvoke( + ValueToValueMapTy &VMap, const InvokeInst *Invoke, BasicBlock *NewBB) { + // All invokes in cleanup handlers can be replaced with calls. + SmallVector CallArgs(Invoke->op_begin(), Invoke->op_end() - 3); + // Insert a normal call instruction... + CallInst *NewCall = + CallInst::Create(const_cast(Invoke->getCalledValue()), CallArgs, + Invoke->getName(), NewBB); + NewCall->setCallingConv(Invoke->getCallingConv()); + NewCall->setAttributes(Invoke->getAttributes()); + NewCall->setDebugLoc(Invoke->getDebugLoc()); + VMap[Invoke] = NewCall; + + // Insert an unconditional branch to the normal destination. + BranchInst::Create(Invoke->getNormalDest(), NewBB); + + // The unwind destination won't be cloned into the new function, so + // we don't need to clean up its phi nodes. + + // We just added a terminator to the cloned block. + // Tell the caller to stop processing the current basic block. + return CloningDirector::StopCloningBB; +} + +CloningDirector::CloningAction WinEHCleanupDirector::handleResume( + ValueToValueMapTy &VMap, const ResumeInst *Resume, BasicBlock *NewBB) { + ReturnInst::Create(NewBB->getContext(), nullptr, NewBB); + + // We just added a terminator to the cloned block. + // Tell the caller to stop processing the current basic block so that + // the branch instruction will be skipped. + return CloningDirector::StopCloningBB; +} + WinEHFrameVariableMaterializer::WinEHFrameVariableMaterializer( Function *OutlinedFn, FrameVarInfoMap &FrameVarInfo) : FrameVarInfo(FrameVarInfo), Builder(OutlinedFn->getContext()) { Builder.SetInsertPoint(&OutlinedFn->getEntryBlock()); - // FIXME: Do something with the FrameVarMapped so that it is shared across the - // function. } Value *WinEHFrameVariableMaterializer::materializeValueFor(Value *V) { - // If we're asked to materialize an alloca variable, we temporarily - // create a matching alloca in the outlined function. When all the - // outlining is complete, we'll collect these into a structure and - // replace these temporary allocas with GEPs referencing the frame - // allocation block. + // If we're asked to materialize a value that is an instruction, we + // temporarily create an alloca in the outlined function and add this + // to the FrameVarInfo map. When all the outlining is complete, we'll + // collect these into a structure, spilling non-alloca values in the + // parent frame as necessary, and replace these temporary allocas with + // GEPs referencing the frame allocation block. + + // If the value is an alloca, the mapping is direct. if (auto *AV = dyn_cast(V)) { - AllocaInst *NewAlloca = Builder.CreateAlloca( - AV->getAllocatedType(), AV->getArraySize(), AV->getName()); - FrameVarInfo[AV].Allocas.push_back(NewAlloca); + AllocaInst *NewAlloca = dyn_cast(AV->clone()); + Builder.Insert(NewAlloca, AV->getName()); + FrameVarInfo[AV].push_back(NewAlloca); return NewAlloca; } -// FIXME: Do PHI nodes need special handling? + // For other types of instructions or arguments, we need an alloca based on + // the value's type and a load of the alloca. The alloca will be replaced + // by a GEP, but the load will stay. In the parent function, the value will + // be spilled to a location in the frame allocation block. + if (isa(V) || isa(V)) { + AllocaInst *NewAlloca = + Builder.CreateAlloca(V->getType(), nullptr, "eh.temp.alloca"); + FrameVarInfo[V].push_back(NewAlloca); + LoadInst *NewLoad = Builder.CreateLoad(NewAlloca, V->getName() + ".reload"); + return NewLoad; + } -// FIXME: Are there other cases we can handle better? GEP, ExtractValue, etc. + // Don't materialize other values. + return nullptr; +} -// FIXME: This doesn't work during cloning because it finds an instruction -// in the use list that isn't yet part of a basic block. -#if 0 - // If we're asked to remap some other instruction, we'll need to - // spill it to an alloca variable in the parent function and add a - // temporary alloca in the outlined function to be processed as - // described above. - Instruction *Inst = dyn_cast(V); - if (Inst) { - AllocaInst *Spill = DemoteRegToStack(*Inst, true); - AllocaInst *NewAlloca = Builder.CreateAlloca(Spill->getAllocatedType(), - Spill->getArraySize()); - FrameVarMap[AV] = NewAlloca; - return NewAlloca; +// This function maps the catch and cleanup handlers that are reachable from the +// specified landing pad. The landing pad sequence will have this basic shape: +// +// +// +// +// +// +// +// +// ... +// +// Any of the cleanup slots may be absent. The cleanup slots may be occupied by +// any arbitrary control flow, but all paths through the cleanup code must +// eventually reach the next selector comparison and no path can skip to a +// different selector comparisons, though some paths may terminate abnormally. +// Therefore, we will use a depth first search from the start of any given +// cleanup block and stop searching when we find the next selector comparison. +// +// If the landingpad instruction does not have a catch clause, we will assume +// that any instructions other than selector comparisons and catch handlers can +// be ignored. In practice, these will only be the boilerplate instructions. +// +// The catch handlers may also have any control structure, but we are only +// interested in the start of the catch handlers, so we don't need to actually +// follow the flow of the catch handlers. The start of the catch handlers can +// be located from the compare instructions, but they can be skipped in the +// flow by following the contrary branch. +void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad, + LandingPadActions &Actions) { + unsigned int NumClauses = LPad->getNumClauses(); + unsigned int HandlersFound = 0; + BasicBlock *BB = LPad->getParent(); + + DEBUG(dbgs() << "Mapping landing pad: " << BB->getName() << "\n"); + + if (NumClauses == 0) { + // This landing pad contains only cleanup code. + CleanupHandler *Action = new CleanupHandler(BB); + CleanupHandlerMap[BB] = Action; + Actions.insertCleanupHandler(Action); + DEBUG(dbgs() << " Assuming cleanup code in block " << BB->getName() + << "\n"); + assert(LPad->isCleanup()); + return; + } + + VisitedBlockSet VisitedBlocks; + + while (HandlersFound != NumClauses) { + BasicBlock *NextBB = nullptr; + + // See if the clause we're looking for is a catch-all. + // If so, the catch begins immediately. + if (isa(LPad->getClause(HandlersFound))) { + // The catch all must occur last. + assert(HandlersFound == NumClauses - 1); + + // For C++ EH, check if there is any interesting cleanup code before we + // begin the catch. This is important because cleanups cannot rethrow + // exceptions but code called from catches can. For SEH, it isn't + // important if some finally code before a catch-all is executed out of + // line or after recovering from the exception. + if (Personality == EHPersonality::MSVC_CXX) { + if (auto *CleanupAction = findCleanupHandler(BB, BB)) { + // Add a cleanup entry to the list + Actions.insertCleanupHandler(CleanupAction); + DEBUG(dbgs() << " Found cleanup code in block " + << CleanupAction->getStartBlock()->getName() << "\n"); + } + } + + // Add the catch handler to the action list. + CatchHandler *Action = + new CatchHandler(BB, LPad->getClause(HandlersFound), nullptr); + CatchHandlerMap[BB] = Action; + Actions.insertCatchHandler(Action); + DEBUG(dbgs() << " Catch all handler at block " << BB->getName() << "\n"); + ++HandlersFound; + + // Once we reach a catch-all, don't expect to hit a resume instruction. + BB = nullptr; + break; + } + + CatchHandler *CatchAction = findCatchHandler(BB, NextBB, VisitedBlocks); + // See if there is any interesting code executed before the dispatch. + if (auto *CleanupAction = + findCleanupHandler(BB, CatchAction->getStartBlock())) { + // Add a cleanup entry to the list + Actions.insertCleanupHandler(CleanupAction); + DEBUG(dbgs() << " Found cleanup code in block " + << CleanupAction->getStartBlock()->getName() << "\n"); + } + + assert(CatchAction); + ++HandlersFound; + + // Add the catch handler to the action list. + Actions.insertCatchHandler(CatchAction); + DEBUG(dbgs() << " Found catch dispatch in block " + << CatchAction->getStartBlock()->getName() << "\n"); + + // Move on to the block after the catch handler. + BB = NextBB; + } + + // If we didn't wind up in a catch-all, see if there is any interesting code + // executed before the resume. + if (auto *CleanupAction = findCleanupHandler(BB, BB)) { + // Add a cleanup entry to the list + Actions.insertCleanupHandler(CleanupAction); + DEBUG(dbgs() << " Found cleanup code in block " + << CleanupAction->getStartBlock()->getName() << "\n"); + } + + // It's possible that some optimization moved code into a landingpad that + // wasn't + // previously being used for cleanup. If that happens, we need to execute + // that + // extra code from a cleanup handler. + if (Actions.includesCleanup() && !LPad->isCleanup()) + LPad->setCleanup(true); +} + +// This function searches starting with the input block for the next +// block that terminates with a branch whose condition is based on a selector +// comparison. This may be the input block. See the mapLandingPadBlocks +// comments for a discussion of control flow assumptions. +// +CatchHandler *WinEHPrepare::findCatchHandler(BasicBlock *BB, + BasicBlock *&NextBB, + VisitedBlockSet &VisitedBlocks) { + // See if we've already found a catch handler use it. + // Call count() first to avoid creating a null entry for blocks + // we haven't seen before. + if (CatchHandlerMap.count(BB) && CatchHandlerMap[BB] != nullptr) { + CatchHandler *Action = cast(CatchHandlerMap[BB]); + NextBB = Action->getNextBB(); + return Action; } -#endif + // VisitedBlocks applies only to the current search. We still + // need to consider blocks that we've visited while mapping other + // landing pads. + VisitedBlocks.insert(BB); + + BasicBlock *CatchBlock = nullptr; + Constant *Selector = nullptr; + + // If this is the first time we've visited this block from any landing pad + // look to see if it is a selector dispatch block. + if (!CatchHandlerMap.count(BB)) { + if (isSelectorDispatch(BB, CatchBlock, Selector, NextBB)) { + CatchHandler *Action = new CatchHandler(BB, Selector, NextBB); + CatchHandlerMap[BB] = Action; + return Action; + } + } + + // Visit each successor, looking for the dispatch. + // FIXME: We expect to find the dispatch quickly, so this will probably + // work better as a breadth first search. + for (BasicBlock *Succ : successors(BB)) { + if (VisitedBlocks.count(Succ)) + continue; + + CatchHandler *Action = findCatchHandler(Succ, NextBB, VisitedBlocks); + if (Action) + return Action; + } + return nullptr; +} + +// These are helper functions to combine repeated code from findCleanupHandler. +static CleanupHandler *createCleanupHandler(CleanupHandlerMapTy &CleanupHandlerMap, + BasicBlock *BB) { + CleanupHandler *Action = new CleanupHandler(BB); + CleanupHandlerMap[BB] = Action; + return Action; +} + +// This function searches starting with the input block for the next block that +// contains code that is not part of a catch handler and would not be eliminated +// during handler outlining. +// +CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, + BasicBlock *EndBB) { + // Here we will skip over the following: + // + // landing pad prolog: + // + // Unconditional branches + // + // Selector dispatch + // + // Resume pattern + // + // Anything else marks the start of an interesting block + + BasicBlock *BB = StartBB; + // Anything other than an unconditional branch will kick us out of this loop + // one way or another. + while (BB) { + // If we've already scanned this block, don't scan it again. If it is + // a cleanup block, there will be an action in the CleanupHandlerMap. + // If we've scanned it and it is not a cleanup block, there will be a + // nullptr in the CleanupHandlerMap. If we have not scanned it, there will + // be no entry in the CleanupHandlerMap. We must call count() first to + // avoid creating a null entry for blocks we haven't scanned. + if (CleanupHandlerMap.count(BB)) { + if (auto *Action = CleanupHandlerMap[BB]) { + return cast(Action); + } else { + // Here we handle the case where the cleanup handler map contains a + // value for this block but the value is a nullptr. This means that + // we have previously analyzed the block and determined that it did + // not contain any cleanup code. Based on the earlier analysis, we + // know the the block must end in either an unconditional branch, a + // resume or a conditional branch that is predicated on a comparison + // with a selector. Either the resume or the selector dispatch + // would terminate the search for cleanup code, so the unconditional + // branch is the only case for which we might need to continue + // searching. + if (BB == EndBB) + return nullptr; + BasicBlock *SuccBB; + if (!match(BB->getTerminator(), m_UnconditionalBr(SuccBB))) + return nullptr; + BB = SuccBB; + continue; + } + } + + // Create an entry in the cleanup handler map for this block. Initially + // we create an entry that says this isn't a cleanup block. If we find + // cleanup code, the caller will replace this entry. + CleanupHandlerMap[BB] = nullptr; + + TerminatorInst *Terminator = BB->getTerminator(); + + // Landing pad blocks have extra instructions we need to accept. + LandingPadMap *LPadMap = nullptr; + if (BB->isLandingPad()) { + LandingPadInst *LPad = BB->getLandingPadInst(); + LPadMap = &LPadMaps[LPad]; + if (!LPadMap->isInitialized()) + LPadMap->mapLandingPad(LPad); + } + + // Look for the bare resume pattern: + // %exn2 = load i8** %exn.slot + // %sel2 = load i32* %ehselector.slot + // %lpad.val1 = insertvalue { i8*, i32 } undef, i8* %exn2, 0 + // %lpad.val2 = insertvalue { i8*, i32 } %lpad.val1, i32 %sel2, 1 + // resume { i8*, i32 } %lpad.val2 + if (auto *Resume = dyn_cast(Terminator)) { + InsertValueInst *Insert1 = nullptr; + InsertValueInst *Insert2 = nullptr; + Value *ResumeVal = Resume->getOperand(0); + // If there is only one landingpad, we may use the lpad directly with no + // insertions. + if (isa(ResumeVal)) + return nullptr; + if (!isa(ResumeVal)) { + Insert2 = dyn_cast(ResumeVal); + if (!Insert2) + return createCleanupHandler(CleanupHandlerMap, BB); + Insert1 = dyn_cast(Insert2->getAggregateOperand()); + if (!Insert1) + return createCleanupHandler(CleanupHandlerMap, BB); + } + for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), IE = BB->end(); + II != IE; ++II) { + Instruction *Inst = II; + if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) + continue; + if (Inst == Insert1 || Inst == Insert2 || Inst == Resume) + continue; + if (!Inst->hasOneUse() || + (Inst->user_back() != Insert1 && Inst->user_back() != Insert2)) { + return createCleanupHandler(CleanupHandlerMap, BB); + } + } + return nullptr; + } + + BranchInst *Branch = dyn_cast(Terminator); + if (Branch) { + if (Branch->isConditional()) { + // Look for the selector dispatch. + // %sel = load i32* %ehselector.slot + // %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIf to i8*)) + // %matches = icmp eq i32 %sel12, %2 + // br i1 %matches, label %catch14, label %eh.resume + CmpInst *Compare = dyn_cast(Branch->getCondition()); + if (!Compare || !Compare->isEquality()) + return createCleanupHandler(CleanupHandlerMap, BB); + for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), + IE = BB->end(); + II != IE; ++II) { + Instruction *Inst = II; + if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) + continue; + if (Inst == Compare || Inst == Branch) + continue; + if (!Inst->hasOneUse() || (Inst->user_back() != Compare)) + return createCleanupHandler(CleanupHandlerMap, BB); + if (match(Inst, m_Intrinsic())) + continue; + if (!isa(Inst)) + return createCleanupHandler(CleanupHandlerMap, BB); + } + // The selector dispatch block should always terminate our search. + assert(BB == EndBB); + return nullptr; + } else { + // Look for empty blocks with unconditional branches. + for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), + IE = BB->end(); + II != IE; ++II) { + Instruction *Inst = II; + if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) + continue; + if (Inst == Branch) + continue; + if (match(Inst, m_Intrinsic())) + continue; + // Anything else makes this interesting cleanup code. + return createCleanupHandler(CleanupHandlerMap, BB); + } + if (BB == EndBB) + return nullptr; + // The branch was unconditional. + BB = Branch->getSuccessor(0); + continue; + } // End else of if branch was conditional + } // End if Branch + + // Anything else makes this interesting cleanup code. + return createCleanupHandler(CleanupHandlerMap, BB); + } return nullptr; } diff --git a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp index 7d77290..9f56214 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp @@ -9,6 +9,7 @@ #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/Casting.h" #include "llvm/Support/DataTypes.h" diff --git a/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp index fdb6dd2..cd6fbef 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp @@ -67,8 +67,7 @@ void DWARFDebugLoc::parse(DataExtractor data, unsigned AddressSize) { // A single location description describing the location of the object... StringRef str = data.getData().substr(Offset, Bytes); Offset += Bytes; - E.Loc.reserve(str.size()); - std::copy(str.begin(), str.end(), std::back_inserter(E.Loc)); + E.Loc.append(str.begin(), str.end()); Loc.Entries.push_back(std::move(E)); } } diff --git a/lib/DebugInfo/DWARF/DWARFFormValue.cpp b/lib/DebugInfo/DWARF/DWARFFormValue.cpp index 45bd197..6946f83 100644 --- a/lib/DebugInfo/DWARF/DWARFFormValue.cpp +++ b/lib/DebugInfo/DWARF/DWARFFormValue.cpp @@ -18,6 +18,7 @@ #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include +#include using namespace llvm; using namespace dwarf; using namespace syntax; @@ -557,6 +558,24 @@ Optional DWARFFormValue::getAsUnsignedConstant() const { return Value.uval; } +Optional DWARFFormValue::getAsSignedConstant() const { + if ((!isFormClass(FC_Constant) && !isFormClass(FC_Flag)) || + (Form == DW_FORM_udata && uint64_t(LLONG_MAX) < Value.uval)) + return None; + switch (Form) { + case DW_FORM_data4: + return int32_t(Value.uval); + case DW_FORM_data2: + return int16_t(Value.uval); + case DW_FORM_data1: + return int8_t(Value.uval); + case DW_FORM_sdata: + case DW_FORM_data8: + default: + return Value.sval; + } +} + Optional> DWARFFormValue::getAsBlock() const { if (!isFormClass(FC_Block) && !isFormClass(FC_Exprloc)) return None; diff --git a/lib/DebugInfo/PDB/CMakeLists.txt b/lib/DebugInfo/PDB/CMakeLists.txt index 87e357e..ed8c674 100644 --- a/lib/DebugInfo/PDB/CMakeLists.txt +++ b/lib/DebugInfo/PDB/CMakeLists.txt @@ -73,4 +73,4 @@ add_llvm_library(LLVMDebugInfoPDB ${LIBPDB_ADDITIONAL_HEADER_DIRS} ) -target_link_libraries(LLVMDebugInfoPDB ${cmake_2_8_12_INTERFACE} "${LIBPDB_ADDITIONAL_LIBRARIES}") +target_link_libraries(LLVMDebugInfoPDB INTERFACE "${LIBPDB_ADDITIONAL_LIBRARIES}") diff --git a/lib/DebugInfo/PDB/DIA/DIASession.cpp b/lib/DebugInfo/PDB/DIA/DIASession.cpp index 24791f2..4966bea 100644 --- a/lib/DebugInfo/PDB/DIA/DIASession.cpp +++ b/lib/DebugInfo/PDB/DIA/DIASession.cpp @@ -23,28 +23,45 @@ namespace {} DIASession::DIASession(CComPtr DiaSession) : Session(DiaSession) {} -DIASession *DIASession::createFromPdb(StringRef Path) { - CComPtr DataSource; - CComPtr Session; +PDB_ErrorCode DIASession::createFromPdb(StringRef Path, + std::unique_ptr &Session) { + CComPtr DiaDataSource; + CComPtr DiaSession; // We assume that CoInitializeEx has already been called by the executable. - HRESULT Result = ::CoCreateInstance(CLSID_DiaSource, nullptr, - CLSCTX_INPROC_SERVER, IID_IDiaDataSource, - reinterpret_cast(&DataSource)); + HRESULT Result = ::CoCreateInstance( + CLSID_DiaSource, nullptr, CLSCTX_INPROC_SERVER, IID_IDiaDataSource, + reinterpret_cast(&DiaDataSource)); if (FAILED(Result)) - return nullptr; + return PDB_ErrorCode::NoPdbImpl; llvm::SmallVector Path16; if (!llvm::convertUTF8ToUTF16String(Path, Path16)) - return nullptr; + return PDB_ErrorCode::InvalidPath; const wchar_t *Path16Str = reinterpret_cast(Path16.data()); - if (FAILED(DataSource->loadDataFromPdb(Path16Str))) - return nullptr; - - if (FAILED(DataSource->openSession(&Session))) - return nullptr; - return new DIASession(Session); + if (FAILED(Result = DiaDataSource->loadDataFromPdb(Path16Str))) { + if (Result == E_PDB_NOT_FOUND) + return PDB_ErrorCode::InvalidPath; + else if (Result == E_PDB_FORMAT) + return PDB_ErrorCode::InvalidFileFormat; + else if (Result == E_INVALIDARG) + return PDB_ErrorCode::InvalidParameter; + else if (Result == E_UNEXPECTED) + return PDB_ErrorCode::AlreadyLoaded; + else + return PDB_ErrorCode::UnknownError; + } + + if (FAILED(Result = DiaDataSource->openSession(&DiaSession))) { + if (Result == E_OUTOFMEMORY) + return PDB_ErrorCode::NoMemory; + else + return PDB_ErrorCode::UnknownError; + } + + Session.reset(new DIASession(DiaSession)); + return PDB_ErrorCode::Success; } uint64_t DIASession::getLoadAddress() const { diff --git a/lib/DebugInfo/PDB/PDB.cpp b/lib/DebugInfo/PDB/PDB.cpp index aa84c28..a07396d 100644 --- a/lib/DebugInfo/PDB/PDB.cpp +++ b/lib/DebugInfo/PDB/PDB.cpp @@ -20,11 +20,11 @@ using namespace llvm; -std::unique_ptr llvm::createPDBReader(PDB_ReaderType Type, - StringRef Path) { +PDB_ErrorCode llvm::createPDBReader(PDB_ReaderType Type, StringRef Path, + std::unique_ptr &Session) { // Create the correct concrete instance type based on the value of Type. #if HAVE_DIA_SDK - return std::unique_ptr(DIASession::createFromPdb(Path)); + return DIASession::createFromPdb(Path, Session); #endif - return nullptr; + return PDB_ErrorCode::NoPdbImpl; } diff --git a/lib/DebugInfo/PDB/PDBExtras.cpp b/lib/DebugInfo/PDB/PDBExtras.cpp index 1002b2e..4b9437c 100644 --- a/lib/DebugInfo/PDB/PDBExtras.cpp +++ b/lib/DebugInfo/PDB/PDBExtras.cpp @@ -247,27 +247,21 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const PDB_SymType &Tag) { return OS; } -raw_ostream &llvm::operator<<(raw_ostream &OS, const PDB_BuiltinType &Type) { +raw_ostream &llvm::operator<<(raw_ostream &OS, const PDB_MemberAccess &Access) { + switch (Access) { + CASE_OUTPUT_ENUM_CLASS_STR(PDB_MemberAccess, Public, "public", OS) + CASE_OUTPUT_ENUM_CLASS_STR(PDB_MemberAccess, Protected, "protected", OS) + CASE_OUTPUT_ENUM_CLASS_STR(PDB_MemberAccess, Private, "private", OS) + } + return OS; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const PDB_UdtType &Type) { switch (Type) { - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Void, "void", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Char, "char", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, WCharT, "wchar_t", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Int, "int", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, UInt, "uint", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Float, "float", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, BCD, "BCD", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Bool, "bool", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Long, "long", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, ULong, "ulong", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Currency, "CURRENCY", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Date, "DATE", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Variant, "VARIANT", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Complex, "complex", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, Bitfield, "bitfield", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, BSTR, "BSTR", OS) - CASE_OUTPUT_ENUM_CLASS_STR(PDB_BuiltinType, HResult, "HRESULT", OS) - default: - break; + CASE_OUTPUT_ENUM_CLASS_STR(PDB_UdtType, Class, "class", OS) + CASE_OUTPUT_ENUM_CLASS_STR(PDB_UdtType, Struct, "struct", OS) + CASE_OUTPUT_ENUM_CLASS_STR(PDB_UdtType, Interface, "interface", OS) + CASE_OUTPUT_ENUM_CLASS_STR(PDB_UdtType, Union, "union", OS) } return OS; } @@ -309,7 +303,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const Variant &Value) { OS << Value.Int64; break; case PDB_VariantType::Int8: - OS << Value.Int8; + OS << static_cast(Value.Int8); break; case PDB_VariantType::Single: OS << Value.Single; @@ -324,12 +318,11 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const Variant &Value) { OS << Value.UInt64; break; case PDB_VariantType::UInt8: - OS << Value.UInt8; + OS << static_cast(Value.UInt8); break; default: OS << Value.Type; } - OS << " {" << Value.Type << "}"; return OS; } diff --git a/lib/DebugInfo/PDB/PDBSymDumper.cpp b/lib/DebugInfo/PDB/PDBSymDumper.cpp index 0f29c74..121e2d1 100644 --- a/lib/DebugInfo/PDB/PDBSymDumper.cpp +++ b/lib/DebugInfo/PDB/PDBSymDumper.cpp @@ -21,157 +21,126 @@ PDBSymDumper::PDBSymDumper(bool ShouldRequireImpl) PDBSymDumper::~PDBSymDumper() {} -void PDBSymDumper::dump(const PDBSymbolAnnotation &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolAnnotation &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolAnnotation) } -void PDBSymDumper::dump(const PDBSymbolBlock &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolBlock &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolBlock) } -void PDBSymDumper::dump(const PDBSymbolCompiland &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolCompiland &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolCompiland) } -void PDBSymDumper::dump(const PDBSymbolCompilandDetails &Symbol, - raw_ostream &OS, int Indent) { +void PDBSymDumper::dump(const PDBSymbolCompilandDetails &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolCompilandDetails) } -void PDBSymDumper::dump(const PDBSymbolCompilandEnv &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolCompilandEnv &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolCompilandEnv) } -void PDBSymDumper::dump(const PDBSymbolCustom &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolCustom &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolCustom) } -void PDBSymDumper::dump(const PDBSymbolData &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolData &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolData) } -void PDBSymDumper::dump(const PDBSymbolExe &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolExe &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolExe) } -void PDBSymDumper::dump(const PDBSymbolFunc &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolFunc &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolFunc) } -void PDBSymDumper::dump(const PDBSymbolFuncDebugEnd &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolFuncDebugEnd &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolFuncDebugEnd) } -void PDBSymDumper::dump(const PDBSymbolFuncDebugStart &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolFuncDebugStart &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolFuncDebugStart) } -void PDBSymDumper::dump(const PDBSymbolLabel &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolLabel &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolLabel) } -void PDBSymDumper::dump(const PDBSymbolPublicSymbol &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolPublicSymbol &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolPublicSymbol) } -void PDBSymDumper::dump(const PDBSymbolThunk &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolThunk &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolThunk) } -void PDBSymDumper::dump(const PDBSymbolTypeArray &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeArray &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeArray) } -void PDBSymDumper::dump(const PDBSymbolTypeBaseClass &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeBaseClass &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeBaseClass) } -void PDBSymDumper::dump(const PDBSymbolTypeBuiltin &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeBuiltin &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeBuiltin) } -void PDBSymDumper::dump(const PDBSymbolTypeCustom &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeCustom &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeCustom) } -void PDBSymDumper::dump(const PDBSymbolTypeDimension &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeDimension &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeDimension) } -void PDBSymDumper::dump(const PDBSymbolTypeEnum &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeEnum &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeEnum) } -void PDBSymDumper::dump(const PDBSymbolTypeFriend &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeFriend &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeFriend) } -void PDBSymDumper::dump(const PDBSymbolTypeFunctionArg &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeFunctionArg &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeFunctionArg) } -void PDBSymDumper::dump(const PDBSymbolTypeFunctionSig &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeFunctionSig &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeFunctionSig) } -void PDBSymDumper::dump(const PDBSymbolTypeManaged &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeManaged &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeManaged) } -void PDBSymDumper::dump(const PDBSymbolTypePointer &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypePointer &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypePointer) } -void PDBSymDumper::dump(const PDBSymbolTypeTypedef &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeTypedef &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeTypedef) } -void PDBSymDumper::dump(const PDBSymbolTypeUDT &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeUDT &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeUDT) } -void PDBSymDumper::dump(const PDBSymbolTypeVTable &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeVTable &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeVTable) } -void PDBSymDumper::dump(const PDBSymbolTypeVTableShape &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolTypeVTableShape &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolTypeVTableShape) } -void PDBSymDumper::dump(const PDBSymbolUnknown &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolUnknown &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolUnknown) } -void PDBSymDumper::dump(const PDBSymbolUsingNamespace &Symbol, raw_ostream &OS, - int Indent) { +void PDBSymDumper::dump(const PDBSymbolUsingNamespace &Symbol) { PDB_SYMDUMP_UNREACHABLE(PDBSymbolUsingNamespace) } diff --git a/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp b/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp index 4c76e3b..a782cad 100644 --- a/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp @@ -19,7 +19,6 @@ PDBSymbolAnnotation::PDBSymbolAnnotation(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolAnnotation::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolAnnotation::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolBlock.cpp b/lib/DebugInfo/PDB/PDBSymbolBlock.cpp index bb159d5..46b0ea5 100644 --- a/lib/DebugInfo/PDB/PDBSymbolBlock.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolBlock.cpp @@ -20,7 +20,4 @@ PDBSymbolBlock::PDBSymbolBlock(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolBlock::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolBlock::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp b/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp index 0c9b190..7436914 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp @@ -19,7 +19,6 @@ PDBSymbolCompiland::PDBSymbolCompiland(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolCompiland::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolCompiland::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp b/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp index 208d68f..7b351a0 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp @@ -20,7 +20,6 @@ PDBSymbolCompilandDetails::PDBSymbolCompilandDetails( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolCompilandDetails::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolCompilandDetails::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp b/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp index c54b8fb..e863ccf 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp @@ -26,7 +26,6 @@ std::string PDBSymbolCompilandEnv::getValue() const { return std::string(); } -void PDBSymbolCompilandEnv::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolCompilandEnv::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp index 1b6b50b..45faa0b 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp @@ -25,7 +25,4 @@ void PDBSymbolCustom::getDataBytes(llvm::SmallVector &bytes) { RawSymbol->getDataBytes(bytes); } -void PDBSymbolCustom::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} \ No newline at end of file +void PDBSymbolCustom::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } \ No newline at end of file diff --git a/lib/DebugInfo/PDB/PDBSymbolData.cpp b/lib/DebugInfo/PDB/PDBSymbolData.cpp index 6bf7e0f..60dcbc1 100644 --- a/lib/DebugInfo/PDB/PDBSymbolData.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolData.cpp @@ -24,7 +24,4 @@ std::unique_ptr PDBSymbolData::getType() const { return Session.getSymbolById(getTypeId()); } -void PDBSymbolData::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} \ No newline at end of file +void PDBSymbolData::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } \ No newline at end of file diff --git a/lib/DebugInfo/PDB/PDBSymbolExe.cpp b/lib/DebugInfo/PDB/PDBSymbolExe.cpp index ef09193..c9e34ea 100644 --- a/lib/DebugInfo/PDB/PDBSymbolExe.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolExe.cpp @@ -19,7 +19,4 @@ PDBSymbolExe::PDBSymbolExe(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolExe::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolExe::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp index e2d859f..b14af07 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp @@ -98,7 +98,4 @@ std::unique_ptr PDBSymbolFunc::getClassParent() const { return Session.getConcreteSymbolById(getClassParentId()); } -void PDBSymbolFunc::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolFunc::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp b/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp index c207488..8e559b3 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp @@ -20,7 +20,6 @@ PDBSymbolFuncDebugEnd::PDBSymbolFuncDebugEnd( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolFuncDebugEnd::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolFuncDebugEnd::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp b/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp index 83df22e..ff4254f 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp @@ -20,7 +20,6 @@ PDBSymbolFuncDebugStart::PDBSymbolFuncDebugStart( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolFuncDebugStart::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolFuncDebugStart::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolLabel.cpp b/lib/DebugInfo/PDB/PDBSymbolLabel.cpp index ce569e2..f39dee8 100644 --- a/lib/DebugInfo/PDB/PDBSymbolLabel.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolLabel.cpp @@ -19,7 +19,4 @@ PDBSymbolLabel::PDBSymbolLabel(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolLabel::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolLabel::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp b/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp index a7f156c..bd6fe89 100644 --- a/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp @@ -20,7 +20,6 @@ PDBSymbolPublicSymbol::PDBSymbolPublicSymbol( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolPublicSymbol::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolPublicSymbol::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolThunk.cpp b/lib/DebugInfo/PDB/PDBSymbolThunk.cpp index edade83..733eb5f 100644 --- a/lib/DebugInfo/PDB/PDBSymbolThunk.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolThunk.cpp @@ -19,7 +19,4 @@ PDBSymbolThunk::PDBSymbolThunk(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolThunk::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolThunk::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp index ffe6c80..1980965 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp @@ -24,7 +24,6 @@ std::unique_ptr PDBSymbolTypeArray::getElementType() const { return Session.getSymbolById(getTypeId()); } -void PDBSymbolTypeArray::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeArray::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp index c44cc52..c1f0d2f 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp @@ -20,7 +20,6 @@ PDBSymbolTypeBaseClass::PDBSymbolTypeBaseClass( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeBaseClass::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeBaseClass::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp index f0c94c7..b302b66 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp @@ -19,7 +19,6 @@ PDBSymbolTypeBuiltin::PDBSymbolTypeBuiltin( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeBuiltin::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeBuiltin::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp index 0fa8f45..cc391f1 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp @@ -20,7 +20,6 @@ PDBSymbolTypeCustom::PDBSymbolTypeCustom(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeCustom::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeCustom::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp index 47fb08d..1e19d0b 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp @@ -21,7 +21,6 @@ PDBSymbolTypeDimension::PDBSymbolTypeDimension( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeDimension::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeDimension::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp index 121d41e..8dd26a3 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp @@ -9,7 +9,10 @@ #include "llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h" +#include "llvm/DebugInfo/PDB/IPDBSession.h" #include "llvm/DebugInfo/PDB/PDBSymDumper.h" +#include "llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h" +#include "llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h" #include @@ -19,7 +22,13 @@ PDBSymbolTypeEnum::PDBSymbolTypeEnum(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeEnum::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +std::unique_ptr PDBSymbolTypeEnum::getClassParent() const { + return Session.getConcreteSymbolById(getClassParentId()); } + +std::unique_ptr +PDBSymbolTypeEnum::getUnderlyingType() const { + return Session.getConcreteSymbolById(getTypeId()); +} + +void PDBSymbolTypeEnum::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp index b2bf72e..d332660 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp @@ -20,7 +20,6 @@ PDBSymbolTypeFriend::PDBSymbolTypeFriend(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeFriend::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeFriend::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp index f394c04..f8f71ea 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp @@ -19,7 +19,6 @@ PDBSymbolTypeFunctionArg::PDBSymbolTypeFunctionArg( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeFunctionArg::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeFunctionArg::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp index 1ba397b..8018206 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp @@ -83,7 +83,6 @@ std::unique_ptr PDBSymbolTypeFunctionSig::getClassParent() const { return Session.getSymbolById(ClassId); } -void PDBSymbolTypeFunctionSig::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeFunctionSig::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp index e04fb66..a7fac30 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp @@ -20,7 +20,6 @@ PDBSymbolTypeManaged::PDBSymbolTypeManaged( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeManaged::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeManaged::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp b/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp index d274bf5..082ed83 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp @@ -24,7 +24,6 @@ std::unique_ptr PDBSymbolTypePointer::getPointeeType() const { return Session.getSymbolById(getTypeId()); } -void PDBSymbolTypePointer::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypePointer::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp index 12e3ead..5a42699 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp @@ -19,7 +19,6 @@ PDBSymbolTypeTypedef::PDBSymbolTypeTypedef( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeTypedef::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeTypedef::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp index 8a72368..2b5da29 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp @@ -19,7 +19,4 @@ PDBSymbolTypeUDT::PDBSymbolTypeUDT(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeUDT::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolTypeUDT::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp index a100526..b465d02 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp @@ -19,7 +19,6 @@ PDBSymbolTypeVTable::PDBSymbolTypeVTable(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeVTable::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeVTable::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp index 6aaa668..16052f1 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp @@ -20,7 +20,6 @@ PDBSymbolTypeVTableShape::PDBSymbolTypeVTableShape( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolTypeVTableShape::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolTypeVTableShape::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp b/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp index 9cfb88a..48dc115 100644 --- a/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp @@ -20,7 +20,4 @@ PDBSymbolUnknown::PDBSymbolUnknown(const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolUnknown::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); -} +void PDBSymbolUnknown::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); } diff --git a/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp b/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp index 9176dfb..6cf13de 100644 --- a/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp @@ -20,7 +20,6 @@ PDBSymbolUsingNamespace::PDBSymbolUsingNamespace( const IPDBSession &PDBSession, std::unique_ptr Symbol) : PDBSymbol(PDBSession, std::move(Symbol)) {} -void PDBSymbolUsingNamespace::dump(raw_ostream &OS, int Indent, - PDBSymDumper &Dumper) const { - Dumper.dump(*this, OS, Indent); +void PDBSymbolUsingNamespace::dump(PDBSymDumper &Dumper) const { + Dumper.dump(*this); } diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp index 12e0e6a..c586ba7 100644 --- a/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/lib/ExecutionEngine/ExecutionEngine.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Statistic.h" #include "llvm/ExecutionEngine/GenericValue.h" @@ -399,33 +400,12 @@ int ExecutionEngine::runFunctionAsMain(Function *Fn, return runFunction(Fn, GVArgs).IntVal.getZExtValue(); } -EngineBuilder::EngineBuilder() { - InitEngine(); -} +EngineBuilder::EngineBuilder() : EngineBuilder(nullptr) {} EngineBuilder::EngineBuilder(std::unique_ptr M) - : M(std::move(M)), MCJMM(nullptr) { - InitEngine(); -} - -EngineBuilder::~EngineBuilder() {} - -EngineBuilder &EngineBuilder::setMCJITMemoryManager( - std::unique_ptr mcjmm) { - MCJMM = std::move(mcjmm); - return *this; -} - -void EngineBuilder::InitEngine() { - WhichEngine = EngineKind::Either; - ErrorStr = nullptr; - OptLevel = CodeGenOpt::Default; - MCJMM = nullptr; - Options = TargetOptions(); - RelocModel = Reloc::Default; - CMModel = CodeModel::JITDefault; - UseOrcMCJITReplacement = false; - + : M(std::move(M)), WhichEngine(EngineKind::Either), ErrorStr(nullptr), + OptLevel(CodeGenOpt::Default), MCJMM(nullptr), RelocModel(Reloc::Default), + CMModel(CodeModel::JITDefault), UseOrcMCJITReplacement(false) { // IR module verification is enabled by default in debug builds, and disabled // by default in release builds. #ifndef NDEBUG @@ -435,6 +415,14 @@ void EngineBuilder::InitEngine() { #endif } +EngineBuilder::~EngineBuilder() = default; + +EngineBuilder &EngineBuilder::setMCJITMemoryManager( + std::unique_ptr mcjmm) { + MCJMM = std::move(mcjmm); + return *this; +} + ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { std::unique_ptr TheTM(TM); // Take ownership. diff --git a/lib/ExecutionEngine/Interpreter/CMakeLists.txt b/lib/ExecutionEngine/Interpreter/CMakeLists.txt index 1aac3ac..4dbc2df 100644 --- a/lib/ExecutionEngine/Interpreter/CMakeLists.txt +++ b/lib/ExecutionEngine/Interpreter/CMakeLists.txt @@ -13,7 +13,7 @@ add_llvm_library(LLVMInterpreter ) if( LLVM_ENABLE_FFI ) - target_link_libraries( LLVMInterpreter ${cmake_2_8_12_PRIVATE} ${FFI_LIBRARY_PATH} ) + target_link_libraries( LLVMInterpreter PRIVATE ${FFI_LIBRARY_PATH} ) endif() add_dependencies(LLVMInterpreter intrinsics_gen) diff --git a/lib/ExecutionEngine/Interpreter/Execution.cpp b/lib/ExecutionEngine/Interpreter/Execution.cpp index 93bb2d1..2e8eb16 100644 --- a/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -23,6 +23,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include #include using namespace llvm; @@ -464,14 +465,14 @@ static GenericValue executeFCMP_OGT(GenericValue Src1, GenericValue Src2, return Dest; \ } -#define IMPLEMENT_VECTOR_UNORDERED(TY, X,Y, _FUNC) \ - if (TY->isVectorTy()) { \ - GenericValue DestMask = Dest; \ - Dest = _FUNC(Src1, Src2, Ty); \ - for( size_t _i=0; _iisVectorTy()) { \ + GenericValue DestMask = Dest; \ + Dest = FUNC(Src1, Src2, Ty); \ + for (size_t _i = 0; _i < Src1.AggregateVal.size(); _i++) \ + if (DestMask.AggregateVal[_i].IntVal == true) \ + Dest.AggregateVal[_i].IntVal = APInt(1, true); \ + return Dest; \ } static GenericValue executeFCMP_UEQ(GenericValue Src1, GenericValue Src2, diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp index e500d3d..20b8553 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -137,8 +137,7 @@ std::unique_ptr MCJIT::emitObject(Module *M) { legacy::PassManager PM; - M->setDataLayout(TM->getDataLayout()); - PM.add(new DataLayoutPass()); + M->setDataLayout(*TM->getDataLayout()); // The RuntimeDyld will take ownership of this shortly SmallVector ObjBufferSV; @@ -258,7 +257,7 @@ uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) { Mangler Mang(TM->getDataLayout()); SmallString<128> FullName; Mang.getNameWithPrefix(FullName, Name); - return Dyld.getSymbolLoadAddress(FullName); + return Dyld.getSymbol(FullName).getAddress(); } Module *MCJIT::findModuleForSymbol(const std::string &Name, @@ -384,7 +383,7 @@ void *MCJIT::getPointerToFunction(Function *F) { // // This is the accessor for the target address, so make sure to check the // load address of the symbol, not the local address. - return (void*)Dyld.getSymbolLoadAddress(Name); + return (void*)Dyld.getSymbol(Name).getAddress(); } void MCJIT::runStaticConstructorsDestructorsInModulePtrSet( diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 61c947f..8cf490f 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/Orc/CloneSubModule.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 1b7b161..00e39bb 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -133,8 +133,8 @@ public: // If this module doesn't have a DataLayout attached then attach the // default. - if (!M->getDataLayout()) - M->setDataLayout(getDataLayout()); + if (M->getDataLayout().isDefault()) + M->setDataLayout(*getDataLayout()); Modules.push_back(std::move(M)); std::vector Ms; diff --git a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp index b5dda8e..6fe5301 100644 --- a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp @@ -39,7 +39,7 @@ template void restoreX86Regs(OStream &OS) { } template -uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM, +uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM, TargetAddress CallbackID) { return JCBM->executeCompileCallback(CallbackID); } @@ -52,7 +52,8 @@ namespace orc { const char* OrcX86_64::ResolverBlockName = "orc_resolver_block"; void OrcX86_64::insertResolverBlock( - Module &M, JITCompileCallbackManagerBase &JCBM) { + Module &M, JITCompileCallbackManagerBase &JCBM) { + const unsigned X86_64_TrampolineLength = 6; auto CallbackPtr = executeCompileCallback; uint64_t CallbackAddr = static_cast(reinterpret_cast(CallbackPtr)); @@ -77,6 +78,7 @@ void OrcX86_64::insertResolverBlock( AsmStream << " leaq jit_callback_manager_addr(%rip), %rdi\n" << " movq (%rdi), %rdi\n" << " movq " << ReturnAddrOffset << "(%rsp), %rsi\n" + << " subq $" << X86_64_TrampolineLength << ", %rsi\n" << " movabsq $" << CallbackAddr << ", %rax\n" << " callq *%rax\n" << " movq %rax, " << ReturnAddrOffset << "(%rsp)\n"; diff --git a/lib/ExecutionEngine/RuntimeDyld/Android.mk b/lib/ExecutionEngine/RuntimeDyld/Android.mk index 76aae67..40fdd7c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Android.mk +++ b/lib/ExecutionEngine/RuntimeDyld/Android.mk @@ -8,6 +8,7 @@ LOCAL_SRC_FILES := \ RTDyldMemoryManager.cpp \ RuntimeDyldChecker.cpp \ RuntimeDyld.cpp \ + RuntimeDyldCOFF.cpp \ RuntimeDyldELF.cpp \ RuntimeDyldMachO.cpp diff --git a/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt b/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt index 12bbcc6..e78408a 100644 --- a/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt +++ b/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt @@ -2,6 +2,7 @@ add_llvm_library(LLVMRuntimeDyld RTDyldMemoryManager.cpp RuntimeDyld.cpp RuntimeDyldChecker.cpp + RuntimeDyldCOFF.cpp RuntimeDyldELF.cpp RuntimeDyldMachO.cpp ) diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 54f1a1c..a0ed7cf 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -13,10 +13,12 @@ #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "RuntimeDyldCheckerImpl.h" +#include "RuntimeDyldCOFF.h" #include "RuntimeDyldELF.h" #include "RuntimeDyldImpl.h" #include "RuntimeDyldMachO.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/COFF.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/MutexGuard.h" @@ -195,10 +197,13 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset) << " flags: " << Flags << "\n"); - SymbolInfo::Visibility Vis = - (Flags & SymbolRef::SF_Exported) ? - SymbolInfo::Default : SymbolInfo::Hidden; - GlobalSymbolTable[Name] = SymbolInfo(SectionID, SectOffset, Vis); + JITSymbolFlags RTDyldSymFlags = JITSymbolFlags::None; + if (Flags & SymbolRef::SF_Weak) + RTDyldSymFlags |= JITSymbolFlags::Weak; + if (Flags & SymbolRef::SF_Exported) + RTDyldSymFlags |= JITSymbolFlags::Exported; + GlobalSymbolTable[Name] = + SymbolTableEntry(SectionID, SectOffset, RTDyldSymFlags); } } } @@ -264,6 +269,20 @@ static bool isRequiredForExecution(const SectionRef &Section) { const ObjectFile *Obj = Section.getObject(); if (auto *ELFObj = dyn_cast(Obj)) return ELFObj->getSectionFlags(Section) & ELF::SHF_ALLOC; + if (auto *COFFObj = dyn_cast(Obj)) { + const coff_section *CoffSection = COFFObj->getCOFFSection(Section); + // Avoid loading zero-sized COFF sections. + // In PE files, VirtualSize gives the section size, and SizeOfRawData + // may be zero for sections with content. In Obj files, SizeOfRawData + // gives the section size, and VirtualSize is always zero. Hence + // the need to check for both cases below. + bool HasContent = (CoffSection->VirtualSize > 0) + || (CoffSection->SizeOfRawData > 0); + bool IsDiscardable = CoffSection->Characteristics & + (COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_LNK_INFO); + return HasContent && !IsDiscardable; + } + assert(isa(Obj)); return true; } @@ -273,6 +292,15 @@ static bool isReadOnlyData(const SectionRef &Section) { if (auto *ELFObj = dyn_cast(Obj)) return !(ELFObj->getSectionFlags(Section) & (ELF::SHF_WRITE | ELF::SHF_EXECINSTR)); + if (auto *COFFObj = dyn_cast(Obj)) + return ((COFFObj->getCOFFSection(Section)->Characteristics & + (COFF::IMAGE_SCN_CNT_INITIALIZED_DATA + | COFF::IMAGE_SCN_MEM_READ + | COFF::IMAGE_SCN_MEM_WRITE)) + == + (COFF::IMAGE_SCN_CNT_INITIALIZED_DATA + | COFF::IMAGE_SCN_MEM_READ)); + assert(isa(Obj)); return false; } @@ -281,6 +309,9 @@ static bool isZeroInit(const SectionRef &Section) { const ObjectFile *Obj = Section.getObject(); if (auto *ELFObj = dyn_cast(Obj)) return ELFObj->getSectionType(Section) == ELF::SHT_NOBITS; + if (auto *COFFObj = dyn_cast(Obj)) + return COFFObj->getCOFFSection(Section)->Characteristics & + COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA; auto *MachO = cast(Obj); unsigned SectionType = MachO->getSectionType(Section); @@ -497,12 +528,15 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, Offset += AlignOffset; } uint32_t Flags = Sym.getFlags(); - SymbolInfo::Visibility Vis = - (Flags & SymbolRef::SF_Exported) ? - SymbolInfo::Default : SymbolInfo::Hidden; + JITSymbolFlags RTDyldSymFlags = JITSymbolFlags::None; + if (Flags & SymbolRef::SF_Weak) + RTDyldSymFlags |= JITSymbolFlags::Weak; + if (Flags & SymbolRef::SF_Exported) + RTDyldSymFlags |= JITSymbolFlags::Exported; DEBUG(dbgs() << "Allocating common symbol " << Name << " address " << format("%p", Addr) << "\n"); - GlobalSymbolTable[Name] = SymbolInfo(SectionID, Offset, Vis); + GlobalSymbolTable[Name] = + SymbolTableEntry(SectionID, Offset, RTDyldSymFlags); Offset += Size; Addr += Size; } @@ -512,7 +546,6 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, const SectionRef &Section, bool IsCode) { StringRef data; - Check(Section.getContents(data)); uint64_t Alignment64 = Section.getAlignment(); unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; @@ -542,6 +575,7 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, // Some sections, such as debug info, don't need to be loaded for execution. // Leave those where they are. if (IsRequired) { + Check(Section.getContents(data)); Allocate = DataSize + PaddingSize + StubBufSize; Addr = IsCode ? MemMgr->allocateCodeSection(Allocate, Alignment, SectionID, Name) @@ -816,6 +850,15 @@ RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { RuntimeDyld::~RuntimeDyld() {} +static std::unique_ptr +createRuntimeDyldCOFF(Triple::ArchType Arch, RTDyldMemoryManager *MM, + bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { + std::unique_ptr Dyld(RuntimeDyldCOFF::create(Arch, MM)); + Dyld->setProcessAllSections(ProcessAllSections); + Dyld->setRuntimeDyldChecker(Checker); + return Dyld; +} + static std::unique_ptr createRuntimeDyldELF(RTDyldMemoryManager *MM, bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { @@ -843,6 +886,10 @@ RuntimeDyld::loadObject(const ObjectFile &Obj) { Dyld = createRuntimeDyldMachO( static_cast(Obj.getArch()), MM, ProcessAllSections, Checker); + else if (Obj.isCOFF()) + Dyld = createRuntimeDyldCOFF( + static_cast(Obj.getArch()), MM, + ProcessAllSections, Checker); else report_fatal_error("Incompatible object format!"); } @@ -853,22 +900,16 @@ RuntimeDyld::loadObject(const ObjectFile &Obj) { return Dyld->loadObject(Obj); } -void *RuntimeDyld::getSymbolAddress(StringRef Name) const { +void *RuntimeDyld::getSymbolLocalAddress(StringRef Name) const { if (!Dyld) return nullptr; - return Dyld->getSymbolAddress(Name); + return Dyld->getSymbolLocalAddress(Name); } -uint64_t RuntimeDyld::getSymbolLoadAddress(StringRef Name) const { +RuntimeDyld::SymbolInfo RuntimeDyld::getSymbol(StringRef Name) const { if (!Dyld) - return 0; - return Dyld->getSymbolLoadAddress(Name); -} - -uint64_t RuntimeDyld::getExportedSymbolLoadAddress(StringRef Name) const { - if (!Dyld) - return 0; - return Dyld->getExportedSymbolLoadAddress(Name); + return nullptr; + return Dyld->getSymbol(Name); } void RuntimeDyld::resolveRelocations() { Dyld->resolveRelocations(); } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp new file mode 100644 index 0000000..56bcb8e --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -0,0 +1,85 @@ +//===-- RuntimeDyldCOFF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of COFF support for the MC-JIT runtime dynamic linker. +// +//===----------------------------------------------------------------------===// + +#include "RuntimeDyldCOFF.h" +#include "Targets/RuntimeDyldCOFFX86_64.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Triple.h" +#include "llvm/Object/ObjectFile.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "dyld" + +namespace { + +class LoadedCOFFObjectInfo : public RuntimeDyld::LoadedObjectInfo { +public: + LoadedCOFFObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, + unsigned EndIdx) + : RuntimeDyld::LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {} + + OwningBinary + getObjectForDebug(const ObjectFile &Obj) const override { + return OwningBinary(); + } +}; +} + +namespace llvm { + +std::unique_ptr +llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, RTDyldMemoryManager *MM) { + switch (Arch) { + default: + llvm_unreachable("Unsupported target for RuntimeDyldCOFF."); + break; + case Triple::x86_64: + return make_unique(MM); + } +} + +std::unique_ptr +RuntimeDyldCOFF::loadObject(const object::ObjectFile &O) { + unsigned SectionStartIdx, SectionEndIdx; + std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O); + return llvm::make_unique(*this, SectionStartIdx, + SectionEndIdx); +} + +uint64_t RuntimeDyldCOFF::getSymbolOffset(const SymbolRef &Sym) { + uint64_t Address; + if (Sym.getAddress(Address)) + return UnknownAddressOrSize; + + if (Address == UnknownAddressOrSize) + return UnknownAddressOrSize; + + const ObjectFile *Obj = Sym.getObject(); + section_iterator SecI(Obj->section_end()); + if (Sym.getSection(SecI)) + return UnknownAddressOrSize; + + if (SecI == Obj->section_end()) + return UnknownAddressOrSize; + + uint64_t SectionAddress = SecI->getAddress(); + return Address - SectionAddress; +} + +bool RuntimeDyldCOFF::isCompatibleFile(const object::ObjectFile &Obj) const { + return Obj.isCOFF(); +} + +} // namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h new file mode 100644 index 0000000..681a3e5 --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h @@ -0,0 +1,46 @@ +//===-- RuntimeDyldCOFF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// COFF support for MC-JIT runtime dynamic linker. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_RUNTIME_DYLD_COFF_H +#define LLVM_RUNTIME_DYLD_COFF_H + +#include "RuntimeDyldImpl.h" +#include "llvm/ADT/DenseMap.h" + +#define DEBUG_TYPE "dyld" + +using namespace llvm; + +namespace llvm { + +// Common base class for COFF dynamic linker support. +// Concrete subclasses for each target can be found in ./Targets. +class RuntimeDyldCOFF : public RuntimeDyldImpl { + +public: + std::unique_ptr + loadObject(const object::ObjectFile &Obj) override; + bool isCompatibleFile(const object::ObjectFile &Obj) const override; + static std::unique_ptr create(Triple::ArchType Arch, + RTDyldMemoryManager *MM); + +protected: + RuntimeDyldCOFF(RTDyldMemoryManager *MM) : RuntimeDyldImpl(MM) {} + uint64_t getSymbolOffset(const SymbolRef &Sym); +}; + +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 976a434..c991408 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -310,7 +310,7 @@ private: ""); uint64_t SymbolAddr = PCtx.IsInsideLoad - ? Checker.getSymbolLinkerAddr(Symbol) + ? Checker.getSymbolLocalAddr(Symbol) : Checker.getSymbolRemoteAddr(Symbol); uint64_t NextPC = SymbolAddr + InstSize; @@ -437,7 +437,7 @@ private: // The value for the symbol depends on the context we're evaluating in: // Inside a load this is the address in the linker's memory, outside a // load it's the address in the target processes memory. - uint64_t Value = PCtx.IsInsideLoad ? Checker.getSymbolLinkerAddr(Symbol) + uint64_t Value = PCtx.IsInsideLoad ? Checker.getSymbolLocalAddr(Symbol) : Checker.getSymbolRemoteAddr(Symbol); // Looks like a plain symbol reference. @@ -727,17 +727,17 @@ bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, } bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { - return getRTDyld().getSymbolAddress(Symbol) != nullptr; + return getRTDyld().getSymbolLocalAddress(Symbol) != nullptr; } -uint64_t RuntimeDyldCheckerImpl::getSymbolLinkerAddr(StringRef Symbol) const { +uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { return static_cast( - reinterpret_cast(getRTDyld().getSymbolAddress(Symbol))); + reinterpret_cast(getRTDyld().getSymbolLocalAddress(Symbol))); } uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { - if (uint64_t InternalSymbolAddr = getRTDyld().getSymbolLoadAddress(Symbol)) - return InternalSymbolAddr; + if (auto InternalSymbol = getRTDyld().getSymbol(Symbol)) + return InternalSymbol.getAddress(); return getRTDyld().MemMgr->getSymbolAddress(Symbol); } @@ -929,6 +929,6 @@ bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, std::pair RuntimeDyldChecker::getSectionAddr(StringRef FileName, StringRef SectionName, - bool LinkerAddress) { - return Impl->getSectionAddr(FileName, SectionName, LinkerAddress); + bool LocalAddress) { + return Impl->getSectionAddr(FileName, SectionName, LocalAddress); } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h index de20c1e..e8d299a 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -42,7 +42,7 @@ private: RuntimeDyldImpl &getRTDyld() const { return *RTDyld.Dyld; } bool isSymbolValid(StringRef Symbol) const; - uint64_t getSymbolLinkerAddr(StringRef Symbol) const; + uint64_t getSymbolLocalAddr(StringRef Symbol) const; uint64_t getSymbolRemoteAddr(StringRef Symbol) const; uint64_t readMemoryAtAddr(uint64_t Addr, unsigned Size) const; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 0f3ca0f..6278170 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -1128,7 +1128,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( RangeOverflow = true; } } - if (SymType == SymbolRef::ST_Unknown || RangeOverflow == true) { + if (SymType == SymbolRef::ST_Unknown || RangeOverflow) { // It is an external symbol (SymbolRef::ST_Unknown) or within a range // larger than 24-bits. StubMap::const_iterator i = Stubs.find(Value); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index b4414b0..71260d0 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -20,16 +20,6 @@ using namespace llvm; namespace llvm { -namespace { -// Helper for extensive error checking in debug builds. -std::error_code Check(std::error_code Err) { - if (Err) { - report_fatal_error(Err.message()); - } - return Err; -} - -} // end anonymous namespace class RuntimeDyldELF : public RuntimeDyldImpl { diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index f37a9a7..05060dd 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -36,6 +36,14 @@ using namespace llvm::object; namespace llvm { + // Helper for extensive error checking in debug builds. +inline std::error_code Check(std::error_code Err) { + if (Err) { + report_fatal_error(Err.message()); + } + return Err; +} + class Twine; /// SectionEntry - represents a section emitted into memory by the dynamic @@ -156,27 +164,24 @@ public: } }; -/// @brief Symbol info for RuntimeDyld. -class SymbolInfo { +/// @brief Symbol info for RuntimeDyld. +class SymbolTableEntry : public JITSymbolBase { public: - typedef enum { Hidden = 0, Default = 1 } Visibility; - - SymbolInfo() : Offset(0), SectionID(0), Vis(Hidden) {} + SymbolTableEntry() + : JITSymbolBase(JITSymbolFlags::None), Offset(0), SectionID(0) {} - SymbolInfo(unsigned SectionID, uint64_t Offset, Visibility Vis) - : Offset(Offset), SectionID(SectionID), Vis(Vis) {} + SymbolTableEntry(unsigned SectionID, uint64_t Offset, JITSymbolFlags Flags) + : JITSymbolBase(Flags), Offset(Offset), SectionID(SectionID) {} unsigned getSectionID() const { return SectionID; } uint64_t getOffset() const { return Offset; } - Visibility getVisibility() const { return Vis; } private: uint64_t Offset; - unsigned SectionID : 31; - Visibility Vis : 1; + unsigned SectionID; }; -typedef StringMap RTDyldSymbolTable; +typedef StringMap RTDyldSymbolTable; class RuntimeDyldImpl { friend class RuntimeDyld::LoadedObjectInfo; @@ -386,7 +391,7 @@ public: virtual std::unique_ptr loadObject(const object::ObjectFile &Obj) = 0; - uint8_t* getSymbolAddress(StringRef Name) const { + uint8_t* getSymbolLocalAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. RTDyldSymbolTable::const_iterator pos = GlobalSymbolTable.find(Name); @@ -396,24 +401,16 @@ public: return getSectionAddress(SymInfo.getSectionID()) + SymInfo.getOffset(); } - uint64_t getSymbolLoadAddress(StringRef Name) const { + RuntimeDyld::SymbolInfo getSymbol(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. RTDyldSymbolTable::const_iterator pos = GlobalSymbolTable.find(Name); if (pos == GlobalSymbolTable.end()) - return 0; - const auto &SymInfo = pos->second; - return getSectionLoadAddress(SymInfo.getSectionID()) + SymInfo.getOffset(); - } - - uint64_t getExportedSymbolLoadAddress(StringRef Name) const { - RTDyldSymbolTable::const_iterator pos = GlobalSymbolTable.find(Name); - if (pos == GlobalSymbolTable.end()) - return 0; - const auto &SymInfo = pos->second; - if (SymInfo.getVisibility() == SymbolInfo::Hidden) - return 0; - return getSectionLoadAddress(SymInfo.getSectionID()) + SymInfo.getOffset(); + return nullptr; + const auto &SymEntry = pos->second; + uint64_t TargetAddr = + getSectionLoadAddress(SymEntry.getSectionID()) + SymEntry.getOffset(); + return RuntimeDyld::SymbolInfo(TargetAddr, SymEntry.getFlags()); } void resolveRelocations(); diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h new file mode 100644 index 0000000..ce2f4a2 --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -0,0 +1,214 @@ +//===-- RuntimeDyldCOFFX86_64.h --- COFF/X86_64 specific code ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// COFF x86_x64 support for MC-JIT runtime dynamic linker. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFF86_64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFF86_64_H + +#include "llvm/Object/COFF.h" +#include "llvm/Support/COFF.h" +#include "../RuntimeDyldCOFF.h" + +#define DEBUG_TYPE "dyld" + +namespace llvm { + +class RuntimeDyldCOFFX86_64 : public RuntimeDyldCOFF { + +private: + // When a module is loaded we save the SectionID of the unwind + // sections in a table until we receive a request to register all + // unregisteredEH frame sections with the memory manager. + SmallVector UnregisteredEHFrameSections; + SmallVector RegisteredEHFrameSections; + +public: + RuntimeDyldCOFFX86_64(RTDyldMemoryManager *MM) : RuntimeDyldCOFF(MM) {} + + unsigned getMaxStubSize() override { + return 6; // 2-byte jmp instruction + 32-bit relative address + } + + // The target location for the relocation is described by RE.SectionID and + // RE.Offset. RE.SectionID can be used to find the SectionEntry. Each + // SectionEntry has three members describing its location. + // SectionEntry::Address is the address at which the section has been loaded + // into memory in the current (host) process. SectionEntry::LoadAddress is + // the address that the section will have in the target process. + // SectionEntry::ObjAddress is the address of the bits for this section in the + // original emitted object image (also in the current address space). + // + // Relocations will be applied as if the section were loaded at + // SectionEntry::LoadAddress, but they will be applied at an address based + // on SectionEntry::Address. SectionEntry::ObjAddress will be used to refer + // to Target memory contents if they are required for value calculations. + // + // The Value parameter here is the load address of the symbol for the + // relocation to be applied. For relocations which refer to symbols in the + // current object Value will be the LoadAddress of the section in which + // the symbol resides (RE.Addend provides additional information about the + // symbol location). For external symbols, Value will be the address of the + // symbol in the target address space. + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *Target = Section.Address + RE.Offset; + + switch (RE.RelType) { + + case COFF::IMAGE_REL_AMD64_REL32: + case COFF::IMAGE_REL_AMD64_REL32_1: + case COFF::IMAGE_REL_AMD64_REL32_2: + case COFF::IMAGE_REL_AMD64_REL32_3: + case COFF::IMAGE_REL_AMD64_REL32_4: + case COFF::IMAGE_REL_AMD64_REL32_5: { + uint32_t *TargetAddress = (uint32_t *)Target; + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + // Delta is the distance from the start of the reloc to the end of the + // instruction with the reloc. + uint64_t Delta = 4 + (RE.RelType - COFF::IMAGE_REL_AMD64_REL32); + Value -= FinalAddress + Delta; + uint64_t Result = Value + RE.Addend; + assert(((int64_t)Result <= INT32_MAX) && "Relocation overflow"); + assert(((int64_t)Result >= INT32_MIN) && "Relocation underflow"); + *TargetAddress = Result; + break; + } + + case COFF::IMAGE_REL_AMD64_ADDR32NB: { + // Note ADDR32NB requires a well-established notion of + // image base. This address must be less than or equal + // to every section's load address, and all sections must be + // within a 32 bit offset from the base. + // + // For now we just set these to zero. + uint32_t *TargetAddress = (uint32_t *)Target; + *TargetAddress = 0; + break; + } + + case COFF::IMAGE_REL_AMD64_ADDR64: { + uint64_t *TargetAddress = (uint64_t *)Target; + *TargetAddress = Value + RE.Addend; + break; + } + + default: + llvm_unreachable("Relocation type not implemented yet!"); + break; + } + } + + relocation_iterator processRelocationRef(unsigned SectionID, + relocation_iterator RelI, + const ObjectFile &Obj, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { + // Find the symbol referred to in the relocation, and + // get its section and offset. + // + // Insist for now that all symbols be resolvable within + // the scope of this object file. + symbol_iterator Symbol = RelI->getSymbol(); + if (Symbol == Obj.symbol_end()) + report_fatal_error("Unknown symbol in relocation"); + unsigned TargetSectionID = 0; + uint64_t TargetOffset = UnknownAddressOrSize; + section_iterator SecI(Obj.section_end()); + Symbol->getSection(SecI); + if (SecI == Obj.section_end()) + report_fatal_error("Unknown section in relocation"); + bool IsCode = SecI->isText(); + TargetSectionID = findOrEmitSection(Obj, *SecI, IsCode, ObjSectionToID); + TargetOffset = getSymbolOffset(*Symbol); + + // Determine the Addend used to adjust the relocation value. + uint64_t RelType; + Check(RelI->getType(RelType)); + uint64_t Offset; + Check(RelI->getOffset(Offset)); + uint64_t Addend = 0; + SectionEntry &Section = Sections[SectionID]; + uintptr_t ObjTarget = Section.ObjAddress + Offset; + + switch (RelType) { + + case COFF::IMAGE_REL_AMD64_REL32: + case COFF::IMAGE_REL_AMD64_REL32_1: + case COFF::IMAGE_REL_AMD64_REL32_2: + case COFF::IMAGE_REL_AMD64_REL32_3: + case COFF::IMAGE_REL_AMD64_REL32_4: + case COFF::IMAGE_REL_AMD64_REL32_5: + case COFF::IMAGE_REL_AMD64_ADDR32NB: { + uint32_t *Displacement = (uint32_t *)ObjTarget; + Addend = *Displacement; + break; + } + + case COFF::IMAGE_REL_AMD64_ADDR64: { + uint64_t *Displacement = (uint64_t *)ObjTarget; + Addend = *Displacement; + break; + } + + default: + break; + } + + StringRef TargetName; + Symbol->getName(TargetName); + DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelType << " TargetName: " << TargetName + << " Addend " << Addend << "\n"); + + RelocationEntry RE(SectionID, Offset, RelType, TargetOffset + Addend); + addRelocationForSection(RE, TargetSectionID); + + return ++RelI; + } + + unsigned getStubAlignment() override { return 1; } + void registerEHFrames() override { + if (!MemMgr) + return; + for (auto const &EHFrameSID : UnregisteredEHFrameSections) { + uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; + uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; + size_t EHFrameSize = Sections[EHFrameSID].Size; + MemMgr->registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); + RegisteredEHFrameSections.push_back(EHFrameSID); + } + UnregisteredEHFrameSections.clear(); + } + void deregisterEHFrames() override { + // Stub + } + void finalizeLoad(const ObjectFile &Obj, + ObjSectionToIDMap &SectionMap) override { + // Look for and record the EH frame section IDs. + for (const auto &SectionPair : SectionMap) { + const SectionRef &Section = SectionPair.first; + StringRef Name; + Check(Section.getName(Name)); + // Note unwind info is split across .pdata and .xdata, so this + // may not be sufficiently general for all users. + if (Name == ".xdata") { + UnregisteredEHFrameSections.push_back(SectionPair.second); + } + } + } +}; + +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp index 1746afd..9ccd744 100644 --- a/lib/Fuzzer/FuzzerDriver.cpp +++ b/lib/Fuzzer/FuzzerDriver.cpp @@ -158,6 +158,7 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) { Options.DoCrossOver = Flags.cross_over; Options.MutateDepth = Flags.mutate_depth; Options.ExitOnFirst = Flags.exit_on_first; + Options.UseCounters = Flags.use_counters; Options.UseFullCoverageSet = Flags.use_full_coverage_set; Options.UseCoveragePairs = Flags.use_coverage_pairs; Options.PreferSmallDuringInitialShuffle = diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def index 068f245..08176af 100644 --- a/lib/Fuzzer/FuzzerFlags.def +++ b/lib/Fuzzer/FuzzerFlags.def @@ -32,6 +32,7 @@ FUZZER_FLAG(int, help, 0, "Print help.") FUZZER_FLAG( int, save_minimized_corpus, 0, "If 1, the minimized corpus is saved into the first input directory") +FUZZER_FLAG(int, use_counters, 0, "Use coverage counters") FUZZER_FLAG(int, use_full_coverage_set, 0, "Experimental: Maximize the number of different full" " coverage sets as opposed to maximizing the total coverage." diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h index 980b00e..e4e5eb7 100644 --- a/lib/Fuzzer/FuzzerInternal.h +++ b/lib/Fuzzer/FuzzerInternal.h @@ -48,6 +48,7 @@ class Fuzzer { bool DoCrossOver = true; int MutateDepth = 5; bool ExitOnFirst = false; + bool UseCounters = false; bool UseFullCoverageSet = false; bool UseCoveragePairs = false; int PreferSmallDuringInitialShuffle = -1; @@ -95,6 +96,15 @@ class Fuzzer { std::vector Corpus; std::unordered_set FullCoverageSets; std::unordered_set CoveragePairs; + + // For UseCounters + std::vector CounterBitmap; + size_t TotalBits() { // Slow. Call it only for printing stats. + size_t Res = 0; + for (auto x : CounterBitmap) Res += __builtin_popcount(x); + return Res; + } + UserCallback Callback; FuzzingOptions Options; system_clock::time_point ProcessStartTime = system_clock::now(); diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index 70b63eb..563fbf4 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -138,17 +138,28 @@ size_t Fuzzer::RunOneMaximizeFullCoverageSet(const Unit &U) { } size_t Fuzzer::RunOneMaximizeTotalCoverage(const Unit &U) { + size_t NumCounters = __sanitizer_get_number_of_counters(); + if (Options.UseCounters) { + CounterBitmap.resize(NumCounters); + __sanitizer_update_counter_bitset_and_clear_counters(0); + } size_t OldCoverage = __sanitizer_get_total_unique_coverage(); Callback(U.data(), U.size()); size_t NewCoverage = __sanitizer_get_total_unique_coverage(); + size_t NumNewBits = 0; + if (Options.UseCounters) + NumNewBits = __sanitizer_update_counter_bitset_and_clear_counters( + CounterBitmap.data()); + if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && Options.Verbosity) { size_t Seconds = secondsSinceProcessStartUp(); std::cerr << "#" << TotalNumberOfRuns << "\tcov: " << NewCoverage + << "\tbits: " << TotalBits() << "\texec/s: " << (Seconds ? TotalNumberOfRuns / Seconds : 0) << "\n"; } - if (NewCoverage > OldCoverage) + if (NewCoverage > OldCoverage || NumNewBits) return NewCoverage; return 0; } @@ -189,6 +200,7 @@ size_t Fuzzer::MutateAndTestOne(Unit *U) { if (Options.Verbosity) { std::cerr << "#" << TotalNumberOfRuns << "\tNEW: " << NewCoverage + << " B: " << TotalBits() << " L: " << U->size() << " S: " << Corpus.size() << " I: " << i diff --git a/lib/Fuzzer/test/CMakeLists.txt b/lib/Fuzzer/test/CMakeLists.txt index bed9cd8..08130c6 100644 --- a/lib/Fuzzer/test/CMakeLists.txt +++ b/lib/Fuzzer/test/CMakeLists.txt @@ -5,6 +5,7 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O0 -fsanitize-coverage=4") set(Tests + CounterTest FourIndependentBranchesTest FullCoverageSetTest InfiniteTest diff --git a/lib/Fuzzer/test/CounterTest.cpp b/lib/Fuzzer/test/CounterTest.cpp new file mode 100644 index 0000000..332ccfe --- /dev/null +++ b/lib/Fuzzer/test/CounterTest.cpp @@ -0,0 +1,14 @@ +// Test for a fuzzer: must find the case where a particular basic block is +// executed many times. +#include + +extern "C" void TestOneInput(const uint8_t *Data, size_t Size) { + int Num = 0; + for (size_t i = 0; i < Size; i++) + if (Data[i] == 'A' + i) + Num++; + if (Num >= 4) { + std::cerr << "BINGO!\n"; + exit(1); + } +} diff --git a/lib/Fuzzer/test/fuzzer.test b/lib/Fuzzer/test/fuzzer.test index 1e42e72..45691f5 100644 --- a/lib/Fuzzer/test/fuzzer.test +++ b/lib/Fuzzer/test/fuzzer.test @@ -17,3 +17,6 @@ FullCoverageSetTest: BINGO RUN: not ./LLVMFuzzer-FourIndependentBranchesTest -timeout=15 -seed=1 -use_coverage_pairs=1 2>&1 | FileCheck %s --check-prefix=FourIndependentBranchesTest FourIndependentBranchesTest: BINGO + +RUN: not ./LLVMFuzzer-CounterTest -use_counters=1 -max_len=6 -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=CounterTest +CounterTest: BINGO diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index de0e614..ae0beba 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -14,9 +14,9 @@ // //===----------------------------------------------------------------------===// -#include "AsmWriter.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/AssemblyAnnotationWriter.h" @@ -32,12 +32,14 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/TypeFinder.h" +#include "llvm/IR/UseListOrder.h" #include "llvm/IR/ValueSymbolTable.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include #include using namespace llvm; @@ -275,6 +277,15 @@ static const Module *getModuleFromVal(const Value *V) { if (const GlobalValue *GV = dyn_cast(V)) return GV->getParent(); + + if (const auto *MAV = dyn_cast(V)) { + for (const User *U : MAV->users()) + if (isa(U)) + if (const Module *M = getModuleFromVal(U)) + return M; + return nullptr; + } + return nullptr; } @@ -378,7 +389,29 @@ static void PrintLLVMName(raw_ostream &OS, const Value *V) { } -namespace llvm { +namespace { +class TypePrinting { + TypePrinting(const TypePrinting &) = delete; + void operator=(const TypePrinting&) = delete; +public: + + /// NamedTypes - The named types that are used by the current module. + TypeFinder NamedTypes; + + /// NumberedTypes - The numbered types, along with their value. + DenseMap NumberedTypes; + + + TypePrinting() {} + ~TypePrinting() {} + + void incorporateTypes(const Module &M); + + void print(Type *Ty, raw_ostream &OS); + + void printStructBody(StructType *Ty, raw_ostream &OS); +}; +} // namespace void TypePrinting::incorporateTypes(const Module &M) { NamedTypes.run(M, false); @@ -508,6 +541,7 @@ void TypePrinting::printStructBody(StructType *STy, raw_ostream &OS) { OS << '>'; } +namespace { //===----------------------------------------------------------------------===// // SlotTracker Class: Enumerate slot numbers for unnamed values //===----------------------------------------------------------------------===// @@ -525,6 +559,7 @@ private: /// TheFunction - The function for which we are holding slot numbers. const Function* TheFunction; bool FunctionProcessed; + bool ShouldInitializeAllMetadata; /// mMap - The slot map for the module level data. ValueMap mMap; @@ -542,10 +577,20 @@ private: DenseMap asMap; unsigned asNext; public: - /// Construct from a module - explicit SlotTracker(const Module *M); + /// Construct from a module. + /// + /// If \c ShouldInitializeAllMetadata, initializes all metadata in all + /// functions, giving correct numbering for metadata referenced only from + /// within a function (even if no functions have been initialized). + explicit SlotTracker(const Module *M, + bool ShouldInitializeAllMetadata = false); /// Construct from a function, starting out in incorp state. - explicit SlotTracker(const Function *F); + /// + /// If \c ShouldInitializeAllMetadata, initializes all metadata in all + /// functions, giving correct numbering for metadata referenced only from + /// within a function (even if no functions have been initialized). + explicit SlotTracker(const Function *F, + bool ShouldInitializeAllMetadata = false); /// Return the slot number of the specified value in it's type /// plane. If something is not in the SlotTracker, return -1. @@ -606,11 +651,18 @@ private: /// Add all of the functions arguments, basic blocks, and instructions. void processFunction(); + /// Add all of the metadata from a function. + void processFunctionMetadata(const Function &F); + + /// Add all of the metadata from an instruction. + void processInstructionMetadata(const Instruction &I); + SlotTracker(const SlotTracker &) = delete; void operator=(const SlotTracker &) = delete; }; +} // namespace -SlotTracker *createSlotTracker(const Module *M) { +static SlotTracker *createSlotTracker(const Module *M) { return new SlotTracker(M); } @@ -645,15 +697,18 @@ static SlotTracker *createSlotTracker(const Value *V) { // Module level constructor. Causes the contents of the Module (sans functions) // to be added to the slot table. -SlotTracker::SlotTracker(const Module *M) - : TheModule(M), TheFunction(nullptr), FunctionProcessed(false), mNext(0), +SlotTracker::SlotTracker(const Module *M, bool ShouldInitializeAllMetadata) + : TheModule(M), TheFunction(nullptr), FunctionProcessed(false), + ShouldInitializeAllMetadata(ShouldInitializeAllMetadata), mNext(0), fNext(0), mdnNext(0), asNext(0) {} // Function level constructor. Causes the contents of the Module and the one // function provided to be added to the slot table. -SlotTracker::SlotTracker(const Function *F) +SlotTracker::SlotTracker(const Function *F, bool ShouldInitializeAllMetadata) : TheModule(F ? F->getParent() : nullptr), TheFunction(F), - FunctionProcessed(false), mNext(0), fNext(0), mdnNext(0), asNext(0) {} + FunctionProcessed(false), + ShouldInitializeAllMetadata(ShouldInitializeAllMetadata), mNext(0), + fNext(0), mdnNext(0), asNext(0) {} inline void SlotTracker::initialize() { if (TheModule) { @@ -692,6 +747,9 @@ void SlotTracker::processModule() { // Add all the unnamed functions to the table. CreateModuleSlot(I); + if (ShouldInitializeAllMetadata) + processFunctionMetadata(*I); + // Add all the function attributes to the table. // FIXME: Add attributes of other objects? AttributeSet FnAttrs = I->getAttributes().getFnAttributes(); @@ -715,46 +773,30 @@ void SlotTracker::processFunction() { ST_DEBUG("Inserting Instructions:\n"); - SmallVector, 4> MDForInst; - // Add all of the basic blocks and instructions with no names. - for (Function::const_iterator BB = TheFunction->begin(), - E = TheFunction->end(); BB != E; ++BB) { - if (!BB->hasName()) - CreateFunctionSlot(BB); - - for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; - ++I) { - if (!I->getType()->isVoidTy() && !I->hasName()) - CreateFunctionSlot(I); - - // Intrinsics can directly use metadata. We allow direct calls to any - // llvm.foo function here, because the target may not be linked into the - // optimizer. - if (const CallInst *CI = dyn_cast(I)) { - if (Function *F = CI->getCalledFunction()) - if (F->isIntrinsic()) - for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) - if (auto *V = dyn_cast_or_null(I->getOperand(i))) - if (MDNode *N = dyn_cast(V->getMetadata())) - CreateMetadataSlot(N); + for (auto &BB : *TheFunction) { + if (!BB.hasName()) + CreateFunctionSlot(&BB); + + for (auto &I : BB) { + if (!I.getType()->isVoidTy() && !I.hasName()) + CreateFunctionSlot(&I); + + processInstructionMetadata(I); + // We allow direct calls to any llvm.foo function here, because the + // target may not be linked into the optimizer. + if (const CallInst *CI = dyn_cast(&I)) { // Add all the call attributes to the table. AttributeSet Attrs = CI->getAttributes().getFnAttributes(); if (Attrs.hasAttributes(AttributeSet::FunctionIndex)) CreateAttributeSetSlot(Attrs); - } else if (const InvokeInst *II = dyn_cast(I)) { + } else if (const InvokeInst *II = dyn_cast(&I)) { // Add all the call attributes to the table. AttributeSet Attrs = II->getAttributes().getFnAttributes(); if (Attrs.hasAttributes(AttributeSet::FunctionIndex)) CreateAttributeSetSlot(Attrs); } - - // Process metadata attached with this instruction. - I->getAllMetadata(MDForInst); - for (unsigned i = 0, e = MDForInst.size(); i != e; ++i) - CreateMetadataSlot(MDForInst[i].second); - MDForInst.clear(); } } @@ -763,6 +805,29 @@ void SlotTracker::processFunction() { ST_DEBUG("end processFunction!\n"); } +void SlotTracker::processFunctionMetadata(const Function &F) { + for (auto &BB : F) + for (auto &I : BB) + processInstructionMetadata(I); +} + +void SlotTracker::processInstructionMetadata(const Instruction &I) { + // Process metadata used directly by intrinsics. + if (const CallInst *CI = dyn_cast(&I)) + if (Function *F = CI->getCalledFunction()) + if (F->isIntrinsic()) + for (auto &Op : I.operands()) + if (auto *V = dyn_cast_or_null(Op)) + if (MDNode *N = dyn_cast(V->getMetadata())) + CreateMetadataSlot(N); + + // Process metadata attached to this instruction. + SmallVector, 4> MDs; + I.getAllMetadata(MDs); + for (auto &MD : MDs) + CreateMetadataSlot(MD.second); +} + /// Clean up after incorporating a function. This is the only way to get out of /// the function incorporation state that affects get*Slot/Create*Slot. Function /// incorporation state is indicated by TheFunction != 0. @@ -1010,7 +1075,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, (StrVal[1] >= '0' && StrVal[1] <= '9'))) { // Reparse stringized version! if (APFloat(APFloat::IEEEdouble, StrVal).convertToDouble() == Val) { - Out << StrVal.str(); + Out << StrVal; return; } } @@ -1223,6 +1288,14 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, Out << ' ' << getPredicateText(CE->getPredicate()); Out << " ("; + if (const GEPOperator *GEP = dyn_cast(CE)) { + TypePrinter.print( + cast(GEP->getPointerOperandType()->getScalarType()) + ->getElementType(), + Out); + Out << ", "; + } + for (User::const_op_iterator OI=CE->op_begin(); OI != CE->op_end(); ++OI) { TypePrinter.print((*OI)->getType(), Out); Out << ' '; @@ -1285,8 +1358,52 @@ raw_ostream &operator<<(raw_ostream &OS, FieldSeparator &FS) { } return OS << FS.Sep; } +struct MDFieldPrinter { + raw_ostream &Out; + FieldSeparator FS; + TypePrinting *TypePrinter; + SlotTracker *Machine; + const Module *Context; + + explicit MDFieldPrinter(raw_ostream &Out) + : Out(Out), TypePrinter(nullptr), Machine(nullptr), Context(nullptr) {} + MDFieldPrinter(raw_ostream &Out, TypePrinting *TypePrinter, + SlotTracker *Machine, const Module *Context) + : Out(Out), TypePrinter(TypePrinter), Machine(Machine), Context(Context) { + } + void printTag(const DebugNode *N); + void printString(StringRef Name, StringRef Value, + bool ShouldSkipEmpty = true); + void printMetadata(StringRef Name, const Metadata *MD, + bool ShouldSkipNull = true); + template + void printInt(StringRef Name, IntTy Int, bool ShouldSkipZero = true); + void printBool(StringRef Name, bool Value); + void printDIFlags(StringRef Name, unsigned Flags); + template + void printDwarfEnum(StringRef Name, IntTy Value, Stringifier toString, + bool ShouldSkipZero = true); +}; } // end namespace +void MDFieldPrinter::printTag(const DebugNode *N) { + Out << FS << "tag: "; + if (const char *Tag = dwarf::TagString(N->getTag())) + Out << Tag; + else + Out << N->getTag(); +} + +void MDFieldPrinter::printString(StringRef Name, StringRef Value, + bool ShouldSkipEmpty) { + if (ShouldSkipEmpty && Value.empty()) + return; + + Out << FS << Name << ": \""; + PrintEscapedString(Value, Out); + Out << "\""; +} + static void writeMetadataAsOperand(raw_ostream &Out, const Metadata *MD, TypePrinting *TypePrinter, SlotTracker *Machine, @@ -1298,27 +1415,68 @@ static void writeMetadataAsOperand(raw_ostream &Out, const Metadata *MD, WriteAsOperandInternal(Out, MD, TypePrinter, Machine, Context); } -static void writeTag(raw_ostream &Out, FieldSeparator &FS, const DebugNode *N) { - Out << FS << "tag: "; - if (const char *Tag = dwarf::TagString(N->getTag())) - Out << Tag; +void MDFieldPrinter::printMetadata(StringRef Name, const Metadata *MD, + bool ShouldSkipNull) { + if (ShouldSkipNull && !MD) + return; + + Out << FS << Name << ": "; + writeMetadataAsOperand(Out, MD, TypePrinter, Machine, Context); +} + +template +void MDFieldPrinter::printInt(StringRef Name, IntTy Int, bool ShouldSkipZero) { + if (ShouldSkipZero && !Int) + return; + + Out << FS << Name << ": " << Int; +} + +void MDFieldPrinter::printBool(StringRef Name, bool Value) { + Out << FS << Name << ": " << (Value ? "true" : "false"); +} + +void MDFieldPrinter::printDIFlags(StringRef Name, unsigned Flags) { + if (!Flags) + return; + + Out << FS << Name << ": "; + + SmallVector SplitFlags; + unsigned Extra = DIDescriptor::splitFlags(Flags, SplitFlags); + + FieldSeparator FlagsFS(" | "); + for (unsigned F : SplitFlags) { + const char *StringF = DIDescriptor::getFlagString(F); + assert(StringF && "Expected valid flag"); + Out << FlagsFS << StringF; + } + if (Extra || SplitFlags.empty()) + Out << FlagsFS << Extra; +} + +template +void MDFieldPrinter::printDwarfEnum(StringRef Name, IntTy Value, + Stringifier toString, bool ShouldSkipZero) { + if (!Value) + return; + + Out << FS << Name << ": "; + if (const char *S = toString(Value)) + Out << S; else - Out << N->getTag(); + Out << Value; } static void writeGenericDebugNode(raw_ostream &Out, const GenericDebugNode *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!GenericDebugNode("; - FieldSeparator FS; - writeTag(Out, FS, N); - if (!N->getHeader().empty()) { - Out << FS << "header: \""; - PrintEscapedString(N->getHeader(), Out); - Out << "\""; - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printTag(N); + Printer.printString("header", N->getHeader()); if (N->getNumDwarfOperands()) { - Out << FS << "operands: {"; + Out << Printer.FS << "operands: {"; FieldSeparator IFS; for (auto &I : N->dwarf_operands()) { Out << IFS; @@ -1333,111 +1491,64 @@ static void writeMDLocation(raw_ostream &Out, const MDLocation *DL, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDLocation("; - FieldSeparator FS; + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); // Always output the line, since 0 is a relevant and important value for it. - Out << FS << "line: " << DL->getLine(); - if (DL->getColumn()) - Out << FS << "column: " << DL->getColumn(); - Out << FS << "scope: "; - WriteAsOperandInternal(Out, DL->getScope(), TypePrinter, Machine, Context); - if (DL->getInlinedAt()) { - Out << FS << "inlinedAt: "; - WriteAsOperandInternal(Out, DL->getInlinedAt(), TypePrinter, Machine, - Context); - } + Printer.printInt("line", DL->getLine(), /* ShouldSkipZero */ false); + Printer.printInt("column", DL->getColumn()); + Printer.printMetadata("scope", DL->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("inlinedAt", DL->getRawInlinedAt()); Out << ")"; } static void writeMDSubrange(raw_ostream &Out, const MDSubrange *N, TypePrinting *, SlotTracker *, const Module *) { Out << "!MDSubrange("; - FieldSeparator FS; - Out << FS << "count: " << N->getCount(); - if (N->getLo()) - Out << FS << "lowerBound: " << N->getLo(); + MDFieldPrinter Printer(Out); + Printer.printInt("count", N->getCount(), /* ShouldSkipZero */ false); + Printer.printInt("lowerBound", N->getLo()); Out << ")"; } static void writeMDEnumerator(raw_ostream &Out, const MDEnumerator *N, TypePrinting *, SlotTracker *, const Module *) { Out << "!MDEnumerator("; - FieldSeparator FS; - Out << FS << "name: \"" << N->getName() << "\""; - Out << FS << "value: " << N->getValue(); + MDFieldPrinter Printer(Out); + Printer.printString("name", N->getName(), /* ShouldSkipEmpty */ false); + Printer.printInt("value", N->getValue(), /* ShouldSkipZero */ false); Out << ")"; } static void writeMDBasicType(raw_ostream &Out, const MDBasicType *N, TypePrinting *, SlotTracker *, const Module *) { Out << "!MDBasicType("; - FieldSeparator FS; - writeTag(Out, FS, N); - if (!N->getName().empty()) - Out << FS << "name: \"" << N->getName() << "\""; - if (N->getSizeInBits()) - Out << FS << "size: " << N->getSizeInBits(); - if (N->getAlignInBits()) - Out << FS << "align: " << N->getAlignInBits(); - if (unsigned Encoding = N->getEncoding()) { - Out << FS << "encoding: "; - if (const char *S = dwarf::AttributeEncodingString(Encoding)) - Out << S; - else - Out << Encoding; - } + MDFieldPrinter Printer(Out); + if (N->getTag() != dwarf::DW_TAG_base_type) + Printer.printTag(N); + Printer.printString("name", N->getName()); + Printer.printInt("size", N->getSizeInBits()); + Printer.printInt("align", N->getAlignInBits()); + Printer.printDwarfEnum("encoding", N->getEncoding(), + dwarf::AttributeEncodingString); Out << ")"; } -static void writeDIFlags(raw_ostream &Out, unsigned Flags) { - SmallVector SplitFlags; - unsigned Extra = DIDescriptor::splitFlags(Flags, SplitFlags); - - FieldSeparator FS(" | "); - for (unsigned F : SplitFlags) { - const char *StringF = DIDescriptor::getFlagString(F); - assert(StringF && "Expected valid flag"); - Out << FS << StringF; - } - if (Extra || SplitFlags.empty()) - Out << FS << Extra; -} - static void writeMDDerivedType(raw_ostream &Out, const MDDerivedType *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDDerivedType("; - FieldSeparator FS; - writeTag(Out, FS, N); - if (!N->getName().empty()) - Out << FS << "name: \"" << N->getName() << "\""; - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (N->getScope()) { - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - } - Out << FS << "baseType: "; - writeMetadataAsOperand(Out, N->getBaseType(), TypePrinter, Machine, Context); - if (N->getSizeInBits()) - Out << FS << "size: " << N->getSizeInBits(); - if (N->getAlignInBits()) - Out << FS << "align: " << N->getAlignInBits(); - if (N->getOffsetInBits()) - Out << FS << "offset: " << N->getOffsetInBits(); - if (auto Flags = N->getFlags()) { - Out << FS << "flags: "; - writeDIFlags(Out, Flags); - } - if (N->getExtraData()) { - Out << FS << "extraData: "; - writeMetadataAsOperand(Out, N->getExtraData(), TypePrinter, Machine, - Context); - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printTag(N); + Printer.printString("name", N->getName()); + Printer.printMetadata("scope", N->getScope()); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printMetadata("baseType", N->getBaseType(), + /* ShouldSkipNull */ false); + Printer.printInt("size", N->getSizeInBits()); + Printer.printInt("align", N->getAlignInBits()); + Printer.printInt("offset", N->getOffsetInBits()); + Printer.printDIFlags("flags", N->getFlags()); + Printer.printMetadata("extraData", N->getExtraData()); Out << ")"; } @@ -1445,61 +1556,23 @@ static void writeMDCompositeType(raw_ostream &Out, const MDCompositeType *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDCompositeType("; - FieldSeparator FS; - writeTag(Out, FS, N); - if (!N->getName().empty()) - Out << FS << "name: \"" << N->getName() << "\""; - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (N->getScope()) { - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - } - if (N->getBaseType()) { - Out << FS << "baseType: "; - writeMetadataAsOperand(Out, N->getBaseType(), TypePrinter, Machine, - Context); - } - if (N->getSizeInBits()) - Out << FS << "size: " << N->getSizeInBits(); - if (N->getAlignInBits()) - Out << FS << "align: " << N->getAlignInBits(); - if (N->getOffsetInBits()) - Out << FS << "offset: " << N->getOffsetInBits(); - if (auto Flags = N->getFlags()) { - Out << FS << "flags: "; - writeDIFlags(Out, Flags); - } - if (N->getElements()) { - Out << FS << "elements: "; - writeMetadataAsOperand(Out, N->getElements(), TypePrinter, Machine, - Context); - } - if (unsigned Lang = N->getRuntimeLang()) { - Out << FS << "runtimeLang: "; - if (const char *S = dwarf::LanguageString(Lang)) - Out << S; - else - Out << Lang; - } - - if (N->getVTableHolder()) { - Out << FS << "vtableHolder: "; - writeMetadataAsOperand(Out, N->getVTableHolder(), TypePrinter, Machine, - Context); - } - if (N->getTemplateParams()) { - Out << FS << "templateParams: "; - writeMetadataAsOperand(Out, N->getTemplateParams(), TypePrinter, Machine, - Context); - } - if (!N->getIdentifier().empty()) - Out << FS << "identifier: \"" << N->getIdentifier() << "\""; + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printTag(N); + Printer.printString("name", N->getName()); + Printer.printMetadata("scope", N->getScope()); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printMetadata("baseType", N->getBaseType()); + Printer.printInt("size", N->getSizeInBits()); + Printer.printInt("align", N->getAlignInBits()); + Printer.printInt("offset", N->getOffsetInBits()); + Printer.printDIFlags("flags", N->getFlags()); + Printer.printMetadata("elements", N->getElements()); + Printer.printDwarfEnum("runtimeLang", N->getRuntimeLang(), + dwarf::LanguageString); + Printer.printMetadata("vtableHolder", N->getVTableHolder()); + Printer.printMetadata("templateParams", N->getTemplateParams()); + Printer.printString("identifier", N->getIdentifier()); Out << ")"; } @@ -1507,22 +1580,20 @@ static void writeMDSubroutineType(raw_ostream &Out, const MDSubroutineType *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDSubroutineType("; - FieldSeparator FS; - if (auto Flags = N->getFlags()) { - Out << FS << "flags: "; - writeDIFlags(Out, Flags); - } - Out << FS << "types: "; - writeMetadataAsOperand(Out, N->getTypeArray(), TypePrinter, Machine, Context); + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printDIFlags("flags", N->getFlags()); + Printer.printMetadata("types", N->getTypeArray(), /* ShouldSkipNull */ false); Out << ")"; } static void writeMDFile(raw_ostream &Out, const MDFile *N, TypePrinting *, SlotTracker *, const Module *) { Out << "!MDFile("; - FieldSeparator FS; - Out << FS << "filename: \"" << N->getFilename() << "\""; - Out << FS << "directory: \"" << N->getDirectory() << "\""; + MDFieldPrinter Printer(Out); + Printer.printString("filename", N->getFilename(), + /* ShouldSkipEmpty */ false); + Printer.printString("directory", N->getDirectory(), + /* ShouldSkipEmpty */ false); Out << ")"; } @@ -1530,48 +1601,23 @@ static void writeMDCompileUnit(raw_ostream &Out, const MDCompileUnit *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDCompileUnit("; - FieldSeparator FS; - Out << FS << "language: "; - if (const char *Lang = dwarf::LanguageString(N->getSourceLanguage())) - Out << Lang; - else - Out << N->getSourceLanguage(); - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, Context); - if (!N->getProducer().empty()) - Out << FS << "producer: \"" << N->getProducer() << "\""; - Out << FS << "isOptimized: " << (N->isOptimized() ? "true" : "false"); - if (!N->getFlags().empty()) - Out << FS << "flags: \"" << N->getFlags() << "\""; - Out << FS << "runtimeVersion: " << N->getRuntimeVersion(); - if (!N->getSplitDebugFilename().empty()) - Out << FS << "splitDebugFilename: \"" << N->getSplitDebugFilename() << "\""; - Out << FS << "emissionKind: " << N->getEmissionKind(); - if (N->getEnumTypes()) { - Out << FS << "enums: "; - writeMetadataAsOperand(Out, N->getEnumTypes(), TypePrinter, Machine, - Context); - } - if (N->getRetainedTypes()) { - Out << FS << "retainedTypes: "; - writeMetadataAsOperand(Out, N->getRetainedTypes(), TypePrinter, Machine, - Context); - } - if (N->getSubprograms()) { - Out << FS << "subprograms: "; - writeMetadataAsOperand(Out, N->getSubprograms(), TypePrinter, Machine, - Context); - } - if (N->getGlobalVariables()) { - Out << FS << "globals: "; - writeMetadataAsOperand(Out, N->getGlobalVariables(), TypePrinter, Machine, - Context); - } - if (N->getImportedEntities()) { - Out << FS << "imports: "; - writeMetadataAsOperand(Out, N->getImportedEntities(), TypePrinter, Machine, - Context); - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printDwarfEnum("language", N->getSourceLanguage(), + dwarf::LanguageString, /* ShouldSkipZero */ false); + Printer.printMetadata("file", N->getFile(), /* ShouldSkipNull */ false); + Printer.printString("producer", N->getProducer()); + Printer.printBool("isOptimized", N->isOptimized()); + Printer.printString("flags", N->getFlags()); + Printer.printInt("runtimeVersion", N->getRuntimeVersion(), + /* ShouldSkipZero */ false); + Printer.printString("splitDebugFilename", N->getSplitDebugFilename()); + Printer.printInt("emissionKind", N->getEmissionKind(), + /* ShouldSkipZero */ false); + Printer.printMetadata("enums", N->getEnumTypes()); + Printer.printMetadata("retainedTypes", N->getRetainedTypes()); + Printer.printMetadata("subprograms", N->getSubprograms()); + Printer.printMetadata("globals", N->getGlobalVariables()); + Printer.printMetadata("imports", N->getImportedEntities()); Out << ")"; } @@ -1579,67 +1625,26 @@ static void writeMDSubprogram(raw_ostream &Out, const MDSubprogram *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDSubprogram("; - FieldSeparator FS; - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - Out << FS << "name: \"" << N->getName() << "\""; - if (!N->getLinkageName().empty()) - Out << FS << "linkageName: \"" << N->getLinkageName() << "\""; - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (N->getType()) { - Out << FS << "type: "; - writeMetadataAsOperand(Out, N->getType(), TypePrinter, Machine, - Context); - } - Out << FS << "isLocal: " << (N->isLocalToUnit() ? "true" : "false"); - Out << FS << "isDefinition: " << (N->isDefinition() ? "true" : "false"); - if (N->getScopeLine()) - Out << FS << "scopeLine: " << N->getScopeLine(); - if (N->getContainingType()) { - Out << FS << "containingType: "; - writeMetadataAsOperand(Out, N->getContainingType(), TypePrinter, Machine, - Context); - } - if (unsigned V = N->getVirtuality()) { - Out << FS << "virtuality: "; - if (const char *S = dwarf::VirtualityString(V)) - Out << S; - else - Out << V; - } - if (N->getVirtualIndex()) - Out << FS << "virtualIndex: " << N->getVirtualIndex(); - if (auto Flags = N->getFlags()) { - Out << FS << "flags: "; - writeDIFlags(Out, Flags); - } - Out << FS << "isOptimized: " << (N->isOptimized() ? "true" : "false"); - if (N->getFunction()) { - Out << FS << "function: "; - writeMetadataAsOperand(Out, N->getFunction(), TypePrinter, Machine, - Context); - } - if (N->getTemplateParams()) { - Out << FS << "templateParams: "; - writeMetadataAsOperand(Out, N->getTemplateParams(), TypePrinter, Machine, - Context); - } - if (N->getDeclaration()) { - Out << FS << "declaration: "; - writeMetadataAsOperand(Out, N->getDeclaration(), TypePrinter, Machine, - Context); - } - if (N->getVariables()) { - Out << FS << "variables: "; - writeMetadataAsOperand(Out, N->getVariables(), TypePrinter, Machine, - Context); - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printString("name", N->getName()); + Printer.printString("linkageName", N->getLinkageName()); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printMetadata("type", N->getType()); + Printer.printBool("isLocal", N->isLocalToUnit()); + Printer.printBool("isDefinition", N->isDefinition()); + Printer.printInt("scopeLine", N->getScopeLine()); + Printer.printMetadata("containingType", N->getContainingType()); + Printer.printDwarfEnum("virtuality", N->getVirtuality(), + dwarf::VirtualityString); + Printer.printInt("virtualIndex", N->getVirtualIndex()); + Printer.printDIFlags("flags", N->getFlags()); + Printer.printBool("isOptimized", N->isOptimized()); + Printer.printMetadata("function", N->getFunction()); + Printer.printMetadata("templateParams", N->getTemplateParams()); + Printer.printMetadata("declaration", N->getDeclaration()); + Printer.printMetadata("variables", N->getVariables()); Out << ")"; } @@ -1647,18 +1652,11 @@ static void writeMDLexicalBlock(raw_ostream &Out, const MDLexicalBlock *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDLexicalBlock("; - FieldSeparator FS; - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (N->getColumn()) - Out << FS << "column: " << N->getColumn(); + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printInt("column", N->getColumn()); Out << ")"; } @@ -1668,15 +1666,11 @@ static void writeMDLexicalBlockFile(raw_ostream &Out, SlotTracker *Machine, const Module *Context) { Out << "!MDLexicalBlockFile("; - FieldSeparator FS; - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - Out << FS << "discriminator: " << N->getDiscriminator(); + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("discriminator", N->getDiscriminator(), + /* ShouldSkipZero */ false); Out << ")"; } @@ -1684,17 +1678,11 @@ static void writeMDNamespace(raw_ostream &Out, const MDNamespace *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDNamespace("; - FieldSeparator FS; - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, Context); - } - if (!N->getName().empty()) - Out << FS << "name: \"" << N->getName() << "\""; - if (N->getLine()) - Out << FS << "line: " << N->getLine(); + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printString("name", N->getName()); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); Out << ")"; } @@ -1704,10 +1692,9 @@ static void writeMDTemplateTypeParameter(raw_ostream &Out, SlotTracker *Machine, const Module *Context) { Out << "!MDTemplateTypeParameter("; - FieldSeparator FS; - Out << FS << "name: \"" << N->getName() << "\""; - Out << FS << "type: "; - writeMetadataAsOperand(Out, N->getType(), TypePrinter, Machine, Context); + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printString("name", N->getName()); + Printer.printMetadata("type", N->getType(), /* ShouldSkipNull */ false); Out << ")"; } @@ -1717,13 +1704,12 @@ static void writeMDTemplateValueParameter(raw_ostream &Out, SlotTracker *Machine, const Module *Context) { Out << "!MDTemplateValueParameter("; - FieldSeparator FS; - writeTag(Out, FS, N); - Out << FS << "name: \"" << N->getName() << "\""; - Out << FS << "type: "; - writeMetadataAsOperand(Out, N->getType(), TypePrinter, Machine, Context); - Out << FS << "value: "; - writeMetadataAsOperand(Out, N->getValue(), TypePrinter, Machine, Context); + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + if (N->getTag() != dwarf::DW_TAG_template_value_parameter) + Printer.printTag(N); + Printer.printString("name", N->getName()); + Printer.printMetadata("type", N->getType()); + Printer.printMetadata("value", N->getValue(), /* ShouldSkipNull */ false); Out << ")"; } @@ -1731,36 +1717,17 @@ static void writeMDGlobalVariable(raw_ostream &Out, const MDGlobalVariable *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDGlobalVariable("; - FieldSeparator FS; - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - Out << FS << "name: \"" << N->getName() << "\""; - if (!N->getLinkageName().empty()) - Out << FS << "linkageName: \"" << N->getLinkageName() << "\""; - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (N->getType()) { - Out << FS << "type: "; - writeMetadataAsOperand(Out, N->getType(), TypePrinter, Machine, - Context); - } - Out << FS << "isLocal: " << (N->isLocalToUnit() ? "true" : "false"); - Out << FS << "isDefinition: " << (N->isDefinition() ? "true" : "false"); - if (N->getVariable()) { - Out << FS << "variable: "; - writeMetadataAsOperand(Out, N->getVariable(), TypePrinter, Machine, - Context); - } - if (N->getStaticDataMemberDeclaration()) { - Out << FS << "declaration: "; - writeMetadataAsOperand(Out, N->getStaticDataMemberDeclaration(), - TypePrinter, Machine, Context); - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printString("name", N->getName()); + Printer.printString("linkageName", N->getLinkageName()); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printMetadata("type", N->getType()); + Printer.printBool("isLocal", N->isLocalToUnit()); + Printer.printBool("isDefinition", N->isDefinition()); + Printer.printMetadata("variable", N->getVariable()); + Printer.printMetadata("declaration", N->getStaticDataMemberDeclaration()); Out << ")"; } @@ -1768,34 +1735,18 @@ static void writeMDLocalVariable(raw_ostream &Out, const MDLocalVariable *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDLocalVariable("; - FieldSeparator FS; - writeTag(Out, FS, N); - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - Out << FS << "name: \"" << N->getName() << "\""; - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, - Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (N->getType()) { - Out << FS << "type: "; - writeMetadataAsOperand(Out, N->getType(), TypePrinter, Machine, - Context); - } - if (N->getTag() == dwarf::DW_TAG_arg_variable || N->getArg()) - Out << FS << "arg: " << N->getArg(); - if (auto Flags = N->getFlags()) { - Out << FS << "flags: "; - writeDIFlags(Out, Flags); - } - if (N->getInlinedAt()) { - Out << FS << "inlinedAt: "; - writeMetadataAsOperand(Out, N->getInlinedAt(), TypePrinter, Machine, - Context); - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printTag(N); + Printer.printString("name", N->getName()); + Printer.printInt("arg", N->getArg(), + /* ShouldSkipZero */ + N->getTag() == dwarf::DW_TAG_auto_variable); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printMetadata("type", N->getType()); + Printer.printDIFlags("flags", N->getFlags()); + Printer.printMetadata("inlinedAt", N->getInlinedAt()); Out << ")"; } @@ -1824,24 +1775,14 @@ static void writeMDObjCProperty(raw_ostream &Out, const MDObjCProperty *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDObjCProperty("; - FieldSeparator FS; - Out << FS << "name: \"" << N->getName() << "\""; - if (N->getFile()) { - Out << FS << "file: "; - writeMetadataAsOperand(Out, N->getFile(), TypePrinter, Machine, Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - if (!N->getSetterName().empty()) - Out << FS << "setter: \"" << N->getSetterName() << "\""; - if (!N->getGetterName().empty()) - Out << FS << "getter: \"" << N->getGetterName() << "\""; - if (N->getAttributes()) - Out << FS << "attributes: " << N->getAttributes(); - if (N->getType()) { - Out << FS << "type: "; - writeMetadataAsOperand(Out, N->getType(), TypePrinter, Machine, Context); - } + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printString("name", N->getName()); + Printer.printMetadata("file", N->getFile()); + Printer.printInt("line", N->getLine()); + Printer.printString("setter", N->getSetterName()); + Printer.printString("getter", N->getGetterName()); + Printer.printInt("attributes", N->getAttributes()); + Printer.printMetadata("type", N->getType()); Out << ")"; } @@ -1849,17 +1790,12 @@ static void writeMDImportedEntity(raw_ostream &Out, const MDImportedEntity *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { Out << "!MDImportedEntity("; - FieldSeparator FS; - writeTag(Out, FS, N); - Out << FS << "scope: "; - writeMetadataAsOperand(Out, N->getScope(), TypePrinter, Machine, Context); - if (N->getEntity()) { - Out << FS << "entity: "; - writeMetadataAsOperand(Out, N->getEntity(), TypePrinter, Machine, Context); - } - if (N->getLine()) - Out << FS << "line: " << N->getLine(); - Out << FS << "name: \"" << N->getName() << "\""; + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printTag(N); + Printer.printString("name", N->getName()); + Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("entity", N->getEntity()); + Printer.printInt("line", N->getLine()); Out << ")"; } @@ -1868,10 +1804,10 @@ static void WriteMDNodeBodyInternal(raw_ostream &Out, const MDNode *Node, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { - assert(!Node->isTemporary() && "Unexpected forward declaration"); - if (Node->isDistinct()) Out << "distinct "; + else if (Node->isTemporary()) + Out << " "; // Handle broken code. switch (Node->getMetadataID()) { default: @@ -1998,6 +1934,64 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Metadata *MD, WriteAsOperandInternal(Out, V->getValue(), TypePrinter, Machine, Context); } +namespace { +class AssemblyWriter { + formatted_raw_ostream &Out; + const Module *TheModule; + std::unique_ptr ModuleSlotTracker; + SlotTracker &Machine; + TypePrinting TypePrinter; + AssemblyAnnotationWriter *AnnotationWriter; + SetVector Comdats; + UseListOrderStack UseListOrders; + +public: + /// Construct an AssemblyWriter with an external SlotTracker + AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac, + const Module *M, AssemblyAnnotationWriter *AAW); + + /// Construct an AssemblyWriter with an internally allocated SlotTracker + AssemblyWriter(formatted_raw_ostream &o, const Module *M, + AssemblyAnnotationWriter *AAW); + + void printMDNodeBody(const MDNode *MD); + void printNamedMDNode(const NamedMDNode *NMD); + + void printModule(const Module *M); + + void writeOperand(const Value *Op, bool PrintType); + void writeParamOperand(const Value *Operand, AttributeSet Attrs,unsigned Idx); + void writeAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope); + void writeAtomicCmpXchg(AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope); + + void writeAllMDNodes(); + void writeMDNode(unsigned Slot, const MDNode *Node); + void writeAllAttributeGroups(); + + void printTypeIdentities(); + void printGlobal(const GlobalVariable *GV); + void printAlias(const GlobalAlias *GV); + void printComdat(const Comdat *C); + void printFunction(const Function *F); + void printArgument(const Argument *FA, AttributeSet Attrs, unsigned Idx); + void printBasicBlock(const BasicBlock *BB); + void printInstructionLine(const Instruction &I); + void printInstruction(const Instruction &I); + + void printUseListOrder(const UseListOrder &Order); + void printUseLists(const Function *F); + +private: + void init(); + + // printInfoComment - Print a little comment after the instruction indicating + // which slot it occupies. + void printInfoComment(const Value &V); +}; +} // namespace + void AssemblyWriter::init() { if (!TheModule) return; @@ -2025,8 +2019,6 @@ AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, const Module *M, init(); } -AssemblyWriter::~AssemblyWriter() { } - void AssemblyWriter::writeOperand(const Value *Operand, bool PrintType) { if (!Operand) { Out << ""; @@ -2876,7 +2868,13 @@ void AssemblyWriter::printInstruction(const Instruction &I) { if (AI->isUsedWithInAlloca()) Out << "inalloca "; TypePrinter.print(AI->getAllocatedType(), Out); - if (!AI->getArraySize() || AI->isArrayAllocation()) { + + // Explicitly write the array size if the code is broken, if it's an array + // allocation, or if the type is not canonical for scalar allocations. The + // latter case prevents the type from mutating when round-tripping through + // assembly. + if (!AI->getArraySize() || AI->isArrayAllocation() || + !AI->getArraySize()->getType()->isIntegerTy(32)) { Out << ", "; writeOperand(AI->getArraySize(), true); } @@ -2898,6 +2896,15 @@ void AssemblyWriter::printInstruction(const Instruction &I) { Out << ", "; TypePrinter.print(I.getType(), Out); } else if (Operand) { // Print the normal way. + if (const auto *GEP = dyn_cast(&I)) { + Out << ' '; + TypePrinter.print(GEP->getSourceElementType(), Out); + Out << ','; + } else if (const auto *LI = dyn_cast(&I)) { + Out << ' '; + TypePrinter.print(LI->getType(), Out); + Out << ','; + } // PrintAllTypes - Instructions who have operands of all the same type // omit the type from all but the first operand. If the instruction has @@ -2974,29 +2981,6 @@ void AssemblyWriter::printInstruction(const Instruction &I) { printInfoComment(I); } -static void WriteMDNodeComment(const MDNode *Node, - formatted_raw_ostream &Out) { - if (Node->getNumOperands() < 1) - return; - - Metadata *Op = Node->getOperand(0); - if (!Op || !isa(Op)) - return; - - DIDescriptor Desc(Node); - if (!Desc.Verify()) - return; - - unsigned Tag = Desc.getTag(); - Out.PadToColumn(50); - if (dwarf::TagString(Tag)) { - Out << "; "; - Desc.print(Out); - } else if (Tag == dwarf::DW_TAG_user_base) { - Out << "; [ DW_TAG_user_base ]"; - } -} - void AssemblyWriter::writeMDNode(unsigned Slot, const MDNode *Node) { Out << '!' << Slot << " = "; printMDNodeBody(Node); @@ -3017,7 +3001,6 @@ void AssemblyWriter::writeAllMDNodes() { void AssemblyWriter::printMDNodeBody(const MDNode *Node) { WriteMDNodeBodyInternal(Out, Node, &TypePrinter, &Machine, TheModule); - WriteMDNodeComment(Node, Out); } void AssemblyWriter::writeAllAttributeGroups() { @@ -3034,8 +3017,6 @@ void AssemblyWriter::writeAllAttributeGroups() { << I->first.getAsString(AttributeSet::FunctionIndex, true) << " }\n"; } -} // namespace llvm - void AssemblyWriter::printUseListOrder(const UseListOrder &Order) { bool IsInFunction = Machine.getFunction(); if (IsInFunction) @@ -3130,11 +3111,24 @@ void Type::print(raw_ostream &OS) const { } } +static bool isReferencingMDNode(const Instruction &I) { + if (const auto *CI = dyn_cast(&I)) + if (Function *F = CI->getCalledFunction()) + if (F->isIntrinsic()) + for (auto &Op : I.operands()) + if (auto *V = dyn_cast_or_null(Op)) + if (isa(V->getMetadata())) + return true; + return false; +} + void Value::print(raw_ostream &ROS) const { formatted_raw_ostream OS(ROS); if (const Instruction *I = dyn_cast(this)) { const Function *F = I->getParent() ? I->getParent()->getParent() : nullptr; - SlotTracker SlotTable(F); + SlotTracker SlotTable( + F, + /* ShouldInitializeAllMetadata */ isReferencingMDNode(*I)); AssemblyWriter W(OS, SlotTable, getModuleFromVal(I), nullptr); W.printInstruction(*I); } else if (const BasicBlock *BB = dyn_cast(this)) { @@ -3142,7 +3136,8 @@ void Value::print(raw_ostream &ROS) const { AssemblyWriter W(OS, SlotTable, getModuleFromVal(BB), nullptr); W.printBasicBlock(BB); } else if (const GlobalValue *GV = dyn_cast(this)) { - SlotTracker SlotTable(GV->getParent()); + SlotTracker SlotTable(GV->getParent(), + /* ShouldInitializeAllMetadata */ isa(GV)); AssemblyWriter W(OS, SlotTable, GV->getParent(), nullptr); if (const GlobalVariable *V = dyn_cast(GV)) W.printGlobal(V); @@ -3151,7 +3146,7 @@ void Value::print(raw_ostream &ROS) const { else W.printAlias(cast(GV)); } else if (const MetadataAsValue *V = dyn_cast(this)) { - V->getMetadata()->print(ROS); + V->getMetadata()->print(ROS, getModuleFromVal(V)); } else if (const Constant *C = dyn_cast(this)) { TypePrinting TypePrinter; TypePrinter.print(C->getType(), OS); @@ -3167,8 +3162,9 @@ void Value::print(raw_ostream &ROS) const { void Value::printAsOperand(raw_ostream &O, bool PrintType, const Module *M) const { // Fast path: Don't construct and populate a TypePrinting object if we // won't be needing any types printed. - if (!PrintType && ((!isa(this) && !isa(this)) || - hasName() || isa(this))) { + bool IsMetadata = isa(this); + if (!PrintType && ((!isa(this) && !IsMetadata) || hasName() || + isa(this))) { WriteAsOperandInternal(O, this, nullptr, nullptr, M); return; } @@ -3184,33 +3180,35 @@ void Value::printAsOperand(raw_ostream &O, bool PrintType, const Module *M) cons O << ' '; } - WriteAsOperandInternal(O, this, &TypePrinter, nullptr, M); + SlotTracker Machine(M, /* ShouldInitializeAllMetadata */ IsMetadata); + WriteAsOperandInternal(O, this, &TypePrinter, &Machine, M); } -void Metadata::print(raw_ostream &ROS) const { +static void printMetadataImpl(raw_ostream &ROS, const Metadata &MD, + const Module *M, bool OnlyAsOperand) { formatted_raw_ostream OS(ROS); - if (auto *N = dyn_cast(this)) { - SlotTracker SlotTable(static_cast(nullptr)); - AssemblyWriter W(OS, SlotTable, nullptr, nullptr); - W.printMDNodeBody(N); + auto *N = dyn_cast(&MD); + TypePrinting TypePrinter; + SlotTracker Machine(M, /* ShouldInitializeAllMetadata */ N); + if (M) + TypePrinter.incorporateTypes(*M); + + WriteAsOperandInternal(OS, &MD, &TypePrinter, &Machine, M, + /* FromValue */ true); + if (OnlyAsOperand || !N) return; - } - printAsOperand(OS); + + OS << " = "; + WriteMDNodeBodyInternal(OS, N, &TypePrinter, &Machine, M); } -void Metadata::printAsOperand(raw_ostream &ROS, bool PrintType, - const Module *M) const { - formatted_raw_ostream OS(ROS); +void Metadata::printAsOperand(raw_ostream &OS, const Module *M) const { + printMetadataImpl(OS, *this, M, /* OnlyAsOperand */ true); +} - std::unique_ptr TypePrinter; - if (PrintType) { - TypePrinter.reset(new TypePrinting); - if (M) - TypePrinter->incorporateTypes(*M); - } - WriteAsOperandInternal(OS, this, TypePrinter.get(), nullptr, M, - /* FromValue */ true); +void Metadata::print(raw_ostream &OS, const Module *M) const { + printMetadataImpl(OS, *this, M, /* OnlyAsOperand */ false); } // Value::dump - allow easy printing of Values from the debugger. @@ -3234,7 +3232,10 @@ LLVM_DUMP_METHOD void NamedMDNode::dump() const { print(dbgs()); } LLVM_DUMP_METHOD -void Metadata::dump() const { - print(dbgs()); +void Metadata::dump() const { dump(nullptr); } + +LLVM_DUMP_METHOD +void Metadata::dump(const Module *M) const { + print(dbgs(), M); dbgs() << '\n'; } diff --git a/lib/IR/AsmWriter.h b/lib/IR/AsmWriter.h deleted file mode 100644 index 7716fa6..0000000 --- a/lib/IR/AsmWriter.h +++ /dev/null @@ -1,129 +0,0 @@ -//===-- llvm/IR/AsmWriter.h - Printing LLVM IR as an assembly file - C++ --===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This files defines the interface for the AssemblyWriter class used to print -// LLVM IR and various helper classes that are used in printing. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_IR_ASMWRITER_H -#define LLVM_LIB_IR_ASMWRITER_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/TypeFinder.h" -#include "llvm/IR/UseListOrder.h" -#include "llvm/Support/FormattedStream.h" - -namespace llvm { - -class BasicBlock; -class Function; -class GlobalValue; -class Comdat; -class Module; -class NamedMDNode; -class Value; -class SlotTracker; - -/// Create a new SlotTracker for a Module -SlotTracker *createSlotTracker(const Module *M); - -//===----------------------------------------------------------------------===// -// TypePrinting Class: Type printing machinery -//===----------------------------------------------------------------------===// - -class TypePrinting { - TypePrinting(const TypePrinting &) = delete; - void operator=(const TypePrinting&) = delete; -public: - - /// NamedTypes - The named types that are used by the current module. - TypeFinder NamedTypes; - - /// NumberedTypes - The numbered types, along with their value. - DenseMap NumberedTypes; - - - TypePrinting() {} - ~TypePrinting() {} - - void incorporateTypes(const Module &M); - - void print(Type *Ty, raw_ostream &OS); - - void printStructBody(StructType *Ty, raw_ostream &OS); -}; - -class AssemblyWriter { -protected: - formatted_raw_ostream &Out; - const Module *TheModule; - -private: - std::unique_ptr ModuleSlotTracker; - SlotTracker &Machine; - TypePrinting TypePrinter; - AssemblyAnnotationWriter *AnnotationWriter; - SetVector Comdats; - UseListOrderStack UseListOrders; - -public: - /// Construct an AssemblyWriter with an external SlotTracker - AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac, - const Module *M, AssemblyAnnotationWriter *AAW); - - /// Construct an AssemblyWriter with an internally allocated SlotTracker - AssemblyWriter(formatted_raw_ostream &o, const Module *M, - AssemblyAnnotationWriter *AAW); - - virtual ~AssemblyWriter(); - - void printMDNodeBody(const MDNode *MD); - void printNamedMDNode(const NamedMDNode *NMD); - - void printModule(const Module *M); - - void writeOperand(const Value *Op, bool PrintType); - void writeParamOperand(const Value *Operand, AttributeSet Attrs,unsigned Idx); - void writeAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope); - void writeAtomicCmpXchg(AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); - - void writeAllMDNodes(); - void writeMDNode(unsigned Slot, const MDNode *Node); - void writeAllAttributeGroups(); - - void printTypeIdentities(); - void printGlobal(const GlobalVariable *GV); - void printAlias(const GlobalAlias *GV); - void printComdat(const Comdat *C); - void printFunction(const Function *F); - void printArgument(const Argument *FA, AttributeSet Attrs, unsigned Idx); - void printBasicBlock(const BasicBlock *BB); - void printInstructionLine(const Instruction &I); - void printInstruction(const Instruction &I); - - void printUseListOrder(const UseListOrder &Order); - void printUseLists(const Function *F); - -private: - void init(); - - // printInfoComment - Print a little comment after the instruction indicating - // which slot it occupies. - void printInfoComment(const Value &V); -}; - -} // namespace llvm - -#endif diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp index 0da7784..d2dfeaa 100644 --- a/lib/IR/AutoUpgrade.cpp +++ b/lib/IR/AutoUpgrade.cpp @@ -7,7 +7,9 @@ // //===----------------------------------------------------------------------===// // -// This file implements the auto-upgrade helper functions +// This file implements the auto-upgrade helper functions. +// This is where deprecated IR intrinsics and other IR features are updated to +// current specifications. // //===----------------------------------------------------------------------===// @@ -156,6 +158,14 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { Name.startswith("x86.avx2.pcmpeq.") || Name.startswith("x86.avx2.pcmpgt.") || Name.startswith("x86.avx.vpermil.") || + Name == "x86.avx.vinsertf128.pd.256" || + Name == "x86.avx.vinsertf128.ps.256" || + Name == "x86.avx.vinsertf128.si.256" || + Name == "x86.avx2.vinserti128" || + Name == "x86.avx.vextractf128.pd.256" || + Name == "x86.avx.vextractf128.ps.256" || + Name == "x86.avx.vextractf128.si.256" || + Name == "x86.avx2.vextracti128" || Name == "x86.avx.movnt.dq.256" || Name == "x86.avx.movnt.pd.256" || Name == "x86.avx.movnt.ps.256" || @@ -171,6 +181,15 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { Name == "x86.sse2.psrl.dq.bs" || Name == "x86.avx2.psll.dq.bs" || Name == "x86.avx2.psrl.dq.bs" || + Name == "x86.sse41.pblendw" || + Name == "x86.sse41.blendpd" || + Name == "x86.sse41.blendps" || + Name == "x86.avx.blend.pd.256" || + Name == "x86.avx.blend.ps.256" || + Name == "x86.avx2.pblendw" || + Name == "x86.avx2.pblendd.128" || + Name == "x86.avx2.pblendd.256" || + Name == "x86.avx2.vbroadcasti128" || (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { NewFn = nullptr; return true; @@ -184,17 +203,8 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { if (Name == "x86.sse41.ptestnzc") return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); } - // Several blend and other instructions with maskes used the wrong number of + // Several blend and other instructions with masks used the wrong number of // bits. - if (Name == "x86.sse41.pblendw") - return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_pblendw, - NewFn); - if (Name == "x86.sse41.blendpd") - return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendpd, - NewFn); - if (Name == "x86.sse41.blendps") - return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendps, - NewFn); if (Name == "x86.sse41.insertps") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, NewFn); @@ -207,24 +217,9 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { if (Name == "x86.sse41.mpsadbw") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, NewFn); - if (Name == "x86.avx.blend.pd.256") - return UpgradeX86IntrinsicsWith8BitMask( - F, Intrinsic::x86_avx_blend_pd_256, NewFn); - if (Name == "x86.avx.blend.ps.256") - return UpgradeX86IntrinsicsWith8BitMask( - F, Intrinsic::x86_avx_blend_ps_256, NewFn); if (Name == "x86.avx.dp.ps.256") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, NewFn); - if (Name == "x86.avx2.pblendw") - return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_pblendw, - NewFn); - if (Name == "x86.avx2.pblendd.128") - return UpgradeX86IntrinsicsWith8BitMask( - F, Intrinsic::x86_avx2_pblendd_128, NewFn); - if (Name == "x86.avx2.pblendd.256") - return UpgradeX86IntrinsicsWith8BitMask( - F, Intrinsic::x86_avx2_pblendd_256, NewFn); if (Name == "x86.avx2.mpsadbw") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, NewFn); @@ -569,6 +564,15 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { for (unsigned I = 0; I < EltNum; ++I) Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I)); + } else if (Name == "llvm.x86.avx2.vbroadcasti128") { + // Replace vbroadcasts with a vector shuffle. + Value *Op = Builder.CreatePointerCast( + CI->getArgOperand(0), + PointerType::getUnqual(VectorType::get(Type::getInt64Ty(C), 2))); + Value *Load = Builder.CreateLoad(Op); + const int Idxs[4] = { 0, 1, 0, 1 }; + Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), + Idxs); } else if (Name == "llvm.x86.sse2.psll.dq") { // 128-bit shift left specified in bits. unsigned Shift = cast(CI->getArgOperand(1))->getZExtValue(); @@ -609,6 +613,94 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { unsigned Shift = cast(CI->getArgOperand(1))->getZExtValue(); Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, Shift); + } else if (Name == "llvm.x86.sse41.pblendw" || + Name == "llvm.x86.sse41.blendpd" || + Name == "llvm.x86.sse41.blendps" || + Name == "llvm.x86.avx.blend.pd.256" || + Name == "llvm.x86.avx.blend.ps.256" || + Name == "llvm.x86.avx2.pblendw" || + Name == "llvm.x86.avx2.pblendd.128" || + Name == "llvm.x86.avx2.pblendd.256") { + Value *Op0 = CI->getArgOperand(0); + Value *Op1 = CI->getArgOperand(1); + unsigned Imm = cast (CI->getArgOperand(2))->getZExtValue(); + VectorType *VecTy = cast(CI->getType()); + unsigned NumElts = VecTy->getNumElements(); + + SmallVector Idxs; + for (unsigned i = 0; i != NumElts; ++i) { + unsigned Idx = ((Imm >> (i%8)) & 1) ? i + NumElts : i; + Idxs.push_back(Builder.getInt32(Idx)); + } + + Rep = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs)); + } else if (Name == "llvm.x86.avx.vinsertf128.pd.256" || + Name == "llvm.x86.avx.vinsertf128.ps.256" || + Name == "llvm.x86.avx.vinsertf128.si.256" || + Name == "llvm.x86.avx2.vinserti128") { + Value *Op0 = CI->getArgOperand(0); + Value *Op1 = CI->getArgOperand(1); + unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue(); + VectorType *VecTy = cast(CI->getType()); + unsigned NumElts = VecTy->getNumElements(); + + // Mask off the high bits of the immediate value; hardware ignores those. + Imm = Imm & 1; + + // Extend the second operand into a vector that is twice as big. + Value *UndefV = UndefValue::get(Op1->getType()); + SmallVector Idxs; + for (unsigned i = 0; i != NumElts; ++i) { + Idxs.push_back(Builder.getInt32(i)); + } + Rep = Builder.CreateShuffleVector(Op1, UndefV, ConstantVector::get(Idxs)); + + // Insert the second operand into the first operand. + + // Note that there is no guarantee that instruction lowering will actually + // produce a vinsertf128 instruction for the created shuffles. In + // particular, the 0 immediate case involves no lane changes, so it can + // be handled as a blend. + + // Example of shuffle mask for 32-bit elements: + // Imm = 1 + // Imm = 0 + + SmallVector Idxs2; + // The low half of the result is either the low half of the 1st operand + // or the low half of the 2nd operand (the inserted vector). + for (unsigned i = 0; i != NumElts / 2; ++i) { + unsigned Idx = Imm ? i : (i + NumElts); + Idxs2.push_back(Builder.getInt32(Idx)); + } + // The high half of the result is either the low half of the 2nd operand + // (the inserted vector) or the high half of the 1st operand. + for (unsigned i = NumElts / 2; i != NumElts; ++i) { + unsigned Idx = Imm ? (i + NumElts / 2) : i; + Idxs2.push_back(Builder.getInt32(Idx)); + } + Rep = Builder.CreateShuffleVector(Op0, Rep, ConstantVector::get(Idxs2)); + } else if (Name == "llvm.x86.avx.vextractf128.pd.256" || + Name == "llvm.x86.avx.vextractf128.ps.256" || + Name == "llvm.x86.avx.vextractf128.si.256" || + Name == "llvm.x86.avx2.vextracti128") { + Value *Op0 = CI->getArgOperand(0); + unsigned Imm = cast(CI->getArgOperand(1))->getZExtValue(); + VectorType *VecTy = cast(CI->getType()); + unsigned NumElts = VecTy->getNumElements(); + + // Mask off the high bits of the immediate value; hardware ignores those. + Imm = Imm & 1; + + // Get indexes for either the high half or low half of the input vector. + SmallVector Idxs(NumElts); + for (unsigned i = 0; i != NumElts; ++i) { + unsigned Idx = Imm ? (i + NumElts) : i; + Idxs[i] = Builder.getInt32(Idx); + } + + Value *UndefV = UndefValue::get(Op0->getType()); + Rep = Builder.CreateShuffleVector(Op0, UndefV, ConstantVector::get(Idxs)); } else { bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; if (Name == "llvm.x86.avx.vpermil.pd.256") @@ -739,19 +831,11 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { return; } - case Intrinsic::x86_sse41_pblendw: - case Intrinsic::x86_sse41_blendpd: - case Intrinsic::x86_sse41_blendps: case Intrinsic::x86_sse41_insertps: case Intrinsic::x86_sse41_dppd: case Intrinsic::x86_sse41_dpps: case Intrinsic::x86_sse41_mpsadbw: - case Intrinsic::x86_avx_blend_pd_256: - case Intrinsic::x86_avx_blend_ps_256: case Intrinsic::x86_avx_dp_ps_256: - case Intrinsic::x86_avx2_pblendw: - case Intrinsic::x86_avx2_pblendd_128: - case Intrinsic::x86_avx2_pblendd_256: case Intrinsic::x86_avx2_mpsadbw: { // Need to truncate the last argument from i32 to i8 -- this argument models // an inherently 8-bit immediate operand to these x86 instructions. diff --git a/lib/IR/BasicBlock.cpp b/lib/IR/BasicBlock.cpp index b3b3cbf..fe38385 100644 --- a/lib/IR/BasicBlock.cpp +++ b/lib/IR/BasicBlock.cpp @@ -29,10 +29,6 @@ ValueSymbolTable *BasicBlock::getValueSymbolTable() { return nullptr; } -const DataLayout *BasicBlock::getDataLayout() const { - return getParent()->getDataLayout(); -} - LLVMContext &BasicBlock::getContext() const { return getType()->getContext(); } @@ -102,14 +98,14 @@ void BasicBlock::eraseFromParent() { getParent()->getBasicBlockList().erase(this); } -/// moveBefore - Unlink this basic block from its current function and +/// Unlink this basic block from its current function and /// insert it into the function that MovePos lives in, right before MovePos. void BasicBlock::moveBefore(BasicBlock *MovePos) { MovePos->getParent()->getBasicBlockList().splice(MovePos, getParent()->getBasicBlockList(), this); } -/// moveAfter - Unlink this basic block from its current function and +/// Unlink this basic block from its current function and /// insert it into the function that MovePos lives in, right after MovePos. void BasicBlock::moveAfter(BasicBlock *MovePos) { Function::iterator I = MovePos; @@ -117,6 +113,9 @@ void BasicBlock::moveAfter(BasicBlock *MovePos) { getParent()->getBasicBlockList(), this); } +const Module *BasicBlock::getModule() const { + return getParent()->getParent(); +} TerminatorInst *BasicBlock::getTerminator() { if (InstList.empty()) return nullptr; @@ -210,7 +209,7 @@ void BasicBlock::dropAllReferences() { I->dropAllReferences(); } -/// getSinglePredecessor - If this basic block has a single predecessor block, +/// If this basic block has a single predecessor block, /// return the block, otherwise return a null pointer. BasicBlock *BasicBlock::getSinglePredecessor() { pred_iterator PI = pred_begin(this), E = pred_end(this); @@ -220,7 +219,7 @@ BasicBlock *BasicBlock::getSinglePredecessor() { return (PI == E) ? ThePred : nullptr /*multiple preds*/; } -/// getUniquePredecessor - If this basic block has a unique predecessor block, +/// If this basic block has a unique predecessor block, /// return the block, otherwise return a null pointer. /// Note that unique predecessor doesn't mean single edge, there can be /// multiple edges from the unique predecessor to this block (for example @@ -253,7 +252,7 @@ BasicBlock *BasicBlock::getUniqueSuccessor() { return SuccBB; } -/// removePredecessor - This method is used to notify a BasicBlock that the +/// This method is used to notify a BasicBlock that the /// specified Predecessor of the block is no longer able to reach it. This is /// actually not used to update the Predecessor list, but is actually used to /// update the PHI nodes that reside in the block. Note that this should be @@ -330,7 +329,7 @@ void BasicBlock::removePredecessor(BasicBlock *Pred, } -/// splitBasicBlock - This splits a basic block into two at the specified +/// This splits a basic block into two at the specified /// instruction. Note that all instructions BEFORE the specified iterator stay /// as part of the original basic block, an unconditional branch is added to /// the new BB, and the rest of the instructions in the BB are moved to the new @@ -401,14 +400,13 @@ void BasicBlock::replaceSuccessorsPhiUsesWith(BasicBlock *New) { } } -/// isLandingPad - Return true if this basic block is a landing pad. I.e., it's +/// Return true if this basic block is a landing pad. I.e., it's /// the destination of the 'unwind' edge of an invoke instruction. bool BasicBlock::isLandingPad() const { return isa(getFirstNonPHI()); } -/// getLandingPadInst() - Return the landingpad instruction associated with -/// the landing pad. +/// Return the landingpad instruction associated with the landing pad. LandingPadInst *BasicBlock::getLandingPadInst() { return dyn_cast(getFirstNonPHI()); } diff --git a/lib/IR/ConstantFold.cpp b/lib/IR/ConstantFold.cpp index a915d28..d97d2c4 100644 --- a/lib/IR/ConstantFold.cpp +++ b/lib/IR/ConstantFold.cpp @@ -1120,27 +1120,18 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, return ConstantInt::get(CI1->getContext(), C1V | C2V); case Instruction::Xor: return ConstantInt::get(CI1->getContext(), C1V ^ C2V); - case Instruction::Shl: { - uint32_t shiftAmt = C2V.getZExtValue(); - if (shiftAmt < C1V.getBitWidth()) - return ConstantInt::get(CI1->getContext(), C1V.shl(shiftAmt)); - else - return UndefValue::get(C1->getType()); // too big shift is undef - } - case Instruction::LShr: { - uint32_t shiftAmt = C2V.getZExtValue(); - if (shiftAmt < C1V.getBitWidth()) - return ConstantInt::get(CI1->getContext(), C1V.lshr(shiftAmt)); - else - return UndefValue::get(C1->getType()); // too big shift is undef - } - case Instruction::AShr: { - uint32_t shiftAmt = C2V.getZExtValue(); - if (shiftAmt < C1V.getBitWidth()) - return ConstantInt::get(CI1->getContext(), C1V.ashr(shiftAmt)); - else - return UndefValue::get(C1->getType()); // too big shift is undef - } + case Instruction::Shl: + if (C2V.ult(C1V.getBitWidth())) + return ConstantInt::get(CI1->getContext(), C1V.shl(C2V)); + return UndefValue::get(C1->getType()); // too big shift is undef + case Instruction::LShr: + if (C2V.ult(C1V.getBitWidth())) + return ConstantInt::get(CI1->getContext(), C1V.lshr(C2V)); + return UndefValue::get(C1->getType()); // too big shift is undef + case Instruction::AShr: + if (C2V.ult(C1V.getBitWidth())) + return ConstantInt::get(CI1->getContext(), C1V.ashr(C2V)); + return UndefValue::get(C1->getType()); // too big shift is undef } } @@ -1327,7 +1318,7 @@ static FCmpInst::Predicate evaluateFCmpRelation(Constant *V1, Constant *V2) { if (!isa(V1)) { if (!isa(V2)) { - // We distilled thisUse the standard constant folder for a few cases + // Simple case, use the standard constant folder. ConstantInt *R = nullptr; R = dyn_cast( ConstantExpr::getFCmp(FCmpInst::FCMP_OEQ, V1, V2)); @@ -1665,15 +1656,22 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, // Handle some degenerate cases first if (isa(C1) || isa(C2)) { + CmpInst::Predicate Predicate = CmpInst::Predicate(pred); + bool isIntegerPredicate = ICmpInst::isIntPredicate(Predicate); // For EQ and NE, we can always pick a value for the undef to make the // predicate pass or fail, so we can return undef. - // Also, if both operands are undef, we can return undef. - if (ICmpInst::isEquality(ICmpInst::Predicate(pred)) || - (isa(C1) && isa(C2))) + // Also, if both operands are undef, we can return undef for int comparison. + if (ICmpInst::isEquality(Predicate) || (isIntegerPredicate && C1 == C2)) return UndefValue::get(ResultTy); - // Otherwise, pick the same value as the non-undef operand, and fold - // it to true or false. - return ConstantInt::get(ResultTy, CmpInst::isTrueWhenEqual(pred)); + + // Otherwise, for integer compare, pick the same value as the non-undef + // operand, and fold it to true or false. + if (isIntegerPredicate) + return ConstantInt::get(ResultTy, CmpInst::isTrueWhenEqual(pred)); + + // Choosing NaN for the undef will always make unordered comparison succeed + // and ordered comparison fails. + return ConstantInt::get(ResultTy, CmpInst::isUnordered(Predicate)); } // icmp eq/ne(null,GV) -> false/true @@ -1789,7 +1787,10 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, return ConstantVector::get(ResElts); } - if (C1->getType()->isFloatingPointTy()) { + if (C1->getType()->isFloatingPointTy() && + // Only call evaluateFCmpRelation if we have a constant expr to avoid + // infinite recursive loop + (isa(C1) || isa(C2))) { int Result = -1; // -1 = unknown, 0 = known false, 1 = known true. switch (evaluateFCmpRelation(C1, C2)) { default: llvm_unreachable("Unknown relation!"); diff --git a/lib/IR/ConstantRange.cpp b/lib/IR/ConstantRange.cpp index f8e9ba4..91095cf 100644 --- a/lib/IR/ConstantRange.cpp +++ b/lib/IR/ConstantRange.cpp @@ -49,14 +49,15 @@ ConstantRange::ConstantRange(APIntMoveTy L, APIntMoveTy U) "Lower == Upper, but they aren't min or max value!"); } -ConstantRange ConstantRange::makeICmpRegion(unsigned Pred, - const ConstantRange &CR) { +ConstantRange ConstantRange::makeAllowedICmpRegion(CmpInst::Predicate Pred, + const ConstantRange &CR) { if (CR.isEmptySet()) return CR; uint32_t W = CR.getBitWidth(); switch (Pred) { - default: llvm_unreachable("Invalid ICmp predicate to makeICmpRegion()"); + default: + llvm_unreachable("Invalid ICmp predicate to makeAllowedICmpRegion()"); case CmpInst::ICMP_EQ: return CR; case CmpInst::ICMP_NE: @@ -114,6 +115,16 @@ ConstantRange ConstantRange::makeICmpRegion(unsigned Pred, } } +ConstantRange ConstantRange::makeSatisfyingICmpRegion(CmpInst::Predicate Pred, + const ConstantRange &CR) { + // Follows from De-Morgan's laws: + // + // ~(~A union ~B) == A intersect B. + // + return makeAllowedICmpRegion(CmpInst::getInversePredicate(Pred), CR) + .inverse(); +} + /// isFullSet - Return true if this set contains all of the elements possible /// for this data-type bool ConstantRange::isFullSet() const { @@ -587,6 +598,13 @@ ConstantRange::multiply(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) return ConstantRange(getBitWidth(), /*isFullSet=*/false); + // Multiplication is signedness-independent. However different ranges can be + // obtained depending on how the input ranges are treated. These different + // ranges are all conservatively correct, but one might be better than the + // other. We calculate two ranges; one treating the inputs as unsigned + // and the other signed, then return the smallest of these ranges. + + // Unsigned range first. APInt this_min = getUnsignedMin().zext(getBitWidth() * 2); APInt this_max = getUnsignedMax().zext(getBitWidth() * 2); APInt Other_min = Other.getUnsignedMin().zext(getBitWidth() * 2); @@ -594,7 +612,26 @@ ConstantRange::multiply(const ConstantRange &Other) const { ConstantRange Result_zext = ConstantRange(this_min * Other_min, this_max * Other_max + 1); - return Result_zext.truncate(getBitWidth()); + ConstantRange UR = Result_zext.truncate(getBitWidth()); + + // Now the signed range. Because we could be dealing with negative numbers + // here, the lower bound is the smallest of the cartesian product of the + // lower and upper ranges; for example: + // [-1,4) * [-2,3) = min(-1*-2, -1*2, 3*-2, 3*2) = -6. + // Similarly for the upper bound, swapping min for max. + + this_min = getSignedMin().sext(getBitWidth() * 2); + this_max = getSignedMax().sext(getBitWidth() * 2); + Other_min = Other.getSignedMin().sext(getBitWidth() * 2); + Other_max = Other.getSignedMax().sext(getBitWidth() * 2); + + auto L = {this_min * Other_min, this_min * Other_max, + this_max * Other_min, this_max * Other_max}; + auto Compare = [](const APInt &A, const APInt &B) { return A.slt(B); }; + ConstantRange Result_sext(std::min(L, Compare), std::max(L, Compare) + 1); + ConstantRange SR = Result_sext.truncate(getBitWidth()); + + return UR.getSetSize().ult(SR.getSetSize()) ? UR : SR; } ConstantRange diff --git a/lib/IR/Constants.cpp b/lib/IR/Constants.cpp index 0bf61a7..e51a396 100644 --- a/lib/IR/Constants.cpp +++ b/lib/IR/Constants.cpp @@ -1215,11 +1215,9 @@ ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const { Constant *ConstantExpr::getWithOperands(ArrayRef Ops, Type *Ty, bool OnlyIfReduced) const { assert(Ops.size() == getNumOperands() && "Operand count mismatch!"); - bool AnyChange = Ty != getType(); - for (unsigned i = 0; i != Ops.size(); ++i) - AnyChange |= Ops[i] != getOperand(i); - if (!AnyChange) // No operands changed, return self. + // If no operands changed return self. + if (Ty == getType() && std::equal(Ops.begin(), Ops.end(), op_begin())) return const_cast(this); Type *OnlyIfReducedTy = OnlyIfReduced ? Ty : nullptr; @@ -2971,10 +2969,7 @@ void ConstantExpr::replaceUsesOfWithOnConstant(Value *From, Value *ToV, } Instruction *ConstantExpr::getAsInstruction() { - SmallVector ValueOperands; - for (op_iterator I = op_begin(), E = op_end(); I != E; ++I) - ValueOperands.push_back(cast(I)); - + SmallVector ValueOperands(op_begin(), op_end()); ArrayRef Ops(ValueOperands); switch (getOpcode()) { @@ -3006,12 +3001,14 @@ Instruction *ConstantExpr::getAsInstruction() { case Instruction::ShuffleVector: return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]); - case Instruction::GetElementPtr: - if (cast(this)->isInBounds()) - return GetElementPtrInst::CreateInBounds(Ops[0], Ops.slice(1)); - else - return GetElementPtrInst::Create(Ops[0], Ops.slice(1)); - + case Instruction::GetElementPtr: { + const auto *GO = cast(this); + if (GO->isInBounds()) + return GetElementPtrInst::CreateInBounds(GO->getSourceElementType(), + Ops[0], Ops.slice(1)); + return GetElementPtrInst::Create(GO->getSourceElementType(), Ops[0], + Ops.slice(1)); + } case Instruction::ICmp: case Instruction::FCmp: return CmpInst::Create((Instruction::OtherOps)getOpcode(), diff --git a/lib/IR/Core.cpp b/lib/IR/Core.cpp index f007616..613147e 100644 --- a/lib/IR/Core.cpp +++ b/lib/IR/Core.cpp @@ -2506,7 +2506,7 @@ LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer, LLVMValueRef *Indices, unsigned NumIndices, const char *Name) { ArrayRef IdxList(unwrap(Indices), NumIndices); - return wrap(unwrap(B)->CreateGEP(unwrap(Pointer), IdxList, Name)); + return wrap(unwrap(B)->CreateGEP(nullptr, unwrap(Pointer), IdxList, Name)); } LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer, diff --git a/lib/IR/DIBuilder.cpp b/lib/IR/DIBuilder.cpp index 2cb27ca..9677de4 100644 --- a/lib/IR/DIBuilder.cpp +++ b/lib/IR/DIBuilder.cpp @@ -121,18 +121,10 @@ void DIBuilder::finalize() { } /// If N is compile unit return NULL otherwise return N. -static MDNode *getNonCompileUnitScope(MDNode *N) { - if (DIDescriptor(N).isCompileUnit()) +static MDScope *getNonCompileUnitScope(MDNode *N) { + if (!N || isa(N)) return nullptr; - return N; -} - -static MDNode *createFilePathPair(LLVMContext &VMContext, StringRef Filename, - StringRef Directory) { - assert(!Filename.empty() && "Unable to create file without name"); - Metadata *Pair[] = {MDString::get(VMContext, Filename), - MDString::get(VMContext, Directory)}; - return MDNode::get(VMContext, Pair); + return cast(N); } DICompileUnit DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename, @@ -157,22 +149,12 @@ DICompileUnit DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename, TempGVs = MDTuple::getTemporary(VMContext, None).release(); TempImportedModules = MDTuple::getTemporary(VMContext, None).release(); - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_compile_unit) - .concat(Lang) - .concat(Producer) - .concat(isOptimized) - .concat(Flags) - .concat(RunTimeVer) - .concat(SplitName) - .concat(Kind) - .get(VMContext), - createFilePathPair(VMContext, Filename, Directory), - TempEnumTypes, TempRetainTypes, TempSubprograms, TempGVs, - TempImportedModules}; - // TODO: Switch to getDistinct(). We never want to merge compile units based // on contents. - MDNode *CUNode = MDNode::get(VMContext, Elts); + MDNode *CUNode = MDCompileUnit::get( + VMContext, Lang, MDFile::get(VMContext, Filename, Directory), Producer, + isOptimized, Flags, RunTimeVer, SplitName, Kind, TempEnumTypes, + TempRetainTypes, TempSubprograms, TempGVs, TempImportedModules); // Create a named metadata so that it is easier to find cu in a module. // Note that we only generate this when the caller wants to actually @@ -192,11 +174,7 @@ static DIImportedEntity createImportedModule(LLVMContext &C, dwarf::Tag Tag, DIScope Context, Metadata *NS, unsigned Line, StringRef Name, SmallVectorImpl &AllImportedModules) { - const MDNode *R; - Metadata *Elts[] = {HeaderBuilder::get(Tag).concat(Line).concat(Name).get(C), - Context, NS}; - R = MDNode::get(C, Elts); - DIImportedEntity M(R); + DIImportedEntity M = MDImportedEntity::get(C, Tag, Context, NS, Line, Name); assert(M.Verify() && "Imported module should be valid"); AllImportedModules.emplace_back(M.get()); return M; @@ -236,39 +214,17 @@ DIImportedEntity DIBuilder::createImportedDeclaration(DIScope Context, } DIFile DIBuilder::createFile(StringRef Filename, StringRef Directory) { - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_file_type).get(VMContext), - createFilePathPair(VMContext, Filename, Directory)}; - return DIFile(MDNode::get(VMContext, Elts)); + return MDFile::get(VMContext, Filename, Directory); } DIEnumerator DIBuilder::createEnumerator(StringRef Name, int64_t Val) { assert(!Name.empty() && "Unable to create enumerator without name"); - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_enumerator) - .concat(Name) - .concat(Val) - .get(VMContext)}; - return DIEnumerator(MDNode::get(VMContext, Elts)); + return MDEnumerator::get(VMContext, Val, Name); } DIBasicType DIBuilder::createUnspecifiedType(StringRef Name) { assert(!Name.empty() && "Unable to create type without name"); - // Unspecified types are encoded in DIBasicType format. Line number, filename, - // size, alignment, offset and flags are always empty here. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_unspecified_type) - .concat(Name) - .concat(0) - .concat(0) - .concat(0) - .concat(0) - .concat(0) - .concat(0) - .get(VMContext), - nullptr, // Filename - nullptr // Unused - }; - return DIBasicType(MDNode::get(VMContext, Elts)); + return MDBasicType::get(VMContext, dwarf::DW_TAG_unspecified_type, Name); } DIBasicType DIBuilder::createNullPtrType() { @@ -279,142 +235,61 @@ DIBasicType DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits, uint64_t AlignInBits, unsigned Encoding) { assert(!Name.empty() && "Unable to create type without name"); - // Basic types are encoded in DIBasicType format. Line number, filename, - // offset and flags are always empty here. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_base_type) - .concat(Name) - .concat(0) // Line - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) // Offset - .concat(0) // Flags - .concat(Encoding) - .get(VMContext), - nullptr, // Filename - nullptr // Unused - }; - return DIBasicType(MDNode::get(VMContext, Elts)); + return MDBasicType::get(VMContext, dwarf::DW_TAG_base_type, Name, SizeInBits, + AlignInBits, Encoding); } DIDerivedType DIBuilder::createQualifiedType(unsigned Tag, DIType FromTy) { - // Qualified types are encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(Tag) - .concat(StringRef()) // Name - .concat(0) // Line - .concat(0) // Size - .concat(0) // Align - .concat(0) // Offset - .concat(0) // Flags - .get(VMContext), - nullptr, // Filename - nullptr, // Unused - FromTy.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, + FromTy.getRef(), 0, 0, 0, 0); } DIDerivedType DIBuilder::createPointerType(DIType PointeeTy, uint64_t SizeInBits, uint64_t AlignInBits, StringRef Name) { - // Pointer types are encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_pointer_type) - .concat(Name) - .concat(0) // Line - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) // Offset - .concat(0) // Flags - .get(VMContext), - nullptr, // Filename - nullptr, // Unused - PointeeTy.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + // FIXME: Why is there a name here? + return MDDerivedType::get(VMContext, dwarf::DW_TAG_pointer_type, Name, + nullptr, 0, nullptr, PointeeTy.getRef(), SizeInBits, + AlignInBits, 0, 0); } DIDerivedType DIBuilder::createMemberPointerType(DIType PointeeTy, DIType Base, uint64_t SizeInBits, uint64_t AlignInBits) { - // Pointer types are encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_ptr_to_member_type) - .concat(StringRef()) - .concat(0) // Line - .concat(SizeInBits) // Size - .concat(AlignInBits) // Align - .concat(0) // Offset - .concat(0) // Flags - .get(VMContext), - nullptr, // Filename - nullptr, // Unused - PointeeTy.getRef(), Base.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get(VMContext, dwarf::DW_TAG_ptr_to_member_type, "", + nullptr, 0, nullptr, PointeeTy.getRef(), SizeInBits, + AlignInBits, 0, 0, Base.getRef()); } DIDerivedType DIBuilder::createReferenceType(unsigned Tag, DIType RTy) { assert(RTy.isType() && "Unable to create reference type"); - // References are encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(Tag) - .concat(StringRef()) // Name - .concat(0) // Line - .concat(0) // Size - .concat(0) // Align - .concat(0) // Offset - .concat(0) // Flags - .get(VMContext), - nullptr, // Filename - nullptr, // TheCU, - RTy.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, + RTy.getRef(), 0, 0, 0, 0); } DIDerivedType DIBuilder::createTypedef(DIType Ty, StringRef Name, DIFile File, unsigned LineNo, DIDescriptor Context) { - // typedefs are encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_typedef) - .concat(Name) - .concat(LineNo) - .concat(0) // Size - .concat(0) // Align - .concat(0) // Offset - .concat(0) // Flags - .get(VMContext), - File.getFileNode(), - DIScope(getNonCompileUnitScope(Context)).getRef(), - Ty.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, + File.getFileNode(), LineNo, + DIScope(getNonCompileUnitScope(Context)).getRef(), + Ty.getRef(), 0, 0, 0, 0); } DIDerivedType DIBuilder::createFriend(DIType Ty, DIType FriendTy) { // typedefs are encoded in DIDerivedType format. assert(Ty.isType() && "Invalid type!"); assert(FriendTy.isType() && "Invalid friend type!"); - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_friend) - .concat(StringRef()) // Name - .concat(0) // Line - .concat(0) // Size - .concat(0) // Align - .concat(0) // Offset - .concat(0) // Flags - .get(VMContext), - nullptr, Ty.getRef(), FriendTy.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0, + Ty.getRef(), FriendTy.getRef(), 0, 0, 0, 0); } DIDerivedType DIBuilder::createInheritance(DIType Ty, DIType BaseTy, uint64_t BaseOffset, unsigned Flags) { assert(Ty.isType() && "Unable to create inheritance"); - // TAG_inheritance is encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_inheritance) - .concat(StringRef()) // Name - .concat(0) // Line - .concat(0) // Size - .concat(0) // Align - .concat(BaseOffset) - .concat(Flags) - .get(VMContext), - nullptr, Ty.getRef(), BaseTy.getRef()}; - auto R = DIDerivedType(MDNode::get(VMContext, Elts)); - return R; + return MDDerivedType::get(VMContext, dwarf::DW_TAG_inheritance, "", nullptr, + 0, Ty.getRef(), BaseTy.getRef(), 0, 0, BaseOffset, + Flags); } DIDerivedType DIBuilder::createMemberType(DIDescriptor Scope, StringRef Name, @@ -423,22 +298,13 @@ DIDerivedType DIBuilder::createMemberType(DIDescriptor Scope, StringRef Name, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, DIType Ty) { - // TAG_member is encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_member) - .concat(Name) - .concat(LineNumber) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(OffsetInBits) - .concat(Flags) - .get(VMContext), - File.getFileNode(), - DIScope(getNonCompileUnitScope(Scope)).getRef(), - Ty.getRef()}; - return DIDerivedType(MDNode::get(VMContext, Elts)); -} - -static Metadata *getConstantOrNull(Constant *C) { + return MDDerivedType::get( + VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, + DIScope(getNonCompileUnitScope(Scope)).getRef(), Ty.getRef(), SizeInBits, + AlignInBits, OffsetInBits, Flags); +} + +static ConstantAsMetadata *getConstantOrNull(Constant *C) { if (C) return ConstantAsMetadata::get(C); return nullptr; @@ -451,18 +317,10 @@ DIDerivedType DIBuilder::createStaticMemberType(DIDescriptor Scope, llvm::Constant *Val) { // TAG_member is encoded in DIDerivedType format. Flags |= DIDescriptor::FlagStaticMember; - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_member) - .concat(Name) - .concat(LineNumber) - .concat(0) // Size - .concat(0) // Align - .concat(0) // Offset - .concat(Flags) - .get(VMContext), - File.getFileNode(), - DIScope(getNonCompileUnitScope(Scope)).getRef(), - Ty.getRef(), getConstantOrNull(Val)}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get( + VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, + DIScope(getNonCompileUnitScope(Scope)).getRef(), Ty.getRef(), 0, 0, 0, + Flags, getConstantOrNull(Val)); } DIDerivedType DIBuilder::createObjCIVar(StringRef Name, DIFile File, @@ -471,33 +329,18 @@ DIDerivedType DIBuilder::createObjCIVar(StringRef Name, DIFile File, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, DIType Ty, MDNode *PropertyNode) { - // TAG_member is encoded in DIDerivedType format. - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_member) - .concat(Name) - .concat(LineNumber) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(OffsetInBits) - .concat(Flags) - .get(VMContext), - File.getFileNode(), getNonCompileUnitScope(File), Ty, - PropertyNode}; - return DIDerivedType(MDNode::get(VMContext, Elts)); + return MDDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, + LineNumber, getNonCompileUnitScope(File), + Ty.getRef(), SizeInBits, AlignInBits, OffsetInBits, + Flags, PropertyNode); } DIObjCProperty DIBuilder::createObjCProperty(StringRef Name, DIFile File, unsigned LineNumber, StringRef GetterName, StringRef SetterName, unsigned PropertyAttributes, DIType Ty) { - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_APPLE_property) - .concat(Name) - .concat(LineNumber) - .concat(GetterName) - .concat(SetterName) - .concat(PropertyAttributes) - .get(VMContext), - File, Ty}; - return DIObjCProperty(MDNode::get(VMContext, Elts)); + return MDObjCProperty::get(VMContext, Name, File, LineNumber, GetterName, + SetterName, PropertyAttributes, Ty); } DITemplateTypeParameter @@ -505,13 +348,7 @@ DIBuilder::createTemplateTypeParameter(DIDescriptor Context, StringRef Name, DIType Ty) { assert(!DIScope(getNonCompileUnitScope(Context)).getRef() && "Expected compile unit"); - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_template_type_parameter) - .concat(Name) - .concat(0) - .concat(0) - .get(VMContext), - nullptr, Ty.getRef(), nullptr}; - return DITemplateTypeParameter(MDNode::get(VMContext, Elts)); + return MDTemplateTypeParameter::get(VMContext, Name, Ty.getRef()); } static DITemplateValueParameter @@ -520,10 +357,7 @@ createTemplateValueParameterHelper(LLVMContext &VMContext, unsigned Tag, DIType Ty, Metadata *MD) { assert(!DIScope(getNonCompileUnitScope(Context)).getRef() && "Expected compile unit"); - Metadata *Elts[] = { - HeaderBuilder::get(Tag).concat(Name).concat(0).concat(0).get(VMContext), - nullptr, Ty.getRef(), MD, nullptr}; - return DITemplateValueParameter(MDNode::get(VMContext, Elts)); + return MDTemplateValueParameter::get(VMContext, Tag, Name, Ty.getRef(), MD); } DITemplateValueParameter @@ -563,23 +397,11 @@ DICompositeType DIBuilder::createClassType(DIDescriptor Context, StringRef Name, assert((!Context || Context.isScope() || Context.isType()) && "createClassType should be called with a valid Context"); // TAG_class_type is encoded in DICompositeType format. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_class_type) - .concat(Name) - .concat(LineNumber) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(OffsetInBits) - .concat(Flags) - .concat(0) - .get(VMContext), - File.getFileNode(), DIScope(getNonCompileUnitScope(Context)).getRef(), - DerivedFrom.getRef(), Elements, VTableHolder.getRef(), TemplateParams, - UniqueIdentifier.empty() ? nullptr - : MDString::get(VMContext, UniqueIdentifier)}; - DICompositeType R(MDNode::get(VMContext, Elts)); - assert(R.isCompositeType() && - "createClassType should return a DICompositeType"); + DICompositeType R = MDCompositeType::get( + VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber, + DIScope(getNonCompileUnitScope(Context)).getRef(), DerivedFrom.getRef(), + SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, 0, + VTableHolder.getRef(), TemplateParams, UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(R); trackIfUnresolved(R); @@ -596,24 +418,11 @@ DICompositeType DIBuilder::createStructType(DIDescriptor Context, unsigned RunTimeLang, DIType VTableHolder, StringRef UniqueIdentifier) { - // TAG_structure_type is encoded in DICompositeType format. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_structure_type) - .concat(Name) - .concat(LineNumber) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) - .concat(Flags) - .concat(RunTimeLang) - .get(VMContext), - File.getFileNode(), DIScope(getNonCompileUnitScope(Context)).getRef(), - DerivedFrom.getRef(), Elements, VTableHolder.getRef(), nullptr, - UniqueIdentifier.empty() ? nullptr - : MDString::get(VMContext, UniqueIdentifier)}; - DICompositeType R(MDNode::get(VMContext, Elts)); - assert(R.isCompositeType() && - "createStructType should return a DICompositeType"); + DICompositeType R = MDCompositeType::get( + VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber, + DIScope(getNonCompileUnitScope(Context)).getRef(), DerivedFrom.getRef(), + SizeInBits, AlignInBits, 0, Flags, Elements, RunTimeLang, + VTableHolder.getRef(), nullptr, UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(R); trackIfUnresolved(R); @@ -627,22 +436,11 @@ DICompositeType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name, DIArray Elements, unsigned RunTimeLang, StringRef UniqueIdentifier) { - // TAG_union_type is encoded in DICompositeType format. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_union_type) - .concat(Name) - .concat(LineNumber) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) // Offset - .concat(Flags) - .concat(RunTimeLang) - .get(VMContext), - File.getFileNode(), DIScope(getNonCompileUnitScope(Scope)).getRef(), - nullptr, Elements, nullptr, nullptr, - UniqueIdentifier.empty() ? nullptr - : MDString::get(VMContext, UniqueIdentifier)}; - DICompositeType R(MDNode::get(VMContext, Elts)); + DICompositeType R = MDCompositeType::get( + VMContext, dwarf::DW_TAG_union_type, Name, File, LineNumber, + DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, SizeInBits, + AlignInBits, 0, Flags, Elements, RunTimeLang, nullptr, nullptr, + UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(R); trackIfUnresolved(R); @@ -652,43 +450,18 @@ DICompositeType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name, DISubroutineType DIBuilder::createSubroutineType(DIFile File, DITypeArray ParameterTypes, unsigned Flags) { - // TAG_subroutine_type is encoded in DICompositeType format. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_subroutine_type) - .concat(StringRef()) - .concat(0) // Line - .concat(0) // Size - .concat(0) // Align - .concat(0) // Offset - .concat(Flags) // Flags - .concat(0) - .get(VMContext), - nullptr, nullptr, nullptr, ParameterTypes, nullptr, nullptr, - nullptr // Type Identifer - }; - return DISubroutineType(MDNode::get(VMContext, Elts)); + return MDSubroutineType::get(VMContext, Flags, ParameterTypes); } DICompositeType DIBuilder::createEnumerationType( DIDescriptor Scope, StringRef Name, DIFile File, unsigned LineNumber, uint64_t SizeInBits, uint64_t AlignInBits, DIArray Elements, DIType UnderlyingType, StringRef UniqueIdentifier) { - // TAG_enumeration_type is encoded in DICompositeType format. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_enumeration_type) - .concat(Name) - .concat(LineNumber) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) // Offset - .concat(0) // Flags - .concat(0) - .get(VMContext), - File.getFileNode(), DIScope(getNonCompileUnitScope(Scope)).getRef(), - UnderlyingType.getRef(), Elements, nullptr, nullptr, - UniqueIdentifier.empty() ? nullptr - : MDString::get(VMContext, UniqueIdentifier)}; - DICompositeType CTy(MDNode::get(VMContext, Elts)); + DICompositeType CTy = MDCompositeType::get( + VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber, + DIScope(getNonCompileUnitScope(Scope)).getRef(), UnderlyingType.getRef(), + SizeInBits, AlignInBits, 0, 0, Elements, 0, nullptr, nullptr, + UniqueIdentifier); AllEnumTypes.push_back(CTy); if (!UniqueIdentifier.empty()) retainType(CTy); @@ -698,85 +471,38 @@ DICompositeType DIBuilder::createEnumerationType( DICompositeType DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, DIType Ty, DIArray Subscripts) { - // TAG_array_type is encoded in DICompositeType format. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_array_type) - .concat(StringRef()) - .concat(0) // Line - .concat(Size) - .concat(AlignInBits) - .concat(0) // Offset - .concat(0) // Flags - .concat(0) - .get(VMContext), - nullptr, // Filename/Directory, - nullptr, // Unused - Ty.getRef(), Subscripts, nullptr, nullptr, - nullptr // Type Identifer - }; - DICompositeType R(MDNode::get(VMContext, Elts)); + auto *R = MDCompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", + nullptr, 0, nullptr, Ty.getRef(), Size, + AlignInBits, 0, 0, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } DICompositeType DIBuilder::createVectorType(uint64_t Size, uint64_t AlignInBits, DIType Ty, DIArray Subscripts) { - // A vector is an array type with the FlagVector flag applied. - Metadata *Elts[] = { - HeaderBuilder::get(dwarf::DW_TAG_array_type) - .concat("") - .concat(0) // Line - .concat(Size) - .concat(AlignInBits) - .concat(0) // Offset - .concat(DIType::FlagVector) - .concat(0) - .get(VMContext), - nullptr, // Filename/Directory, - nullptr, // Unused - Ty.getRef(), Subscripts, nullptr, nullptr, - nullptr // Type Identifer - }; - DICompositeType R(MDNode::get(VMContext, Elts)); + auto *R = MDCompositeType::get( + VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, nullptr, Ty.getRef(), + Size, AlignInBits, 0, DIType::FlagVector, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } -static HeaderBuilder setTypeFlagsInHeader(StringRef Header, - unsigned FlagsToSet) { - DIHeaderFieldIterator I(Header); - std::advance(I, 6); - - unsigned Flags; - if (I->getAsInteger(0, Flags)) - Flags = 0; - Flags |= FlagsToSet; - - return HeaderBuilder() - .concat(I.getPrefix()) - .concat(Flags) - .concat(I.getSuffix()); -} - static DIType createTypeWithFlags(LLVMContext &Context, DIType Ty, unsigned FlagsToSet) { - SmallVector Elts; - MDNode *N = Ty; - assert(N && "Unexpected input DIType!"); - // Update header field. - Elts.push_back(setTypeFlagsInHeader(Ty.getHeader(), FlagsToSet).get(Context)); - Elts.append(N->op_begin() + 1, N->op_end()); - - return DIType(MDNode::get(Context, Elts)); + TempMDType NewTy = cast(static_cast(Ty))->clone(); + NewTy->setFlags(NewTy->getFlags() | FlagsToSet); + return MDNode::replaceWithUniqued(std::move(NewTy)); } DIType DIBuilder::createArtificialType(DIType Ty) { + // FIXME: Restrict this to the nodes where it's valid. if (Ty.isArtificial()) return Ty; return createTypeWithFlags(VMContext, Ty, DIType::FlagArtificial); } DIType DIBuilder::createObjectPointerType(DIType Ty) { + // FIXME: Restrict this to the nodes where it's valid. if (Ty.isObjectPointer()) return Ty; unsigned Flags = DIType::FlagObjectPointer | DIType::FlagArtificial; @@ -794,26 +520,13 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIDescriptor Scope, DIFile F, unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, StringRef UniqueIdentifier) { - // Create a temporary MDNode. - Metadata *Elts[] = { - HeaderBuilder::get(Tag) - .concat(Name) - .concat(Line) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) // Offset - .concat(DIDescriptor::FlagFwdDecl) - .concat(RuntimeLang) - .get(VMContext), - F.getFileNode(), DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, - DIArray(), nullptr, - nullptr, // TemplateParams - UniqueIdentifier.empty() ? nullptr - : MDString::get(VMContext, UniqueIdentifier)}; - MDNode *Node = MDNode::get(VMContext, Elts); - DICompositeType RetTy(Node); - assert(RetTy.isCompositeType() && - "createForwardDecl result should be a DIType"); + // FIXME: Define in terms of createReplaceableForwardDecl() by calling + // replaceWithUniqued(). + DICompositeType RetTy = MDCompositeType::get( + VMContext, Tag, Name, F.getFileNode(), Line, + DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, SizeInBits, + AlignInBits, 0, DIDescriptor::FlagFwdDecl, nullptr, RuntimeLang, nullptr, + nullptr, UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(RetTy); trackIfUnresolved(RetTy); @@ -824,25 +537,12 @@ DICompositeType DIBuilder::createReplaceableCompositeType( unsigned Tag, StringRef Name, DIDescriptor Scope, DIFile F, unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, StringRef UniqueIdentifier) { - // Create a temporary MDNode. - Metadata *Elts[] = { - HeaderBuilder::get(Tag) - .concat(Name) - .concat(Line) - .concat(SizeInBits) - .concat(AlignInBits) - .concat(0) // Offset - .concat(Flags) - .concat(RuntimeLang) - .get(VMContext), - F.getFileNode(), DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, - DIArray(), nullptr, - nullptr, // TemplateParams - UniqueIdentifier.empty() ? nullptr - : MDString::get(VMContext, UniqueIdentifier)}; - DICompositeType RetTy(MDNode::getTemporary(VMContext, Elts).release()); - assert(RetTy.isCompositeType() && - "createReplaceableForwardDecl result should be a DIType"); + DICompositeType RetTy = + MDCompositeType::getTemporary( + VMContext, Tag, Name, F.getFileNode(), Line, + DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, SizeInBits, + AlignInBits, 0, Flags, nullptr, RuntimeLang, + nullptr, nullptr, UniqueIdentifier).release(); if (!UniqueIdentifier.empty()) retainType(RetTy); trackIfUnresolved(RetTy); @@ -865,62 +565,39 @@ DITypeArray DIBuilder::getOrCreateTypeArray(ArrayRef Elements) { } DISubrange DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Count) { - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_subrange_type) - .concat(Lo) - .concat(Count) - .get(VMContext)}; - - return DISubrange(MDNode::get(VMContext, Elts)); + return MDSubrange::get(VMContext, Count, Lo); } -static DIGlobalVariable createGlobalVariableHelper( - LLVMContext &VMContext, DIDescriptor Context, StringRef Name, - StringRef LinkageName, DIFile F, unsigned LineNumber, DITypeRef Ty, - bool isLocalToUnit, Constant *Val, MDNode *Decl, bool isDefinition, - std::function)> CreateFunc) { - +static void checkGlobalVariableScope(DIDescriptor Context) { MDNode *TheCtx = getNonCompileUnitScope(Context); if (DIScope(TheCtx).isCompositeType()) { assert(!DICompositeType(TheCtx).getIdentifier() && "Context of a global variable should not be a type with identifier"); } - - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_variable) - .concat(Name) - .concat(Name) - .concat(LinkageName) - .concat(LineNumber) - .concat(isLocalToUnit) - .concat(isDefinition) - .get(VMContext), - TheCtx, F, Ty, getConstantOrNull(Val), - DIDescriptor(Decl)}; - - return DIGlobalVariable(CreateFunc(Elts)); } DIGlobalVariable DIBuilder::createGlobalVariable( DIDescriptor Context, StringRef Name, StringRef LinkageName, DIFile F, unsigned LineNumber, DITypeRef Ty, bool isLocalToUnit, Constant *Val, MDNode *Decl) { - return createGlobalVariableHelper( - VMContext, Context, Name, LinkageName, F, LineNumber, Ty, isLocalToUnit, - Val, Decl, true, [&](ArrayRef Elts) -> MDNode *{ - MDNode *Node = MDNode::get(VMContext, Elts); - AllGVs.push_back(Node); - return Node; - }); + checkGlobalVariableScope(Context); + + auto *N = MDGlobalVariable::get(VMContext, Context, Name, LinkageName, F, + LineNumber, Ty, isLocalToUnit, true, + getConstantOrNull(Val), Decl); + AllGVs.push_back(N); + return N; } DIGlobalVariable DIBuilder::createTempGlobalVariableFwdDecl( DIDescriptor Context, StringRef Name, StringRef LinkageName, DIFile F, unsigned LineNumber, DITypeRef Ty, bool isLocalToUnit, Constant *Val, MDNode *Decl) { - return createGlobalVariableHelper(VMContext, Context, Name, LinkageName, F, - LineNumber, Ty, isLocalToUnit, Val, Decl, - false, [&](ArrayRef Elts) { - return MDNode::getTemporary(VMContext, Elts).release(); - }); + checkGlobalVariableScope(Context); + + return MDGlobalVariable::getTemporary(VMContext, Context, Name, LinkageName, + F, LineNumber, Ty, isLocalToUnit, false, + getConstantOrNull(Val), Decl).release(); } DIVariable DIBuilder::createLocalVariable(unsigned Tag, DIDescriptor Scope, @@ -928,16 +605,17 @@ DIVariable DIBuilder::createLocalVariable(unsigned Tag, DIDescriptor Scope, unsigned LineNo, DITypeRef Ty, bool AlwaysPreserve, unsigned Flags, unsigned ArgNo) { + // FIXME: Why getNonCompileUnitScope()? + // FIXME: Why is "!Context" okay here? + // FIXME: WHy doesn't this check for a subprogram or lexical block (AFAICT + // the only valid scopes)? DIDescriptor Context(getNonCompileUnitScope(Scope)); assert((!Context || Context.isScope()) && "createLocalVariable should be called with a valid Context"); - Metadata *Elts[] = {HeaderBuilder::get(Tag) - .concat(Name) - .concat(LineNo | (ArgNo << 24)) - .concat(Flags) - .get(VMContext), - getNonCompileUnitScope(Scope), File, Ty}; - MDNode *Node = MDNode::get(VMContext, Elts); + + auto *Node = + MDLocalVariable::get(VMContext, Tag, getNonCompileUnitScope(Scope), Name, + File, LineNo, Ty, ArgNo, Flags); if (AlwaysPreserve) { // The optimizer may remove local variable. If there is an interest // to preserve variable info in such situation then stash it in a @@ -946,18 +624,11 @@ DIVariable DIBuilder::createLocalVariable(unsigned Tag, DIDescriptor Scope, assert(Fn && "Missing subprogram for local variable"); PreservedVariables[Fn].emplace_back(Node); } - DIVariable RetVar(Node); - assert(RetVar.isVariable() && - "createLocalVariable should return a valid DIVariable"); - return RetVar; + return Node; } DIExpression DIBuilder::createExpression(ArrayRef Addr) { - auto Header = HeaderBuilder::get(DW_TAG_expression); - for (uint64_t I : Addr) - Header.concat(I); - Metadata *Elts[] = {Header.get(VMContext)}; - return DIExpression(MDNode::get(VMContext, Elts)); + return MDExpression::get(VMContext, Addr); } DIExpression DIBuilder::createExpression(ArrayRef Signed) { @@ -966,10 +637,10 @@ DIExpression DIBuilder::createExpression(ArrayRef Signed) { return createExpression(Addr); } -DIExpression DIBuilder::createBitPieceExpression(unsigned OffsetInBits, - unsigned SizeInBits) { - int64_t Addr[] = {dwarf::DW_OP_bit_piece, OffsetInBits, SizeInBits}; - return createExpression(Addr); +DIExpression DIBuilder::createBitPieceExpression(unsigned OffsetInBytes, + unsigned SizeInBytes) { + uint64_t Addr[] = {dwarf::DW_OP_bit_piece, OffsetInBytes, SizeInBytes}; + return MDExpression::get(VMContext, Addr); } DISubprogram DIBuilder::createFunction(DIScopeRef Context, StringRef Name, @@ -987,38 +658,6 @@ DISubprogram DIBuilder::createFunction(DIScopeRef Context, StringRef Name, Flags, isOptimized, Fn, TParams, Decl); } -static DISubprogram createFunctionHelper( - LLVMContext &VMContext, DIDescriptor Context, StringRef Name, - StringRef LinkageName, DIFile File, unsigned LineNo, DICompositeType Ty, - bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, unsigned Flags, - bool isOptimized, Function *Fn, MDNode *TParams, MDNode *Decl, MDNode *Vars, - std::function)> CreateFunc) { - assert(Ty.getTag() == dwarf::DW_TAG_subroutine_type && - "function types should be subroutines"); - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_subprogram) - .concat(Name) - .concat(Name) - .concat(LinkageName) - .concat(LineNo) - .concat(isLocalToUnit) - .concat(isDefinition) - .concat(0) - .concat(0) - .concat(Flags) - .concat(isOptimized) - .concat(ScopeLine) - .get(VMContext), - File.getFileNode(), - DIScope(getNonCompileUnitScope(Context)).getRef(), Ty, - nullptr, getConstantOrNull(Fn), TParams, Decl, Vars}; - - DISubprogram S(CreateFunc(Elts)); - assert(S.isSubprogram() && - "createFunction should return a valid DISubprogram"); - return S; -} - - DISubprogram DIBuilder::createFunction(DIDescriptor Context, StringRef Name, StringRef LinkageName, DIFile File, unsigned LineNo, DICompositeType Ty, @@ -1026,19 +665,18 @@ DISubprogram DIBuilder::createFunction(DIDescriptor Context, StringRef Name, unsigned ScopeLine, unsigned Flags, bool isOptimized, Function *Fn, MDNode *TParams, MDNode *Decl) { - return createFunctionHelper(VMContext, Context, Name, LinkageName, File, - LineNo, Ty, isLocalToUnit, isDefinition, - ScopeLine, Flags, isOptimized, Fn, TParams, Decl, - MDNode::getTemporary(VMContext, None).release(), - [&](ArrayRef Elts) -> MDNode *{ - MDNode *Node = MDNode::get(VMContext, Elts); - // Create a named metadata so that we - // do not lose this mdnode. - if (isDefinition) - AllSubprograms.push_back(Node); - trackIfUnresolved(Node); - return Node; - }); + assert(Ty.getTag() == dwarf::DW_TAG_subroutine_type && + "function types should be subroutines"); + auto *Node = MDSubprogram::get( + VMContext, DIScope(getNonCompileUnitScope(Context)).getRef(), Name, + LinkageName, File.getFileNode(), LineNo, Ty, isLocalToUnit, isDefinition, + ScopeLine, nullptr, 0, 0, Flags, isOptimized, getConstantOrNull(Fn), + TParams, Decl, MDNode::getTemporary(VMContext, None).release()); + + if (isDefinition) + AllSubprograms.push_back(Node); + trackIfUnresolved(Node); + return Node; } DISubprogram @@ -1049,12 +687,11 @@ DIBuilder::createTempFunctionFwdDecl(DIDescriptor Context, StringRef Name, unsigned ScopeLine, unsigned Flags, bool isOptimized, Function *Fn, MDNode *TParams, MDNode *Decl) { - return createFunctionHelper(VMContext, Context, Name, LinkageName, File, - LineNo, Ty, isLocalToUnit, isDefinition, - ScopeLine, Flags, isOptimized, Fn, TParams, Decl, - nullptr, [&](ArrayRef Elts) { - return MDNode::getTemporary(VMContext, Elts).release(); - }); + return MDSubprogram::getTemporary( + VMContext, DIScope(getNonCompileUnitScope(Context)).getRef(), Name, + LinkageName, File.getFileNode(), LineNo, Ty, isLocalToUnit, + isDefinition, ScopeLine, nullptr, 0, 0, Flags, isOptimized, + getConstantOrNull(Fn), TParams, Decl, nullptr).release(); } DISubprogram DIBuilder::createMethod(DIDescriptor Context, StringRef Name, @@ -1070,24 +707,13 @@ DISubprogram DIBuilder::createMethod(DIDescriptor Context, StringRef Name, assert(getNonCompileUnitScope(Context) && "Methods should have both a Context and a context that isn't " "the compile unit."); - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_subprogram) - .concat(Name) - .concat(Name) - .concat(LinkageName) - .concat(LineNo) - .concat(isLocalToUnit) - .concat(isDefinition) - .concat(VK) - .concat(VIndex) - .concat(Flags) - .concat(isOptimized) - .concat(LineNo) - // FIXME: Do we want to use different scope/lines? - .get(VMContext), - F.getFileNode(), DIScope(Context).getRef(), Ty, - VTableHolder.getRef(), getConstantOrNull(Fn), TParam, - nullptr, nullptr}; - MDNode *Node = MDNode::get(VMContext, Elts); + // FIXME: Do we want to use different scope/lines? + auto *Node = MDSubprogram::get( + VMContext, DIScope(Context).getRef(), Name, LinkageName, F.getFileNode(), + LineNo, Ty, isLocalToUnit, isDefinition, LineNo, VTableHolder.getRef(), + VK, VIndex, Flags, isOptimized, getConstantOrNull(Fn), TParam, nullptr, + nullptr); + if (isDefinition) AllSubprograms.push_back(Node); DISubprogram S(Node); @@ -1098,12 +724,8 @@ DISubprogram DIBuilder::createMethod(DIDescriptor Context, StringRef Name, DINameSpace DIBuilder::createNameSpace(DIDescriptor Scope, StringRef Name, DIFile File, unsigned LineNo) { - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_namespace) - .concat(Name) - .concat(LineNo) - .get(VMContext), - File.getFileNode(), getNonCompileUnitScope(Scope)}; - DINameSpace R(MDNode::get(VMContext, Elts)); + DINameSpace R = MDNamespace::get(VMContext, getNonCompileUnitScope(Scope), + File.getFileNode(), Name, LineNo); assert(R.Verify() && "createNameSpace should return a verifiable DINameSpace"); return R; @@ -1112,11 +734,8 @@ DINameSpace DIBuilder::createNameSpace(DIDescriptor Scope, StringRef Name, DILexicalBlockFile DIBuilder::createLexicalBlockFile(DIDescriptor Scope, DIFile File, unsigned Discriminator) { - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_lexical_block) - .concat(Discriminator) - .get(VMContext), - File.getFileNode(), Scope}; - DILexicalBlockFile R(MDNode::get(VMContext, Elts)); + DILexicalBlockFile R = MDLexicalBlockFile::get( + VMContext, Scope, File.getFileNode(), Discriminator); assert( R.Verify() && "createLexicalBlockFile should return a verifiable DILexicalBlockFile"); @@ -1125,22 +744,10 @@ DILexicalBlockFile DIBuilder::createLexicalBlockFile(DIDescriptor Scope, DILexicalBlock DIBuilder::createLexicalBlock(DIDescriptor Scope, DIFile File, unsigned Line, unsigned Col) { - // FIXME: This isn't thread safe nor the right way to defeat MDNode uniquing. - // I believe the right way is to have a self-referential element in the node. - // Also: why do we bother with line/column - they're not used and the - // documentation (SourceLevelDebugging.rst) claims the line/col are necessary - // for uniquing, yet then we have this other solution (because line/col were - // inadequate) anyway. Remove all 3 and replace them with a self-reference. - - // Defeat MDNode uniquing for lexical blocks by using unique id. - static unsigned int unique_id = 0; - Metadata *Elts[] = {HeaderBuilder::get(dwarf::DW_TAG_lexical_block) - .concat(Line) - .concat(Col) - .concat(unique_id++) - .get(VMContext), - File.getFileNode(), getNonCompileUnitScope(Scope)}; - DILexicalBlock R(MDNode::get(VMContext, Elts)); + // Make these distinct, to avoid merging two lexical blocks on the same + // file/line/column. + DILexicalBlock R = MDLexicalBlock::getDistinct( + VMContext, getNonCompileUnitScope(Scope), File.getFileNode(), Line, Col); assert(R.Verify() && "createLexicalBlock should return a verifiable DILexicalBlock"); return R; diff --git a/lib/IR/DataLayout.cpp b/lib/IR/DataLayout.cpp index 9c1dee0..4d867ef 100644 --- a/lib/IR/DataLayout.cpp +++ b/lib/IR/DataLayout.cpp @@ -33,11 +33,6 @@ #include using namespace llvm; -// Handle the Pass registration stuff necessary to use DataLayout's. - -INITIALIZE_PASS(DataLayoutPass, "datalayout", "Data Layout", false, true) -char DataLayoutPass::ID = 0; - //===----------------------------------------------------------------------===// // Support for StructLayout //===----------------------------------------------------------------------===// @@ -155,8 +150,8 @@ DataLayout::InvalidPointerElem = { 0U, 0U, 0U, ~0U }; const char *DataLayout::getManglingComponent(const Triple &T) { if (T.isOSBinFormatMachO()) return "-m:o"; - if (T.isOSWindows() && T.getArch() == Triple::x86 && T.isOSBinFormatCOFF()) - return "-m:w"; + if (T.isOSWindows() && T.isOSBinFormatCOFF()) + return T.getArch() == Triple::x86 ? "-m:x" : "-m:w"; return "-m:e"; } @@ -221,6 +216,7 @@ static unsigned inBytes(unsigned Bits) { } void DataLayout::parseSpecifier(StringRef Desc) { + StringRepresentation = Desc; while (!Desc.empty()) { // Split at '-'. std::pair Split = split(Desc, '-'); @@ -259,6 +255,8 @@ void DataLayout::parseSpecifier(StringRef Desc) { "Missing size specification for pointer in datalayout string"); Split = split(Rest, ':'); unsigned PointerMemSize = inBytes(getInt(Tok)); + if (!PointerMemSize) + report_fatal_error("Invalid pointer size of 0 bytes"); // ABI alignment. if (Rest.empty()) @@ -266,12 +264,18 @@ void DataLayout::parseSpecifier(StringRef Desc) { "Missing alignment specification for pointer in datalayout string"); Split = split(Rest, ':'); unsigned PointerABIAlign = inBytes(getInt(Tok)); + if (!isPowerOf2_64(PointerABIAlign)) + report_fatal_error( + "Pointer ABI alignment must be a power of 2"); // Preferred alignment. unsigned PointerPrefAlign = PointerABIAlign; if (!Rest.empty()) { Split = split(Rest, ':'); PointerPrefAlign = inBytes(getInt(Tok)); + if (!isPowerOf2_64(PointerPrefAlign)) + report_fatal_error( + "Pointer preferred alignment must be a power of 2"); } setPointerAlignment(AddrSpace, PointerABIAlign, PointerPrefAlign, @@ -304,6 +308,9 @@ void DataLayout::parseSpecifier(StringRef Desc) { "Missing alignment specification in datalayout string"); Split = split(Rest, ':'); unsigned ABIAlign = inBytes(getInt(Tok)); + if (AlignType != AGGREGATE_ALIGN && !ABIAlign) + report_fatal_error( + "ABI alignment specification must be >0 for non-aggregate types"); // Preferred alignment. unsigned PrefAlign = ABIAlign; @@ -352,7 +359,10 @@ void DataLayout::parseSpecifier(StringRef Desc) { ManglingMode = MM_Mips; break; case 'w': - ManglingMode = MM_WINCOFF; + ManglingMode = MM_WinCOFF; + break; + case 'x': + ManglingMode = MM_WinCOFFX86; break; } break; @@ -367,13 +377,7 @@ DataLayout::DataLayout(const Module *M) : LayoutMap(nullptr) { init(M); } -void DataLayout::init(const Module *M) { - const DataLayout *Other = M->getDataLayout(); - if (Other) - *this = *Other; - else - reset(""); -} +void DataLayout::init(const Module *M) { *this = M->getDataLayout(); } bool DataLayout::operator==(const DataLayout &Other) const { bool Ret = BigEndian == Other.BigEndian && @@ -381,7 +385,7 @@ bool DataLayout::operator==(const DataLayout &Other) const { ManglingMode == Other.ManglingMode && LegalIntWidths == Other.LegalIntWidths && Alignments == Other.Alignments && Pointers == Other.Pointers; - assert(Ret == (getStringRepresentation() == Other.getStringRepresentation())); + // Note: getStringRepresentation() might differs, it is not canonicalized return Ret; } @@ -394,6 +398,10 @@ DataLayout::setAlignment(AlignTypeEnum align_type, unsigned abi_align, report_fatal_error("Invalid ABI alignment, must be a 16bit integer"); if (!isUInt<16>(pref_align)) report_fatal_error("Invalid preferred alignment, must be a 16bit integer"); + if (abi_align != 0 && !isPowerOf2_64(abi_align)) + report_fatal_error("Invalid ABI alignment, must be a power of 2"); + if (pref_align != 0 && !isPowerOf2_64(pref_align)) + report_fatal_error("Invalid preferred alignment, must be a power of 2"); if (pref_align < abi_align) report_fatal_error( @@ -474,9 +482,7 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, // If we didn't find an integer alignment, fall back on most conservative. if (AlignType == INTEGER_ALIGN) { BestMatchIdx = LargestInt; - } else { - assert(AlignType == VECTOR_ALIGN && "Unknown alignment type!"); - + } else if (AlignType == VECTOR_ALIGN) { // By default, use natural alignment for vector types. This is consistent // with what clang and llvm-gcc do. unsigned Align = getTypeAllocSize(cast(Ty)->getElementType()); @@ -489,6 +495,19 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, } } + // If we still couldn't find a reasonable default alignment, fall back + // to a simple heuristic that the alignment is the first power of two + // greater-or-equal to the store size of the type. This is a reasonable + // approximation of reality, and if the user wanted something less + // less conservative, they should have specified it explicitly in the data + // layout. + if (BestMatchIdx == -1) { + unsigned Align = getTypeStoreSize(Ty); + if (Align & (Align-1)) + Align = NextPowerOf2(Align); + return Align; + } + // Since we got a "best match" index, just return it. return ABIInfo ? Alignments[BestMatchIdx].ABIAlign : Alignments[BestMatchIdx].PrefAlign; @@ -552,68 +571,6 @@ const StructLayout *DataLayout::getStructLayout(StructType *Ty) const { return L; } -std::string DataLayout::getStringRepresentation() const { - std::string Result; - raw_string_ostream OS(Result); - - OS << (BigEndian ? "E" : "e"); - - switch (ManglingMode) { - case MM_None: - break; - case MM_ELF: - OS << "-m:e"; - break; - case MM_MachO: - OS << "-m:o"; - break; - case MM_WINCOFF: - OS << "-m:w"; - break; - case MM_Mips: - OS << "-m:m"; - break; - } - - for (const PointerAlignElem &PI : Pointers) { - // Skip default. - if (PI.AddressSpace == 0 && PI.ABIAlign == 8 && PI.PrefAlign == 8 && - PI.TypeByteWidth == 8) - continue; - - OS << "-p"; - if (PI.AddressSpace) { - OS << PI.AddressSpace; - } - OS << ":" << PI.TypeByteWidth*8 << ':' << PI.ABIAlign*8; - if (PI.PrefAlign != PI.ABIAlign) - OS << ':' << PI.PrefAlign*8; - } - - for (const LayoutAlignElem &AI : Alignments) { - if (std::find(std::begin(DefaultAlignments), std::end(DefaultAlignments), - AI) != std::end(DefaultAlignments)) - continue; - OS << '-' << (char)AI.AlignType; - if (AI.TypeBitWidth) - OS << AI.TypeBitWidth; - OS << ':' << AI.ABIAlign*8; - if (AI.ABIAlign != AI.PrefAlign) - OS << ':' << AI.PrefAlign*8; - } - - if (!LegalIntWidths.empty()) { - OS << "-n" << (unsigned)LegalIntWidths[0]; - - for (unsigned i = 1, e = LegalIntWidths.size(); i != e; ++i) - OS << ':' << (unsigned)LegalIntWidths[i]; - } - - if (StackNaturalAlign) - OS << "-S" << StackNaturalAlign*8; - - return OS.str(); -} unsigned DataLayout::getPointerABIAlignment(unsigned AS) const { PointersTy::const_iterator I = findPointerLowerBound(AS); @@ -829,18 +786,3 @@ unsigned DataLayout::getPreferredAlignmentLog(const GlobalVariable *GV) const { return Log2_32(getPreferredAlignment(GV)); } -DataLayoutPass::DataLayoutPass() : ImmutablePass(ID), DL("") { - initializeDataLayoutPassPass(*PassRegistry::getPassRegistry()); -} - -DataLayoutPass::~DataLayoutPass() {} - -bool DataLayoutPass::doInitialization(Module &M) { - DL.init(&M); - return false; -} - -bool DataLayoutPass::doFinalization(Module &M) { - DL.reset(""); - return false; -} diff --git a/lib/IR/DebugInfo.cpp b/lib/IR/DebugInfo.cpp index 6590661..9a6b953 100644 --- a/lib/IR/DebugInfo.cpp +++ b/lib/IR/DebugInfo.cpp @@ -92,7 +92,7 @@ bool DIDescriptor::Verify() const { DIObjCProperty(DbgNode).Verify() || DITemplateTypeParameter(DbgNode).Verify() || DITemplateValueParameter(DbgNode).Verify() || - DIImportedEntity(DbgNode).Verify() || DIExpression(DbgNode).Verify()); + DIImportedEntity(DbgNode).Verify()); } static Metadata *getField(const MDNode *DbgNode, unsigned Elt) { @@ -155,21 +155,6 @@ Function *DIDescriptor::getFunctionField(unsigned Elt) const { return dyn_cast_or_null(getConstantField(Elt)); } -void DIDescriptor::replaceFunctionField(unsigned Elt, Function *F) { - if (!DbgNode) - return; - - if (Elt < DbgNode->getNumOperands()) { - MDNode *Node = const_cast(DbgNode); - Node->replaceOperandWith(Elt, F ? ConstantAsMetadata::get(F) : nullptr); - } -} - -static unsigned DIVariableInlinedAtIndex = 4; -MDNode *DIVariable::getInlinedAt() const { - return getNodeField(DbgNode, DIVariableInlinedAtIndex); -} - /// \brief Return the size reported by the variable's type. unsigned DIVariable::getSizeInBits(const DITypeIdentifierMap &Map) { DIType Ty = getType().resolve(Map); @@ -183,13 +168,6 @@ unsigned DIVariable::getSizeInBits(const DITypeIdentifierMap &Map) { return Ty.getSizeInBits(); } -uint64_t DIExpression::getElement(unsigned Idx) const { - unsigned I = Idx + 1; - assert(I < getNumHeaderFields() && - "non-existing complex address element requested"); - return getHeaderFieldAs(I); -} - bool DIExpression::isBitPiece() const { unsigned N = getNumElements(); return N >=3 && getElement(N-3) == dwarf::DW_OP_bit_piece; @@ -205,206 +183,40 @@ uint64_t DIExpression::getBitPieceSize() const { return getElement(getNumElements()-1); } -DIExpression::iterator DIExpression::begin() const { - return DIExpression::iterator(*this); -} - -DIExpression::iterator DIExpression::end() const { - return DIExpression::iterator(); -} - -DIExpression::Operand DIExpression::Operand::getNext() const { +DIExpression::iterator DIExpression::Operand::getNext() const { iterator it(I); - return *(++it); -} - -//===----------------------------------------------------------------------===// -// Predicates -//===----------------------------------------------------------------------===// - -bool DIDescriptor::isSubroutineType() const { - return DbgNode && getTag() == dwarf::DW_TAG_subroutine_type; -} - -bool DIDescriptor::isBasicType() const { - if (!DbgNode) - return false; - switch (getTag()) { - case dwarf::DW_TAG_base_type: - case dwarf::DW_TAG_unspecified_type: - return true; - default: - return false; - } -} - -bool DIDescriptor::isDerivedType() const { - if (!DbgNode) - return false; - switch (getTag()) { - case dwarf::DW_TAG_typedef: - case dwarf::DW_TAG_pointer_type: - case dwarf::DW_TAG_ptr_to_member_type: - case dwarf::DW_TAG_reference_type: - case dwarf::DW_TAG_rvalue_reference_type: - case dwarf::DW_TAG_const_type: - case dwarf::DW_TAG_volatile_type: - case dwarf::DW_TAG_restrict_type: - case dwarf::DW_TAG_member: - case dwarf::DW_TAG_inheritance: - case dwarf::DW_TAG_friend: - return true; - default: - // CompositeTypes are currently modelled as DerivedTypes. - return isCompositeType(); - } -} - -bool DIDescriptor::isCompositeType() const { - if (!DbgNode) - return false; - switch (getTag()) { - case dwarf::DW_TAG_array_type: - case dwarf::DW_TAG_structure_type: - case dwarf::DW_TAG_union_type: - case dwarf::DW_TAG_enumeration_type: - case dwarf::DW_TAG_subroutine_type: - case dwarf::DW_TAG_class_type: - return true; - default: - return false; - } -} - -bool DIDescriptor::isVariable() const { - if (!DbgNode) - return false; - switch (getTag()) { - case dwarf::DW_TAG_auto_variable: - case dwarf::DW_TAG_arg_variable: - return true; - default: - return false; - } -} - -bool DIDescriptor::isType() const { - return isBasicType() || isCompositeType() || isDerivedType(); -} - -bool DIDescriptor::isSubprogram() const { - return DbgNode && getTag() == dwarf::DW_TAG_subprogram; -} - -bool DIDescriptor::isGlobalVariable() const { - return DbgNode && getTag() == dwarf::DW_TAG_variable; -} - -bool DIDescriptor::isScope() const { - if (!DbgNode) - return false; - switch (getTag()) { - case dwarf::DW_TAG_compile_unit: - case dwarf::DW_TAG_lexical_block: - case dwarf::DW_TAG_subprogram: - case dwarf::DW_TAG_namespace: - case dwarf::DW_TAG_file_type: - return true; - default: - break; - } - return isType(); -} - -bool DIDescriptor::isTemplateTypeParameter() const { - return DbgNode && getTag() == dwarf::DW_TAG_template_type_parameter; -} - -bool DIDescriptor::isTemplateValueParameter() const { - return DbgNode && (getTag() == dwarf::DW_TAG_template_value_parameter || - getTag() == dwarf::DW_TAG_GNU_template_template_param || - getTag() == dwarf::DW_TAG_GNU_template_parameter_pack); -} - -bool DIDescriptor::isCompileUnit() const { - return DbgNode && getTag() == dwarf::DW_TAG_compile_unit; -} - -bool DIDescriptor::isFile() const { - return DbgNode && getTag() == dwarf::DW_TAG_file_type; -} - -bool DIDescriptor::isNameSpace() const { - return DbgNode && getTag() == dwarf::DW_TAG_namespace; -} - -bool DIDescriptor::isLexicalBlockFile() const { - return DbgNode && getTag() == dwarf::DW_TAG_lexical_block && - DbgNode->getNumOperands() == 3 && getNumHeaderFields() == 2; -} - -bool DIDescriptor::isLexicalBlock() const { - // FIXME: There are always exactly 4 header fields in DILexicalBlock, but - // something relies on this returning true for DILexicalBlockFile. - return DbgNode && getTag() == dwarf::DW_TAG_lexical_block && - DbgNode->getNumOperands() == 3 && - (getNumHeaderFields() == 2 || getNumHeaderFields() == 4); -} - -bool DIDescriptor::isSubrange() const { - return DbgNode && getTag() == dwarf::DW_TAG_subrange_type; -} - -bool DIDescriptor::isEnumerator() const { - return DbgNode && getTag() == dwarf::DW_TAG_enumerator; -} - -bool DIDescriptor::isObjCProperty() const { - return DbgNode && getTag() == dwarf::DW_TAG_APPLE_property; -} - -bool DIDescriptor::isImportedEntity() const { - return DbgNode && (getTag() == dwarf::DW_TAG_imported_module || - getTag() == dwarf::DW_TAG_imported_declaration); -} - -bool DIDescriptor::isExpression() const { - return DbgNode && (getTag() == dwarf::DW_TAG_expression); + return ++it; } //===----------------------------------------------------------------------===// // Simple Descriptor Constructors and other Methods //===----------------------------------------------------------------------===// -void DIDescriptor::replaceAllUsesWith(LLVMContext &VMContext, DIDescriptor D) { - +void DIDescriptor::replaceAllUsesWith(LLVMContext &, DIDescriptor D) { assert(DbgNode && "Trying to replace an unverified type!"); + assert(DbgNode->isTemporary() && "Expected temporary node"); + TempMDNode Temp(get()); // Since we use a TrackingVH for the node, its easy for clients to manufacture // legitimate situations where they want to replaceAllUsesWith() on something // which, due to uniquing, has merged with the source. We shield clients from // this detail by allowing a value to be replaced with replaceAllUsesWith() // itself. - const MDNode *DN = D; - if (DbgNode == DN) { - SmallVector Ops(DbgNode->op_begin(), DbgNode->op_end()); - DN = MDNode::get(VMContext, Ops); + if (Temp.get() == D.get()) { + DbgNode = MDNode::replaceWithUniqued(std::move(Temp)); + return; } - assert(DbgNode->isTemporary() && "Expected temporary node"); - auto *Node = const_cast(DbgNode); - Node->replaceAllUsesWith(const_cast(DN)); - MDNode::deleteTemporary(Node); - DbgNode = DN; + Temp->replaceAllUsesWith(D.get()); + DbgNode = D.get(); } void DIDescriptor::replaceAllUsesWith(MDNode *D) { assert(DbgNode && "Trying to replace an unverified type!"); assert(DbgNode != D && "This replacement should always happen"); assert(DbgNode->isTemporary() && "Expected temporary node"); - auto *Node = const_cast(DbgNode); + TempMDNode Node(get()); Node->replaceAllUsesWith(D); - MDNode::deleteTemporary(Node); } bool DICompileUnit::Verify() const { @@ -413,31 +225,10 @@ bool DICompileUnit::Verify() const { // Don't bother verifying the compilation directory or producer string // as those could be empty. - if (getFilename().empty()) - return false; - - return DbgNode->getNumOperands() == 7 && getNumHeaderFields() == 8; -} - -bool DIObjCProperty::Verify() const { - if (!isObjCProperty()) - return false; - - // Don't worry about the rest of the strings for now. - return DbgNode->getNumOperands() == 3 && getNumHeaderFields() == 6; -} - -/// \brief Check if a field at position Elt of a MDNode is a MDNode. -static bool fieldIsMDNode(const MDNode *DbgNode, unsigned Elt) { - Metadata *Fld = getField(DbgNode, Elt); - return !Fld || isa(Fld); + return !getFilename().empty(); } -/// \brief Check if a field at position Elt of a MDNode is a MDString. -static bool fieldIsMDString(const MDNode *DbgNode, unsigned Elt) { - Metadata *Fld = getField(DbgNode, Elt); - return !Fld || isa(Fld); -} +bool DIObjCProperty::Verify() const { return isObjCProperty(); } /// \brief Check if a value can be a reference to a type. static bool isTypeRef(const Metadata *MD) { @@ -445,14 +236,7 @@ static bool isTypeRef(const Metadata *MD) { return true; if (auto *S = dyn_cast(MD)) return !S->getString().empty(); - if (auto *N = dyn_cast(MD)) - return DIType(N).isType(); - return false; -} - -/// \brief Check if referenced field might be a type. -static bool fieldIsTypeRef(const MDNode *DbgNode, unsigned Elt) { - return isTypeRef(dyn_cast_or_null(getField(DbgNode, Elt))); + return isa(MD); } /// \brief Check if a value can be a ScopeRef. @@ -461,14 +245,7 @@ static bool isScopeRef(const Metadata *MD) { return true; if (auto *S = dyn_cast(MD)) return !S->getString().empty(); - if (auto *N = dyn_cast(MD)) - return DIScope(N).isScope(); - return false; -} - -/// \brief Check if a field at position Elt of a MDNode can be a ScopeRef. -static bool fieldIsScopeRef(const MDNode *DbgNode, unsigned Elt) { - return isScopeRef(dyn_cast_or_null(getField(DbgNode, Elt))); + return isa(MD); } #ifndef NDEBUG @@ -483,92 +260,81 @@ static bool isDescriptorRef(const Metadata *MD) { #endif bool DIType::Verify() const { - if (!isType()) + auto *N = dyn_cast_or_null(DbgNode); + if (!N) return false; - // Make sure Context @ field 2 is MDNode. - if (!fieldIsScopeRef(DbgNode, 2)) - return false; - - // FIXME: Sink this into the various subclass verifies. - uint16_t Tag = getTag(); - if (!isBasicType() && Tag != dwarf::DW_TAG_const_type && - Tag != dwarf::DW_TAG_volatile_type && Tag != dwarf::DW_TAG_pointer_type && - Tag != dwarf::DW_TAG_ptr_to_member_type && - Tag != dwarf::DW_TAG_reference_type && - Tag != dwarf::DW_TAG_rvalue_reference_type && - Tag != dwarf::DW_TAG_restrict_type && Tag != dwarf::DW_TAG_array_type && - Tag != dwarf::DW_TAG_enumeration_type && - Tag != dwarf::DW_TAG_subroutine_type && - Tag != dwarf::DW_TAG_inheritance && Tag != dwarf::DW_TAG_friend && - getFilename().empty()) + if (!isScopeRef(N->getScope())) return false; // DIType is abstract, it should be a BasicType, a DerivedType or // a CompositeType. if (isBasicType()) return DIBasicType(DbgNode).Verify(); - else if (isCompositeType()) + + // FIXME: Sink this into the various subclass verifies. + if (getFilename().empty()) { + // Check whether the filename is allowed to be empty. + uint16_t Tag = getTag(); + if (Tag != dwarf::DW_TAG_const_type && Tag != dwarf::DW_TAG_volatile_type && + Tag != dwarf::DW_TAG_pointer_type && + Tag != dwarf::DW_TAG_ptr_to_member_type && + Tag != dwarf::DW_TAG_reference_type && + Tag != dwarf::DW_TAG_rvalue_reference_type && + Tag != dwarf::DW_TAG_restrict_type && Tag != dwarf::DW_TAG_array_type && + Tag != dwarf::DW_TAG_enumeration_type && + Tag != dwarf::DW_TAG_subroutine_type && + Tag != dwarf::DW_TAG_inheritance && Tag != dwarf::DW_TAG_friend && + Tag != dwarf::DW_TAG_structure_type && Tag != dwarf::DW_TAG_member && + Tag != dwarf::DW_TAG_typedef) + return false; + } + + if (isCompositeType()) return DICompositeType(DbgNode).Verify(); - else if (isDerivedType()) + if (isDerivedType()) return DIDerivedType(DbgNode).Verify(); - else - return false; + return false; } bool DIBasicType::Verify() const { - return isBasicType() && DbgNode->getNumOperands() == 3 && - getNumHeaderFields() == 8; + return dyn_cast_or_null(DbgNode); } bool DIDerivedType::Verify() const { - // Make sure DerivedFrom @ field 3 is TypeRef. - if (!fieldIsTypeRef(DbgNode, 3)) + auto *N = dyn_cast_or_null(DbgNode); + if (!N) return false; - if (getTag() == dwarf::DW_TAG_ptr_to_member_type) - // Make sure ClassType @ field 4 is a TypeRef. - if (!fieldIsTypeRef(DbgNode, 4)) + if (getTag() == dwarf::DW_TAG_ptr_to_member_type) { + auto *D = dyn_cast(N); + if (!D) return false; - - return isDerivedType() && DbgNode->getNumOperands() >= 4 && - DbgNode->getNumOperands() <= 8 && getNumHeaderFields() >= 7 && - getNumHeaderFields() <= 8; + if (!isTypeRef(D->getExtraData())) + return false; + } + return isTypeRef(N->getBaseType()); } bool DICompositeType::Verify() const { - if (!isCompositeType()) - return false; - - // Make sure DerivedFrom @ field 3 and ContainingType @ field 5 are TypeRef. - if (!fieldIsTypeRef(DbgNode, 3)) - return false; - if (!fieldIsTypeRef(DbgNode, 5)) - return false; - - // Make sure the type identifier at field 7 is MDString, it can be null. - if (!fieldIsMDString(DbgNode, 7)) - return false; - - // A subroutine type can't be both & and &&. - if (isLValueReference() && isRValueReference()) - return false; - - return DbgNode->getNumOperands() == 8 && getNumHeaderFields() == 8; + auto *N = dyn_cast_or_null(DbgNode); + return N && isTypeRef(N->getBaseType()) && isTypeRef(N->getVTableHolder()) && + !(isLValueReference() && isRValueReference()); } bool DISubprogram::Verify() const { - if (!isSubprogram()) + auto *N = dyn_cast_or_null(DbgNode); + if (!N) return false; - // Make sure context @ field 2 is a ScopeRef and type @ field 3 is a MDNode. - if (!fieldIsScopeRef(DbgNode, 2)) + if (!isScopeRef(N->getScope())) return false; - if (!fieldIsMDNode(DbgNode, 3)) - return false; - // Containing type @ field 4. - if (!fieldIsTypeRef(DbgNode, 4)) + + if (auto *Op = N->getType()) + if (!isa(Op)) + return false; + + if (!isTypeRef(getContainingType())) return false; - // A subprogram can't be both & and &&. if (isLValueReference() && isRValueReference()) return false; @@ -603,164 +369,78 @@ bool DISubprogram::Verify() const { } } } - return DbgNode->getNumOperands() == 9 && getNumHeaderFields() == 12; + + return true; } bool DIGlobalVariable::Verify() const { - if (!isGlobalVariable()) - return false; + auto *N = dyn_cast_or_null(DbgNode); - if (getDisplayName().empty()) - return false; - // Make sure context @ field 1 is an MDNode. - if (!fieldIsMDNode(DbgNode, 1)) - return false; - // Make sure that type @ field 3 is a DITypeRef. - if (!fieldIsTypeRef(DbgNode, 3)) - return false; - // Make sure StaticDataMemberDeclaration @ field 5 is MDNode. - if (!fieldIsMDNode(DbgNode, 5)) - return false; - - return DbgNode->getNumOperands() == 6 && getNumHeaderFields() == 7; -} - -bool DIVariable::Verify() const { - if (!isVariable()) + if (!N) return false; - // Make sure context @ field 1 is an MDNode. - if (!fieldIsMDNode(DbgNode, 1)) - return false; - // Make sure that type @ field 3 is a DITypeRef. - if (!fieldIsTypeRef(DbgNode, 3)) + if (N->getDisplayName().empty()) return false; - // Check the number of header fields, which is common between complex and - // simple variables. - if (getNumHeaderFields() != 4) - return false; + if (auto *Op = N->getScope()) + if (!isa(Op)) + return false; - // Variable without an inline location. - if (DbgNode->getNumOperands() == 4) - return true; + if (auto *Op = N->getStaticDataMemberDeclaration()) + if (!isa(Op)) + return false; - // Variable with an inline location. - return getInlinedAt() != nullptr && DbgNode->getNumOperands() == 5; + return isTypeRef(N->getType()); } -bool DIExpression::Verify() const { - // Empty DIExpressions may be represented as a nullptr. - if (!DbgNode) - return true; +bool DIVariable::Verify() const { + auto *N = dyn_cast_or_null(DbgNode); - if (!(isExpression() && DbgNode->getNumOperands() == 1)) + if (!N) return false; - for (auto Op : *this) - switch (Op) { - case DW_OP_bit_piece: - // Must be the last element of the expression. - return std::distance(Op.getBase(), DIHeaderFieldIterator()) == 3; - case DW_OP_plus: - if (std::distance(Op.getBase(), DIHeaderFieldIterator()) < 2) - return false; - break; - case DW_OP_deref: - break; - default: - // Other operators are not yet supported by the backend. + if (auto *Op = N->getScope()) + if (!isa(Op)) return false; - } - return true; + + return isTypeRef(N->getType()); } bool DILocation::Verify() const { - return DbgNode && isa(DbgNode); + return dyn_cast_or_null(DbgNode); } - bool DINameSpace::Verify() const { - if (!isNameSpace()) - return false; - return DbgNode->getNumOperands() == 3 && getNumHeaderFields() == 3; -} - -MDNode *DIFile::getFileNode() const { return getNodeField(DbgNode, 1); } - -bool DIFile::Verify() const { - return isFile() && DbgNode->getNumOperands() == 2; + return dyn_cast_or_null(DbgNode); } - +bool DIFile::Verify() const { return dyn_cast_or_null(DbgNode); } bool DIEnumerator::Verify() const { - return isEnumerator() && DbgNode->getNumOperands() == 1 && - getNumHeaderFields() == 3; + return dyn_cast_or_null(DbgNode); } - bool DISubrange::Verify() const { - return isSubrange() && DbgNode->getNumOperands() == 1 && - getNumHeaderFields() == 3; + return dyn_cast_or_null(DbgNode); } - bool DILexicalBlock::Verify() const { - return isLexicalBlock() && DbgNode->getNumOperands() == 3 && - getNumHeaderFields() == 4; + return dyn_cast_or_null(DbgNode); } - bool DILexicalBlockFile::Verify() const { - return isLexicalBlockFile() && DbgNode->getNumOperands() == 3 && - getNumHeaderFields() == 2; + return dyn_cast_or_null(DbgNode); } - bool DITemplateTypeParameter::Verify() const { - return isTemplateTypeParameter() && DbgNode->getNumOperands() == 4 && - getNumHeaderFields() == 4; + return dyn_cast_or_null(DbgNode); } - bool DITemplateValueParameter::Verify() const { - return isTemplateValueParameter() && DbgNode->getNumOperands() == 5 && - getNumHeaderFields() == 4; + return dyn_cast_or_null(DbgNode); } - bool DIImportedEntity::Verify() const { - return isImportedEntity() && DbgNode->getNumOperands() == 3 && - getNumHeaderFields() == 3; -} - -MDNode *DIDerivedType::getObjCProperty() const { - return getNodeField(DbgNode, 4); + return dyn_cast_or_null(DbgNode); } -MDString *DICompositeType::getIdentifier() const { - return cast_or_null(getField(DbgNode, 7)); -} - -#ifndef NDEBUG -static void VerifySubsetOf(const MDNode *LHS, const MDNode *RHS) { - for (unsigned i = 0; i != LHS->getNumOperands(); ++i) { - // Skip the 'empty' list (that's a single i32 0, rather than truly empty). - if (i == 0 && mdconst::hasa(LHS->getOperand(i))) - continue; - const MDNode *E = cast(LHS->getOperand(i)); - bool found = false; - for (unsigned j = 0; !found && j != RHS->getNumOperands(); ++j) - found = (E == cast(RHS->getOperand(j))); - assert(found && "Losing a member during member list replacement"); - } -} -#endif - void DICompositeType::setArraysHelper(MDNode *Elements, MDNode *TParams) { - TrackingMDNodeRef N(*this); - if (Elements) { -#ifndef NDEBUG - // Check that the new list of members contains all the old members as well. - if (const MDNode *El = cast_or_null(N->getOperand(4))) - VerifySubsetOf(El, Elements); -#endif - N->replaceOperandWith(4, Elements); - } + TypedTrackingMDRef N(get()); + if (Elements) + N->replaceElements(cast(Elements)); if (TParams) - N->replaceOperandWith(6, TParams); + N->replaceTemplateParams(cast(TParams)); DbgNode = N; } @@ -774,8 +454,8 @@ DIScopeRef DIScope::getRef() const { } void DICompositeType::setContainingType(DICompositeType ContainingType) { - TrackingMDNodeRef N(*this); - N->replaceOperandWith(5, ContainingType.getRef()); + TypedTrackingMDRef N(get()); + N->replaceVTableHolder(ContainingType.getRef()); DbgNode = N; } @@ -788,6 +468,13 @@ bool DIVariable::isInlinedFnArgument(const Function *CurFn) { return !DISubprogram(getContext()).describes(CurFn); } +Function *DISubprogram::getFunction() const { + if (auto *N = get()) + if (auto *C = dyn_cast_or_null(N->getFunction())) + return dyn_cast(C->getValue()); + return nullptr; +} + bool DISubprogram::describes(const Function *F) { assert(F && "Invalid function"); if (F == getFunction()) @@ -800,16 +487,8 @@ bool DISubprogram::describes(const Function *F) { return false; } -MDNode *DISubprogram::getVariablesNodes() const { - return getNodeField(DbgNode, 8); -} - -DIArray DISubprogram::getVariables() const { - return DIArray(getNodeField(DbgNode, 8)); -} - -Metadata *DITemplateValueParameter::getValue() const { - return DbgNode->getOperand(3); +GlobalVariable *DIGlobalVariable::getGlobal() const { + return dyn_cast_or_null(getConstant()); } DIScopeRef DIScope::getContext() const { @@ -847,66 +526,25 @@ StringRef DIScope::getName() const { } StringRef DIScope::getFilename() const { - if (!DbgNode) - return StringRef(); - return ::getStringField(getNodeField(DbgNode, 1), 0); + if (auto *N = get()) + return ::getStringField(dyn_cast_or_null(N->getFile()), 0); + return ""; } StringRef DIScope::getDirectory() const { - if (!DbgNode) - return StringRef(); - return ::getStringField(getNodeField(DbgNode, 1), 1); -} - -DIArray DICompileUnit::getEnumTypes() const { - if (!DbgNode || DbgNode->getNumOperands() < 7) - return DIArray(); - - return DIArray(getNodeField(DbgNode, 2)); -} - -DIArray DICompileUnit::getRetainedTypes() const { - if (!DbgNode || DbgNode->getNumOperands() < 7) - return DIArray(); - - return DIArray(getNodeField(DbgNode, 3)); -} - -DIArray DICompileUnit::getSubprograms() const { - if (!DbgNode || DbgNode->getNumOperands() < 7) - return DIArray(); - - return DIArray(getNodeField(DbgNode, 4)); -} - -DIArray DICompileUnit::getGlobalVariables() const { - if (!DbgNode || DbgNode->getNumOperands() < 7) - return DIArray(); - - return DIArray(getNodeField(DbgNode, 5)); -} - -DIArray DICompileUnit::getImportedEntities() const { - if (!DbgNode || DbgNode->getNumOperands() < 7) - return DIArray(); - - return DIArray(getNodeField(DbgNode, 6)); + if (auto *N = get()) + return ::getStringField(dyn_cast_or_null(N->getFile()), 1); + return ""; } void DICompileUnit::replaceSubprograms(DIArray Subprograms) { assert(Verify() && "Expected compile unit"); - if (Subprograms == getSubprograms()) - return; - - const_cast(DbgNode)->replaceOperandWith(4, Subprograms); + get()->replaceSubprograms(cast_or_null(Subprograms.get())); } void DICompileUnit::replaceGlobalVariables(DIArray GlobalVariables) { assert(Verify() && "Expected compile unit"); - if (GlobalVariables == getGlobalVariables()) - return; - - const_cast(DbgNode)->replaceOperandWith(5, GlobalVariables); + get()->replaceGlobalVariables(cast_or_null(GlobalVariables.get())); } DILocation DILocation::copyWithNewScope(LLVMContext &Ctx, @@ -927,31 +565,13 @@ unsigned DILocation::computeNewDiscriminator(LLVMContext &Ctx) { DIVariable llvm::createInlinedVariable(MDNode *DV, MDNode *InlinedScope, LLVMContext &VMContext) { assert(DIVariable(DV).Verify() && "Expected a DIVariable"); - if (!InlinedScope) - return cleanseInlinedVariable(DV, VMContext); - - // Insert inlined scope. - SmallVector Elts(DV->op_begin(), - DV->op_begin() + DIVariableInlinedAtIndex); - Elts.push_back(InlinedScope); - - DIVariable Inlined(MDNode::get(VMContext, Elts)); - assert(Inlined.Verify() && "Expected to create a DIVariable"); - return Inlined; + return cast(DV) + ->withInline(cast_or_null(InlinedScope)); } DIVariable llvm::cleanseInlinedVariable(MDNode *DV, LLVMContext &VMContext) { assert(DIVariable(DV).Verify() && "Expected a DIVariable"); - if (!DIVariable(DV).getInlinedAt()) - return DIVariable(DV); - - // Remove inlined scope. - SmallVector Elts(DV->op_begin(), - DV->op_begin() + DIVariableInlinedAtIndex); - - DIVariable Cleansed(MDNode::get(VMContext, Elts)); - assert(Cleansed.Verify() && "Expected to create a DIVariable"); - return Cleansed; + return cast(DV)->withoutInline(); } DISubprogram llvm::getDISubprogram(const MDNode *Scope) { @@ -1075,6 +695,8 @@ void DebugInfoFinder::processModule(const Module &M) { DIArray Imports = CU.getImportedEntities(); for (unsigned i = 0, e = Imports.getNumElements(); i != e; ++i) { DIImportedEntity Import = DIImportedEntity(Imports.getElement(i)); + if (!Import) + continue; DIDescriptor Entity = Import.getEntity().resolve(TypeIdentifierMap); if (Entity.isType()) processType(DIType(Entity)); @@ -1267,220 +889,9 @@ void DIDescriptor::dump() const { } void DIDescriptor::print(raw_ostream &OS) const { - if (!DbgNode) - return; - - if (const char *Tag = dwarf::TagString(getTag())) - OS << "[ " << Tag << " ]"; - - if (this->isSubrange()) { - DISubrange(DbgNode).printInternal(OS); - } else if (this->isCompileUnit()) { - DICompileUnit(DbgNode).printInternal(OS); - } else if (this->isFile()) { - DIFile(DbgNode).printInternal(OS); - } else if (this->isEnumerator()) { - DIEnumerator(DbgNode).printInternal(OS); - } else if (this->isBasicType()) { - DIType(DbgNode).printInternal(OS); - } else if (this->isDerivedType()) { - DIDerivedType(DbgNode).printInternal(OS); - } else if (this->isCompositeType()) { - DICompositeType(DbgNode).printInternal(OS); - } else if (this->isSubprogram()) { - DISubprogram(DbgNode).printInternal(OS); - } else if (this->isGlobalVariable()) { - DIGlobalVariable(DbgNode).printInternal(OS); - } else if (this->isVariable()) { - DIVariable(DbgNode).printInternal(OS); - } else if (this->isObjCProperty()) { - DIObjCProperty(DbgNode).printInternal(OS); - } else if (this->isNameSpace()) { - DINameSpace(DbgNode).printInternal(OS); - } else if (this->isScope()) { - DIScope(DbgNode).printInternal(OS); - } else if (this->isExpression()) { - DIExpression(DbgNode).printInternal(OS); - } -} - -void DISubrange::printInternal(raw_ostream &OS) const { - int64_t Count = getCount(); - if (Count != -1) - OS << " [" << getLo() << ", " << Count - 1 << ']'; - else - OS << " [unbounded]"; -} - -void DIScope::printInternal(raw_ostream &OS) const { - OS << " [" << getDirectory() << "/" << getFilename() << ']'; -} - -void DICompileUnit::printInternal(raw_ostream &OS) const { - DIScope::printInternal(OS); - OS << " ["; - unsigned Lang = getLanguage(); - if (const char *LangStr = dwarf::LanguageString(Lang)) - OS << LangStr; - else - (OS << "lang 0x").write_hex(Lang); - OS << ']'; -} - -void DIEnumerator::printInternal(raw_ostream &OS) const { - OS << " [" << getName() << " :: " << getEnumValue() << ']'; -} - -void DIType::printInternal(raw_ostream &OS) const { - if (!DbgNode) + if (!get()) return; - - StringRef Res = getName(); - if (!Res.empty()) - OS << " [" << Res << "]"; - - // TODO: Print context? - - OS << " [line " << getLineNumber() << ", size " << getSizeInBits() - << ", align " << getAlignInBits() << ", offset " << getOffsetInBits(); - if (isBasicType()) - if (const char *Enc = - dwarf::AttributeEncodingString(DIBasicType(DbgNode).getEncoding())) - OS << ", enc " << Enc; - OS << "]"; - - if (isPrivate()) - OS << " [private]"; - else if (isProtected()) - OS << " [protected]"; - else if (isPublic()) - OS << " [public]"; - - if (isArtificial()) - OS << " [artificial]"; - - if (isForwardDecl()) - OS << " [decl]"; - else if (getTag() == dwarf::DW_TAG_structure_type || - getTag() == dwarf::DW_TAG_union_type || - getTag() == dwarf::DW_TAG_enumeration_type || - getTag() == dwarf::DW_TAG_class_type) - OS << " [def]"; - if (isVector()) - OS << " [vector]"; - if (isStaticMember()) - OS << " [static]"; - - if (isLValueReference()) - OS << " [reference]"; - - if (isRValueReference()) - OS << " [rvalue reference]"; -} - -void DIDerivedType::printInternal(raw_ostream &OS) const { - DIType::printInternal(OS); - OS << " [from " << getTypeDerivedFrom().getName() << ']'; -} - -void DICompositeType::printInternal(raw_ostream &OS) const { - DIType::printInternal(OS); - DIArray A = getElements(); - OS << " [" << A.getNumElements() << " elements]"; -} - -void DINameSpace::printInternal(raw_ostream &OS) const { - StringRef Name = getName(); - if (!Name.empty()) - OS << " [" << Name << ']'; - - OS << " [line " << getLineNumber() << ']'; -} - -void DISubprogram::printInternal(raw_ostream &OS) const { - // TODO : Print context - OS << " [line " << getLineNumber() << ']'; - - if (isLocalToUnit()) - OS << " [local]"; - - if (isDefinition()) - OS << " [def]"; - - if (getScopeLineNumber() != getLineNumber()) - OS << " [scope " << getScopeLineNumber() << "]"; - - if (isPrivate()) - OS << " [private]"; - else if (isProtected()) - OS << " [protected]"; - else if (isPublic()) - OS << " [public]"; - - if (isLValueReference()) - OS << " [reference]"; - - if (isRValueReference()) - OS << " [rvalue reference]"; - - StringRef Res = getName(); - if (!Res.empty()) - OS << " [" << Res << ']'; -} - -void DIGlobalVariable::printInternal(raw_ostream &OS) const { - StringRef Res = getName(); - if (!Res.empty()) - OS << " [" << Res << ']'; - - OS << " [line " << getLineNumber() << ']'; - - // TODO : Print context - - if (isLocalToUnit()) - OS << " [local]"; - - if (isDefinition()) - OS << " [def]"; -} - -void DIVariable::printInternal(raw_ostream &OS) const { - StringRef Res = getName(); - if (!Res.empty()) - OS << " [" << Res << ']'; - - OS << " [line " << getLineNumber() << ']'; -} - -void DIExpression::printInternal(raw_ostream &OS) const { - for (auto Op : *this) { - OS << " [" << OperationEncodingString(Op); - switch (Op) { - case DW_OP_plus: { - OS << " " << Op.getArg(1); - break; - } - case DW_OP_bit_piece: { - OS << " offset=" << Op.getArg(1) << ", size=" << Op.getArg(2); - break; - } - case DW_OP_deref: - // No arguments. - break; - default: - llvm_unreachable("unhandled operation"); - } - OS << "]"; - } -} - -void DIObjCProperty::printInternal(raw_ostream &OS) const { - StringRef Name = getObjCPropertyName(); - if (!Name.empty()) - OS << " [" << Name << ']'; - - OS << " [line " << getLineNumber() << ", properties " << getUnsignedField(6) - << ']'; + get()->print(OS); } static void printDebugLoc(DebugLoc DL, raw_ostream &CommentOS, diff --git a/lib/IR/DiagnosticInfo.cpp b/lib/IR/DiagnosticInfo.cpp index cfb699a..5608589 100644 --- a/lib/IR/DiagnosticInfo.cpp +++ b/lib/IR/DiagnosticInfo.cpp @@ -129,7 +129,7 @@ void DiagnosticInfoSampleProfile::print(DiagnosticPrinter &DP) const { } bool DiagnosticInfoOptimizationBase::isLocationAvailable() const { - return getDebugLoc().isUnknown() == false; + return !getDebugLoc().isUnknown(); } void DiagnosticInfoOptimizationBase::getLocation(StringRef *Filename, diff --git a/lib/IR/GCOV.cpp b/lib/IR/GCOV.cpp index 08f44e0..7010ceb 100644 --- a/lib/IR/GCOV.cpp +++ b/lib/IR/GCOV.cpp @@ -19,6 +19,7 @@ #include "llvm/Support/Format.h" #include "llvm/Support/MemoryObject.h" #include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" #include #include using namespace llvm; @@ -302,10 +303,12 @@ bool GCOVFunction::readGCDA(GCOVBuffer &Buff, GCOV::GCOVVersion Version) { // required to combine the edge counts that are contained in the GCDA file. for (uint32_t BlockNo = 0; Count > 0; ++BlockNo) { // The last block is always reserved for exit block - if (BlockNo >= Blocks.size() - 1) { + if (BlockNo >= Blocks.size()) { errs() << "Unexpected number of edges (in " << Name << ").\n"; return false; } + if (BlockNo == Blocks.size() - 1) + errs() << "(" << Name << ") has arcs from exit block.\n"; GCOVBlock &Block = *Blocks[BlockNo]; for (size_t EdgeNo = 0, End = Block.getNumDstEdges(); EdgeNo < End; ++EdgeNo) { @@ -443,6 +446,7 @@ static uint32_t branchDiv(uint64_t Numerator, uint64_t Divisor) { return Res; } +namespace { struct formatBranchInfo { formatBranchInfo(const GCOVOptions &Options, uint64_t Count, uint64_t Total) : Options(Options), Count(Count), Total(Total) {} @@ -466,7 +470,6 @@ static raw_ostream &operator<<(raw_ostream &OS, const formatBranchInfo &FBI) { return OS; } -namespace { class LineConsumer { std::unique_ptr Buffer; StringRef Remaining; diff --git a/lib/IR/Globals.cpp b/lib/IR/Globals.cpp index 54197d9..5a6adb3 100644 --- a/lib/IR/Globals.cpp +++ b/lib/IR/Globals.cpp @@ -42,10 +42,6 @@ void GlobalValue::Dematerialize() { getParent()->Dematerialize(this); } -const DataLayout *GlobalValue::getDataLayout() const { - return getParent()->getDataLayout(); -} - /// Override destroyConstant to make sure it doesn't get called on /// GlobalValue's because they shouldn't be treated like other constants. void GlobalValue::destroyConstant() { diff --git a/lib/IR/InlineAsm.cpp b/lib/IR/InlineAsm.cpp index 5b73561..b456d9f 100644 --- a/lib/IR/InlineAsm.cpp +++ b/lib/IR/InlineAsm.cpp @@ -75,7 +75,7 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, ConstraintCodeVector *pCodes = &Codes; // Initialize - isMultipleAlternative = (multipleAlternativeCount > 1 ? true : false); + isMultipleAlternative = multipleAlternativeCount > 1; if (isMultipleAlternative) { multipleAlternatives.resize(multipleAlternativeCount); pCodes = &multipleAlternatives[0].Codes; diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp index 92c6e9f..7d9bd7e 100644 --- a/lib/IR/Instruction.cpp +++ b/lib/IR/Instruction.cpp @@ -32,10 +32,6 @@ Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps, } } -const DataLayout *Instruction::getDataLayout() const { - return getParent()->getDataLayout(); -} - Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps, BasicBlock *InsertAtEnd) : User(ty, Value::InstructionVal + it, Ops, NumOps), Parent(nullptr) { @@ -58,6 +54,10 @@ void Instruction::setParent(BasicBlock *P) { Parent = P; } +const Module *Instruction::getModule() const { + return getParent()->getModule(); +} + void Instruction::removeFromParent() { getParent()->getInstList().remove(this); } diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp index 7136923..af2aeb9 100644 --- a/lib/IR/Instructions.cpp +++ b/lib/IR/Instructions.cpp @@ -841,41 +841,19 @@ static Value *getAISize(LLVMContext &Context, Value *Amt) { return Amt; } -AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, - const Twine &Name, Instruction *InsertBefore) - : UnaryInstruction(PointerType::getUnqual(Ty), Alloca, - getAISize(Ty->getContext(), ArraySize), InsertBefore) { - setAlignment(0); - assert(!Ty->isVoidTy() && "Cannot allocate void!"); - setName(Name); -} +AllocaInst::AllocaInst(Type *Ty, const Twine &Name, Instruction *InsertBefore) + : AllocaInst(Ty, /*ArraySize=*/nullptr, Name, InsertBefore) {} -AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, - const Twine &Name, BasicBlock *InsertAtEnd) - : UnaryInstruction(PointerType::getUnqual(Ty), Alloca, - getAISize(Ty->getContext(), ArraySize), InsertAtEnd) { - setAlignment(0); - assert(!Ty->isVoidTy() && "Cannot allocate void!"); - setName(Name); -} +AllocaInst::AllocaInst(Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd) + : AllocaInst(Ty, /*ArraySize=*/nullptr, Name, InsertAtEnd) {} -AllocaInst::AllocaInst(Type *Ty, const Twine &Name, +AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, const Twine &Name, Instruction *InsertBefore) - : UnaryInstruction(PointerType::getUnqual(Ty), Alloca, - getAISize(Ty->getContext(), nullptr), InsertBefore) { - setAlignment(0); - assert(!Ty->isVoidTy() && "Cannot allocate void!"); - setName(Name); -} + : AllocaInst(Ty, ArraySize, /*Align=*/0, Name, InsertBefore) {} -AllocaInst::AllocaInst(Type *Ty, const Twine &Name, +AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, const Twine &Name, BasicBlock *InsertAtEnd) - : UnaryInstruction(PointerType::getUnqual(Ty), Alloca, - getAISize(Ty->getContext(), nullptr), InsertAtEnd) { - setAlignment(0); - assert(!Ty->isVoidTy() && "Cannot allocate void!"); - setName(Name); -} + : AllocaInst(Ty, ArraySize, /*Align=*/0, Name, InsertAtEnd) {} AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align, const Twine &Name, Instruction *InsertBefore) @@ -942,67 +920,27 @@ void LoadInst::AssertOK() { } LoadInst::LoadInst(Value *Ptr, const Twine &Name, Instruction *InsertBef) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertBef) { - setVolatile(false); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); - setName(Name); -} + : LoadInst(Ptr, Name, /*isVolatile=*/false, InsertBef) {} LoadInst::LoadInst(Value *Ptr, const Twine &Name, BasicBlock *InsertAE) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertAE) { - setVolatile(false); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); - setName(Name); -} + : LoadInst(Ptr, Name, /*isVolatile=*/false, InsertAE) {} LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, Instruction *InsertBef) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertBef) { - setVolatile(isVolatile); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); - setName(Name); -} + : LoadInst(Ptr, Name, isVolatile, /*Align=*/0, InsertBef) {} LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, BasicBlock *InsertAE) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertAE) { - setVolatile(isVolatile); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); - setName(Name); -} + : LoadInst(Ptr, Name, isVolatile, /*Align=*/0, InsertAE) {} -LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, +LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, Instruction *InsertBef) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertBef) { - setVolatile(isVolatile); - setAlignment(Align); - setAtomic(NotAtomic); - AssertOK(); - setName(Name); -} + : LoadInst(Ptr, Name, isVolatile, Align, NotAtomic, CrossThread, + InsertBef) {} -LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, +LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, BasicBlock *InsertAE) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertAE) { - setVolatile(isVolatile); - setAlignment(Align); - setAtomic(NotAtomic); - AssertOK(); - setName(Name); + : LoadInst(Ptr, Name, isVolatile, Align, NotAtomic, CrossThread, InsertAE) { } LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, @@ -1097,60 +1035,29 @@ void StoreInst::AssertOK() { "Alignment required for atomic store"); } - StoreInst::StoreInst(Value *val, Value *addr, Instruction *InsertBefore) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertBefore) { - Op<0>() = val; - Op<1>() = addr; - setVolatile(false); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); -} + : StoreInst(val, addr, /*isVolatile=*/false, InsertBefore) {} StoreInst::StoreInst(Value *val, Value *addr, BasicBlock *InsertAtEnd) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertAtEnd) { - Op<0>() = val; - Op<1>() = addr; - setVolatile(false); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); -} + : StoreInst(val, addr, /*isVolatile=*/false, InsertAtEnd) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Instruction *InsertBefore) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertBefore) { - Op<0>() = val; - Op<1>() = addr; - setVolatile(isVolatile); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); -} + : StoreInst(val, addr, isVolatile, /*Align=*/0, InsertBefore) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, - unsigned Align, Instruction *InsertBefore) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertBefore) { - Op<0>() = val; - Op<1>() = addr; - setVolatile(isVolatile); - setAlignment(Align); - setAtomic(NotAtomic); - AssertOK(); -} + BasicBlock *InsertAtEnd) + : StoreInst(val, addr, isVolatile, /*Align=*/0, InsertAtEnd) {} + +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, + Instruction *InsertBefore) + : StoreInst(val, addr, isVolatile, Align, NotAtomic, CrossThread, + InsertBefore) {} + +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, + BasicBlock *InsertAtEnd) + : StoreInst(val, addr, isVolatile, Align, NotAtomic, CrossThread, + InsertAtEnd) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, AtomicOrdering Order, @@ -1169,34 +1076,6 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, } StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, - BasicBlock *InsertAtEnd) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertAtEnd) { - Op<0>() = val; - Op<1>() = addr; - setVolatile(isVolatile); - setAlignment(0); - setAtomic(NotAtomic); - AssertOK(); -} - -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, - unsigned Align, BasicBlock *InsertAtEnd) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertAtEnd) { - Op<0>() = val; - Op<1>() = addr; - setVolatile(isVolatile); - setAlignment(Align); - setAtomic(NotAtomic); - AssertOK(); -} - -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, AtomicOrdering Order, SynchronizationScope SynchScope, BasicBlock *InsertAtEnd) @@ -2169,21 +2048,15 @@ bool CastInst::isNoopCast(Type *IntPtrTy) const { return isNoopCast(getOpcode(), getOperand(0)->getType(), getType(), IntPtrTy); } -bool CastInst::isNoopCast(const DataLayout *DL) const { - if (!DL) { - // Assume maximum pointer size. - return isNoopCast(Type::getInt64Ty(getContext())); - } - +bool CastInst::isNoopCast(const DataLayout &DL) const { Type *PtrOpTy = nullptr; if (getOpcode() == Instruction::PtrToInt) PtrOpTy = getOperand(0)->getType(); else if (getOpcode() == Instruction::IntToPtr) PtrOpTy = getType(); - Type *IntPtrTy = PtrOpTy - ? DL->getIntPtrType(PtrOpTy) - : DL->getIntPtrType(getContext(), 0); + Type *IntPtrTy = + PtrOpTy ? DL.getIntPtrType(PtrOpTy) : DL.getIntPtrType(getContext(), 0); return isNoopCast(getOpcode(), getOperand(0)->getType(), getType(), IntPtrTy); } @@ -2656,44 +2529,38 @@ bool CastInst::isCastable(Type *SrcTy, Type *DestTy) { // Run through the possibilities ... if (DestTy->isIntegerTy()) { // Casting to integral - if (SrcTy->isIntegerTy()) { // Casting from integral + if (SrcTy->isIntegerTy()) // Casting from integral return true; - } else if (SrcTy->isFloatingPointTy()) { // Casting from floating pt + if (SrcTy->isFloatingPointTy()) // Casting from floating pt return true; - } else if (SrcTy->isVectorTy()) { // Casting from vector + if (SrcTy->isVectorTy()) // Casting from vector return DestBits == SrcBits; - } else { // Casting from something else - return SrcTy->isPointerTy(); - } - } else if (DestTy->isFloatingPointTy()) { // Casting to floating pt - if (SrcTy->isIntegerTy()) { // Casting from integral + // Casting from something else + return SrcTy->isPointerTy(); + } + if (DestTy->isFloatingPointTy()) { // Casting to floating pt + if (SrcTy->isIntegerTy()) // Casting from integral return true; - } else if (SrcTy->isFloatingPointTy()) { // Casting from floating pt + if (SrcTy->isFloatingPointTy()) // Casting from floating pt return true; - } else if (SrcTy->isVectorTy()) { // Casting from vector + if (SrcTy->isVectorTy()) // Casting from vector return DestBits == SrcBits; - } else { // Casting from something else - return false; - } - } else if (DestTy->isVectorTy()) { // Casting to vector + // Casting from something else + return false; + } + if (DestTy->isVectorTy()) // Casting to vector return DestBits == SrcBits; - } else if (DestTy->isPointerTy()) { // Casting to pointer - if (SrcTy->isPointerTy()) { // Casting from pointer + if (DestTy->isPointerTy()) { // Casting to pointer + if (SrcTy->isPointerTy()) // Casting from pointer return true; - } else if (SrcTy->isIntegerTy()) { // Casting from integral - return true; - } else { // Casting from something else - return false; - } - } else if (DestTy->isX86_MMXTy()) { - if (SrcTy->isVectorTy()) { + return SrcTy->isIntegerTy(); // Casting from integral + } + if (DestTy->isX86_MMXTy()) { + if (SrcTy->isVectorTy()) return DestBits == SrcBits; // 64-bit vector to MMX - } else { - return false; - } - } else { // Casting to something else return false; - } + } // Casting to something else + return false; } bool CastInst::isBitCastable(Type *SrcTy, Type *DestTy) { @@ -2737,13 +2604,13 @@ bool CastInst::isBitCastable(Type *SrcTy, Type *DestTy) { } bool CastInst::isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, - const DataLayout *DL) { + const DataLayout &DL) { if (auto *PtrTy = dyn_cast(SrcTy)) if (auto *IntTy = dyn_cast(DestTy)) - return DL && IntTy->getBitWidth() == DL->getPointerTypeSizeInBits(PtrTy); + return IntTy->getBitWidth() == DL.getPointerTypeSizeInBits(PtrTy); if (auto *PtrTy = dyn_cast(DestTy)) if (auto *IntTy = dyn_cast(SrcTy)) - return DL && IntTy->getBitWidth() == DL->getPointerTypeSizeInBits(PtrTy); + return IntTy->getBitWidth() == DL.getPointerTypeSizeInBits(PtrTy); return isBitCastable(SrcTy, DestTy); } diff --git a/lib/IR/LLVMContextImpl.h b/lib/IR/LLVMContextImpl.h index 4631246..e380665 100644 --- a/lib/IR/LLVMContextImpl.h +++ b/lib/IR/LLVMContextImpl.h @@ -240,12 +240,12 @@ template <> struct MDNodeKeyImpl { : Line(Line), Column(Column), Scope(Scope), InlinedAt(InlinedAt) {} MDNodeKeyImpl(const MDLocation *L) - : Line(L->getLine()), Column(L->getColumn()), Scope(L->getScope()), - InlinedAt(L->getInlinedAt()) {} + : Line(L->getLine()), Column(L->getColumn()), Scope(L->getRawScope()), + InlinedAt(L->getRawInlinedAt()) {} bool isKeyOf(const MDLocation *RHS) const { return Line == RHS->getLine() && Column == RHS->getColumn() && - Scope == RHS->getScope() && InlinedAt == RHS->getInlinedAt(); + Scope == RHS->getRawScope() && InlinedAt == RHS->getRawInlinedAt(); } unsigned getHashValue() const { return hash_combine(Line, Column, Scope, InlinedAt); diff --git a/lib/IR/LegacyPassManager.cpp b/lib/IR/LegacyPassManager.cpp index fa8d50e..9a365d1 100644 --- a/lib/IR/LegacyPassManager.cpp +++ b/lib/IR/LegacyPassManager.cpp @@ -652,7 +652,7 @@ void PMTopLevelManager::schedulePass(Pass *P) { // are already checked are still available. checkAnalysis = true; } else - // Do not schedule this analysis. Lower level analsyis + // Do not schedule this analysis. Lower level analysis // passes are run on the fly. delete AnalysisPass; } diff --git a/lib/IR/Mangler.cpp b/lib/IR/Mangler.cpp index 5eeb797..a0e1b25 100644 --- a/lib/IR/Mangler.cpp +++ b/lib/IR/Mangler.cpp @@ -73,7 +73,7 @@ static bool hasByteCountSuffix(CallingConv::ID CC) { /// Microsoft fastcall and stdcall functions require a suffix on their name /// indicating the number of words of arguments they take. static void addByteCountSuffix(raw_ostream &OS, const Function *F, - const DataLayout &TD) { + const DataLayout &DL) { // Calculate arguments size total. unsigned ArgWords = 0; for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end(); @@ -83,8 +83,8 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F, if (AI->hasByValOrInAllocaAttr()) Ty = cast(Ty)->getElementType(); // Size should be aligned to pointer size. - unsigned PtrSize = TD.getPointerSize(); - ArgWords += RoundUpToAlignment(TD.getTypeAllocSize(Ty), PtrSize); + unsigned PtrSize = DL.getPointerSize(); + ArgWords += RoundUpToAlignment(DL.getTypeAllocSize(Ty), PtrSize); } OS << '@' << ArgWords; diff --git a/lib/IR/Module.cpp b/lib/IR/Module.cpp index b0abe8c..3e8f91f 100644 --- a/lib/IR/Module.cpp +++ b/lib/IR/Module.cpp @@ -365,31 +365,11 @@ void Module::addModuleFlag(MDNode *Node) { void Module::setDataLayout(StringRef Desc) { DL.reset(Desc); - - if (Desc.empty()) { - DataLayoutStr = ""; - } else { - DataLayoutStr = DL.getStringRepresentation(); - // DataLayoutStr is now equivalent to Desc, but since the representation - // is not unique, they may not be identical. - } } -void Module::setDataLayout(const DataLayout *Other) { - if (!Other) { - DataLayoutStr = ""; - DL.reset(""); - } else { - DL = *Other; - DataLayoutStr = DL.getStringRepresentation(); - } -} +void Module::setDataLayout(const DataLayout &Other) { DL = Other; } -const DataLayout *Module::getDataLayout() const { - if (DataLayoutStr.empty()) - return nullptr; - return &DL; -} +const DataLayout &Module::getDataLayout() const { return DL; } //===----------------------------------------------------------------------===// // Methods to control the materialization of GlobalValues in the Module. @@ -433,6 +413,12 @@ std::error_code Module::materializeAllPermanently() { return std::error_code(); } +std::error_code Module::materializeMetadata() { + if (!Materializer) + return std::error_code(); + return Materializer->materializeMetadata(); +} + //===----------------------------------------------------------------------===// // Other module related stuff. // diff --git a/lib/IR/TypeFinder.cpp b/lib/IR/TypeFinder.cpp index e2fb8f8..1d2b808 100644 --- a/lib/IR/TypeFinder.cpp +++ b/lib/IR/TypeFinder.cpp @@ -68,7 +68,7 @@ void TypeFinder::run(const Module &M, bool onlyNamed) { // instructions with this loop.) for (User::const_op_iterator OI = I.op_begin(), OE = I.op_end(); OI != OE; ++OI) - if (!isa(OI)) + if (*OI && !isa(OI)) incorporateValue(*OI); // Incorporate types hiding in metadata. diff --git a/lib/IR/Value.cpp b/lib/IR/Value.cpp index 7d205f9..78bfca4 100644 --- a/lib/IR/Value.cpp +++ b/lib/IR/Value.cpp @@ -32,6 +32,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/raw_ostream.h" #include using namespace llvm; @@ -69,15 +70,13 @@ Value::~Value() { #ifndef NDEBUG // Only in -g mode... // Check to make sure that there are no uses of this value that are still // around when the value is destroyed. If there are, then we have a dangling - // reference and something is wrong. This code is here to print out what is - // still being referenced. The value in question should be printed as - // a + // reference and something is wrong. This code is here to print out where + // the value is still being referenced. // if (!use_empty()) { dbgs() << "While deleting: " << *VTy << " %" << getName() << "\n"; - for (use_iterator I = use_begin(), E = use_end(); I != E; ++I) - dbgs() << "Use still stuck around after Def is destroyed:" - << **I << "\n"; + for (auto *U : users()) + dbgs() << "Use still stuck around after Def is destroyed:" << *U << "\n"; } #endif assert(use_empty() && "Uses remain when a value is destroyed!"); @@ -482,7 +481,7 @@ Value *Value::stripInBoundsOffsets() { /// /// Test if V is always a pointer to allocated and suitably aligned memory for /// a simple load or store. -static bool isDereferenceablePointer(const Value *V, const DataLayout *DL, +static bool isDereferenceablePointer(const Value *V, const DataLayout &DL, SmallPtrSetImpl &Visited) { // Note that it is not safe to speculate into a malloc'd region because // malloc may return null. @@ -497,17 +496,14 @@ static bool isDereferenceablePointer(const Value *V, const DataLayout *DL, // to a type of smaller size (or the same size), and the alignment // is at least as large as for the resulting pointer type, then // we can look through the bitcast. - if (DL) - if (const BitCastOperator *BC = dyn_cast(V)) { - Type *STy = BC->getSrcTy()->getPointerElementType(), - *DTy = BC->getDestTy()->getPointerElementType(); - if (STy->isSized() && DTy->isSized() && - (DL->getTypeStoreSize(STy) >= - DL->getTypeStoreSize(DTy)) && - (DL->getABITypeAlignment(STy) >= - DL->getABITypeAlignment(DTy))) - return isDereferenceablePointer(BC->getOperand(0), DL, Visited); - } + if (const BitCastOperator *BC = dyn_cast(V)) { + Type *STy = BC->getSrcTy()->getPointerElementType(), + *DTy = BC->getDestTy()->getPointerElementType(); + if (STy->isSized() && DTy->isSized() && + (DL.getTypeStoreSize(STy) >= DL.getTypeStoreSize(DTy)) && + (DL.getABITypeAlignment(STy) >= DL.getABITypeAlignment(DTy))) + return isDereferenceablePointer(BC->getOperand(0), DL, Visited); + } // Global variables which can't collapse to null are ok. if (const GlobalVariable *GV = dyn_cast(V)) @@ -520,7 +516,7 @@ static bool isDereferenceablePointer(const Value *V, const DataLayout *DL, return true; else if (uint64_t Bytes = A->getDereferenceableBytes()) { Type *Ty = V->getType()->getPointerElementType(); - if (Ty->isSized() && DL && DL->getTypeStoreSize(Ty) <= Bytes) + if (Ty->isSized() && DL.getTypeStoreSize(Ty) <= Bytes) return true; } @@ -532,7 +528,7 @@ static bool isDereferenceablePointer(const Value *V, const DataLayout *DL, if (ImmutableCallSite CS = V) { if (uint64_t Bytes = CS.getDereferenceableBytes(0)) { Type *Ty = V->getType()->getPointerElementType(); - if (Ty->isSized() && DL && DL->getTypeStoreSize(Ty) <= Bytes) + if (Ty->isSized() && DL.getTypeStoreSize(Ty) <= Bytes) return true; } } @@ -586,15 +582,15 @@ static bool isDereferenceablePointer(const Value *V, const DataLayout *DL, return false; } -bool Value::isDereferenceablePointer(const DataLayout *DL) const { +bool Value::isDereferenceablePointer(const DataLayout &DL) const { // When dereferenceability information is provided by a dereferenceable // attribute, we know exactly how many bytes are dereferenceable. If we can // determine the exact offset to the attributed variable, we can use that // information here. Type *Ty = getType()->getPointerElementType(); - if (Ty->isSized() && DL) { - APInt Offset(DL->getTypeStoreSizeInBits(getType()), 0); - const Value *BV = stripAndAccumulateInBoundsConstantOffsets(*DL, Offset); + if (Ty->isSized()) { + APInt Offset(DL.getTypeStoreSizeInBits(getType()), 0); + const Value *BV = stripAndAccumulateInBoundsConstantOffsets(DL, Offset); APInt DerefBytes(Offset.getBitWidth(), 0); if (const Argument *A = dyn_cast(BV)) @@ -603,7 +599,7 @@ bool Value::isDereferenceablePointer(const DataLayout *DL) const { DerefBytes = CS.getDereferenceableBytes(0); if (DerefBytes.getBoolValue() && Offset.isNonNegative()) { - if (DerefBytes.uge(Offset + DL->getTypeStoreSize(Ty))) + if (DerefBytes.uge(Offset + DL.getTypeStoreSize(Ty))) return true; } } diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index d01e138..fcf48c4 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -78,7 +78,7 @@ #include using namespace llvm; -static cl::opt VerifyDebugInfo("verify-debug-info", cl::init(false)); +static cl::opt VerifyDebugInfo("verify-debug-info", cl::init(true)); namespace { struct VerifierSupport { @@ -87,11 +87,13 @@ struct VerifierSupport { /// \brief Track the brokenness of the module while recursively visiting. bool Broken; + bool EverBroken; explicit VerifierSupport(raw_ostream &OS) - : OS(OS), M(nullptr), Broken(false) {} + : OS(OS), M(nullptr), Broken(false), EverBroken(false) {} - void WriteValue(const Value *V) { +private: + void Write(const Value *V) { if (!V) return; if (isa(V)) { @@ -102,81 +104,61 @@ struct VerifierSupport { } } - void WriteMetadata(const Metadata *MD) { + void Write(const Metadata *MD) { if (!MD) return; - MD->printAsOperand(OS, true, M); + MD->print(OS, M); + OS << '\n'; + } + + void Write(const NamedMDNode *NMD) { + if (!NMD) + return; + NMD->print(OS); OS << '\n'; } - void WriteType(Type *T) { + void Write(Type *T) { if (!T) return; OS << ' ' << *T; } - void WriteComdat(const Comdat *C) { + void Write(const Comdat *C) { if (!C) return; OS << *C; } - // CheckFailed - A check failed, so print out the condition and the message - // that failed. This provides a nice place to put a breakpoint if you want - // to see why something is not correct. - void CheckFailed(const Twine &Message, const Value *V1 = nullptr, - const Value *V2 = nullptr, const Value *V3 = nullptr, - const Value *V4 = nullptr) { - OS << Message.str() << "\n"; - WriteValue(V1); - WriteValue(V2); - WriteValue(V3); - WriteValue(V4); - Broken = true; - } - - void CheckFailed(const Twine &Message, const Metadata *V1, const Metadata *V2, - const Metadata *V3 = nullptr, const Metadata *V4 = nullptr) { - OS << Message.str() << "\n"; - WriteMetadata(V1); - WriteMetadata(V2); - WriteMetadata(V3); - WriteMetadata(V4); - Broken = true; - } - - void CheckFailed(const Twine &Message, const Metadata *V1, - const Value *V2 = nullptr) { - OS << Message.str() << "\n"; - WriteMetadata(V1); - WriteValue(V2); - Broken = true; - } - - void CheckFailed(const Twine &Message, const Value *V1, Type *T2, - const Value *V3 = nullptr) { - OS << Message.str() << "\n"; - WriteValue(V1); - WriteType(T2); - WriteValue(V3); - Broken = true; - } - - void CheckFailed(const Twine &Message, Type *T1, Type *T2 = nullptr, - Type *T3 = nullptr) { - OS << Message.str() << "\n"; - WriteType(T1); - WriteType(T2); - WriteType(T3); - Broken = true; - } - - void CheckFailed(const Twine &Message, const Comdat *C) { - OS << Message.str() << "\n"; - WriteComdat(C); - Broken = true; + template + void WriteTs(const T1 &V1, const Ts &... Vs) { + Write(V1); + WriteTs(Vs...); + } + + template void WriteTs() {} + +public: + /// \brief A check failed, so printout out the condition and the message. + /// + /// This provides a nice place to put a breakpoint if you want to see why + /// something is not correct. + void CheckFailed(const Twine &Message) { + OS << Message << '\n'; + EverBroken = Broken = true; + } + + /// \brief A check failed (with values to print). + /// + /// This calls the Message-only version so that the above is easier to set a + /// breakpoint on. + template + void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) { + CheckFailed(Message); + WriteTs(V1, Vs...); } }; + class Verifier : public InstVisitor, VerifierSupport { friend class InstVisitor; @@ -198,14 +180,18 @@ class Verifier : public InstVisitor, VerifierSupport { /// personality function. const Value *PersonalityFn; - /// \brief Whether we've seen a call to @llvm.frameallocate in this function + /// \brief Whether we've seen a call to @llvm.frameescape in this function /// already. - bool SawFrameAllocate; + bool SawFrameEscape; + + /// Stores the count of how many objects were passed to llvm.frameescape for a + /// given function and the largest index passed to llvm.framerecover. + DenseMap> FrameEscapeInfo; public: - explicit Verifier(raw_ostream &OS = dbgs()) + explicit Verifier(raw_ostream &OS) : VerifierSupport(OS), Context(nullptr), PersonalityFn(nullptr), - SawFrameAllocate(false) {} + SawFrameEscape(false) {} bool verify(const Function &F) { M = F.getParent(); @@ -240,7 +226,7 @@ public: visit(const_cast(F)); InstsInThisBlock.clear(); PersonalityFn = nullptr; - SawFrameAllocate = false; + SawFrameEscape = false; return !Broken; } @@ -259,6 +245,10 @@ public: visitFunction(*I); } + // Now that we've visited every function, verify that we never asked to + // recover a frame index that wasn't escaped. + verifyFrameRecoverIndices(); + for (Module::const_global_iterator I = M.global_begin(), E = M.global_end(); I != E; ++I) visitGlobalVariable(*I); @@ -278,6 +268,9 @@ public: visitModuleFlags(M); visitModuleIdents(M); + // Verify debug info last. + verifyDebugInfo(); + return !Broken; } @@ -347,6 +340,8 @@ private: void visitUserOp1(Instruction &I); void visitUserOp2(Instruction &I) { visitUserOp1(I); } void visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI); + template + void visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII); void visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI); void visitAtomicRMWInst(AtomicRMWInst &RMWI); void visitFenceInst(FenceInst &FI); @@ -373,18 +368,9 @@ private: void VerifyConstantExprBitcastType(const ConstantExpr *CE); void VerifyStatepoint(ImmutableCallSite CS); -}; -class DebugInfoVerifier : public VerifierSupport { -public: - explicit DebugInfoVerifier(raw_ostream &OS = dbgs()) : VerifierSupport(OS) {} + void verifyFrameRecoverIndices(); - bool verify(const Module &M) { - this->M = &M; - verifyDebugInfo(); - return !Broken; - } - -private: + // Module-level debug info verification... void verifyDebugInfo(); void processInstructions(DebugInfoFinder &Finder); void processCallInst(DebugInfoFinder &Finder, const CallInst &CI); @@ -392,66 +378,58 @@ private: } // End anonymous namespace // Assert - We know that cond should be true, if not print an error message. -#define Assert(C, M) \ - do { if (!(C)) { CheckFailed(M); return; } } while (0) -#define Assert1(C, M, V1) \ - do { if (!(C)) { CheckFailed(M, V1); return; } } while (0) -#define Assert2(C, M, V1, V2) \ - do { if (!(C)) { CheckFailed(M, V1, V2); return; } } while (0) -#define Assert3(C, M, V1, V2, V3) \ - do { if (!(C)) { CheckFailed(M, V1, V2, V3); return; } } while (0) -#define Assert4(C, M, V1, V2, V3, V4) \ - do { if (!(C)) { CheckFailed(M, V1, V2, V3, V4); return; } } while (0) +#define Assert(C, ...) \ + do { if (!(C)) { CheckFailed(__VA_ARGS__); return; } } while (0) void Verifier::visit(Instruction &I) { for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) - Assert1(I.getOperand(i) != nullptr, "Operand is null", &I); + Assert(I.getOperand(i) != nullptr, "Operand is null", &I); InstVisitor::visit(I); } void Verifier::visitGlobalValue(const GlobalValue &GV) { - Assert1(!GV.isDeclaration() || GV.hasExternalLinkage() || - GV.hasExternalWeakLinkage(), - "Global is external, but doesn't have external or weak linkage!", - &GV); + Assert(!GV.isDeclaration() || GV.hasExternalLinkage() || + GV.hasExternalWeakLinkage(), + "Global is external, but doesn't have external or weak linkage!", &GV); - Assert1(GV.getAlignment() <= Value::MaximumAlignment, - "huge alignment values are unsupported", &GV); - Assert1(!GV.hasAppendingLinkage() || isa(GV), - "Only global variables can have appending linkage!", &GV); + Assert(GV.getAlignment() <= Value::MaximumAlignment, + "huge alignment values are unsupported", &GV); + Assert(!GV.hasAppendingLinkage() || isa(GV), + "Only global variables can have appending linkage!", &GV); if (GV.hasAppendingLinkage()) { const GlobalVariable *GVar = dyn_cast(&GV); - Assert1(GVar && GVar->getType()->getElementType()->isArrayTy(), - "Only global arrays can have appending linkage!", GVar); + Assert(GVar && GVar->getType()->getElementType()->isArrayTy(), + "Only global arrays can have appending linkage!", GVar); } } void Verifier::visitGlobalVariable(const GlobalVariable &GV) { if (GV.hasInitializer()) { - Assert1(GV.getInitializer()->getType() == GV.getType()->getElementType(), - "Global variable initializer type does not match global " - "variable type!", &GV); + Assert(GV.getInitializer()->getType() == GV.getType()->getElementType(), + "Global variable initializer type does not match global " + "variable type!", + &GV); // If the global has common linkage, it must have a zero initializer and // cannot be constant. if (GV.hasCommonLinkage()) { - Assert1(GV.getInitializer()->isNullValue(), - "'common' global must have a zero initializer!", &GV); - Assert1(!GV.isConstant(), "'common' global may not be marked constant!", - &GV); - Assert1(!GV.hasComdat(), "'common' global may not be in a Comdat!", &GV); + Assert(GV.getInitializer()->isNullValue(), + "'common' global must have a zero initializer!", &GV); + Assert(!GV.isConstant(), "'common' global may not be marked constant!", + &GV); + Assert(!GV.hasComdat(), "'common' global may not be in a Comdat!", &GV); } } else { - Assert1(GV.hasExternalLinkage() || GV.hasExternalWeakLinkage(), - "invalid linkage type for global declaration", &GV); + Assert(GV.hasExternalLinkage() || GV.hasExternalWeakLinkage(), + "invalid linkage type for global declaration", &GV); } if (GV.hasName() && (GV.getName() == "llvm.global_ctors" || GV.getName() == "llvm.global_dtors")) { - Assert1(!GV.hasInitializer() || GV.hasAppendingLinkage(), - "invalid linkage for intrinsic global variable", &GV); + Assert(!GV.hasInitializer() || GV.hasAppendingLinkage(), + "invalid linkage for intrinsic global variable", &GV); // Don't worry about emitting an error for it not being an array, // visitGlobalValue will complain on appending non-array. if (ArrayType *ATy = dyn_cast(GV.getType()->getElementType())) { @@ -459,48 +437,48 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { PointerType *FuncPtrTy = FunctionType::get(Type::getVoidTy(*Context), false)->getPointerTo(); // FIXME: Reject the 2-field form in LLVM 4.0. - Assert1(STy && (STy->getNumElements() == 2 || - STy->getNumElements() == 3) && - STy->getTypeAtIndex(0u)->isIntegerTy(32) && - STy->getTypeAtIndex(1) == FuncPtrTy, - "wrong type for intrinsic global variable", &GV); + Assert(STy && + (STy->getNumElements() == 2 || STy->getNumElements() == 3) && + STy->getTypeAtIndex(0u)->isIntegerTy(32) && + STy->getTypeAtIndex(1) == FuncPtrTy, + "wrong type for intrinsic global variable", &GV); if (STy->getNumElements() == 3) { Type *ETy = STy->getTypeAtIndex(2); - Assert1(ETy->isPointerTy() && - cast(ETy)->getElementType()->isIntegerTy(8), - "wrong type for intrinsic global variable", &GV); + Assert(ETy->isPointerTy() && + cast(ETy)->getElementType()->isIntegerTy(8), + "wrong type for intrinsic global variable", &GV); } } } if (GV.hasName() && (GV.getName() == "llvm.used" || GV.getName() == "llvm.compiler.used")) { - Assert1(!GV.hasInitializer() || GV.hasAppendingLinkage(), - "invalid linkage for intrinsic global variable", &GV); + Assert(!GV.hasInitializer() || GV.hasAppendingLinkage(), + "invalid linkage for intrinsic global variable", &GV); Type *GVType = GV.getType()->getElementType(); if (ArrayType *ATy = dyn_cast(GVType)) { PointerType *PTy = dyn_cast(ATy->getElementType()); - Assert1(PTy, "wrong type for intrinsic global variable", &GV); + Assert(PTy, "wrong type for intrinsic global variable", &GV); if (GV.hasInitializer()) { const Constant *Init = GV.getInitializer(); const ConstantArray *InitArray = dyn_cast(Init); - Assert1(InitArray, "wrong initalizer for intrinsic global variable", - Init); + Assert(InitArray, "wrong initalizer for intrinsic global variable", + Init); for (unsigned i = 0, e = InitArray->getNumOperands(); i != e; ++i) { Value *V = Init->getOperand(i)->stripPointerCastsNoFollowAliases(); - Assert1( - isa(V) || isa(V) || isa(V), - "invalid llvm.used member", V); - Assert1(V->hasName(), "members of llvm.used must be named", V); + Assert(isa(V) || isa(V) || + isa(V), + "invalid llvm.used member", V); + Assert(V->hasName(), "members of llvm.used must be named", V); } } } } - Assert1(!GV.hasDLLImportStorageClass() || - (GV.isDeclaration() && GV.hasExternalLinkage()) || - GV.hasAvailableExternallyLinkage(), - "Global is marked as dllimport, but not external", &GV); + Assert(!GV.hasDLLImportStorageClass() || + (GV.isDeclaration() && GV.hasExternalLinkage()) || + GV.hasAvailableExternallyLinkage(), + "Global is marked as dllimport, but not external", &GV); if (!GV.hasInitializer()) { visitGlobalValue(GV); @@ -540,13 +518,13 @@ void Verifier::visitAliaseeSubExpr(const GlobalAlias &GA, const Constant &C) { void Verifier::visitAliaseeSubExpr(SmallPtrSetImpl &Visited, const GlobalAlias &GA, const Constant &C) { if (const auto *GV = dyn_cast(&C)) { - Assert1(!GV->isDeclaration(), "Alias must point to a definition", &GA); + Assert(!GV->isDeclaration(), "Alias must point to a definition", &GA); if (const auto *GA2 = dyn_cast(GV)) { - Assert1(Visited.insert(GA2).second, "Aliases cannot form a cycle", &GA); + Assert(Visited.insert(GA2).second, "Aliases cannot form a cycle", &GA); - Assert1(!GA2->mayBeOverridden(), "Alias cannot point to a weak alias", - &GA); + Assert(!GA2->mayBeOverridden(), "Alias cannot point to a weak alias", + &GA); } else { // Only continue verifying subexpressions of GlobalAliases. // Do not recurse into global initializers. @@ -567,19 +545,18 @@ void Verifier::visitAliaseeSubExpr(SmallPtrSetImpl &Visited, } void Verifier::visitGlobalAlias(const GlobalAlias &GA) { - Assert1(!GA.getName().empty(), - "Alias name cannot be empty!", &GA); - Assert1(GlobalAlias::isValidLinkage(GA.getLinkage()), - "Alias should have private, internal, linkonce, weak, linkonce_odr, " - "weak_odr, or external linkage!", - &GA); + Assert(!GA.getName().empty(), "Alias name cannot be empty!", &GA); + Assert(GlobalAlias::isValidLinkage(GA.getLinkage()), + "Alias should have private, internal, linkonce, weak, linkonce_odr, " + "weak_odr, or external linkage!", + &GA); const Constant *Aliasee = GA.getAliasee(); - Assert1(Aliasee, "Aliasee cannot be NULL!", &GA); - Assert1(GA.getType() == Aliasee->getType(), - "Alias and aliasee types should match!", &GA); + Assert(Aliasee, "Aliasee cannot be NULL!", &GA); + Assert(GA.getType() == Aliasee->getType(), + "Alias and aliasee types should match!", &GA); - Assert1(isa(Aliasee) || isa(Aliasee), - "Aliasee should be either GlobalValue or ConstantExpr", &GA); + Assert(isa(Aliasee) || isa(Aliasee), + "Aliasee should be either GlobalValue or ConstantExpr", &GA); visitAliaseeSubExpr(GA, *Aliasee); @@ -592,6 +569,10 @@ void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { if (!MD) continue; + if (NMD.getName() == "llvm.dbg.cu") { + Assert(isa(MD), "invalid compile unit", &NMD, MD); + } + visitMDNode(*MD); } } @@ -618,8 +599,8 @@ void Verifier::visitMDNode(const MDNode &MD) { Metadata *Op = MD.getOperand(i); if (!Op) continue; - Assert2(!isa(Op), "Invalid operand for global metadata!", - &MD, Op); + Assert(!isa(Op), "Invalid operand for global metadata!", + &MD, Op); if (auto *N = dyn_cast(Op)) { visitMDNode(*N); continue; @@ -631,26 +612,26 @@ void Verifier::visitMDNode(const MDNode &MD) { } // Check these last, so we diagnose problems in operands first. - Assert1(!MD.isTemporary(), "Expected no forward declarations!", &MD); - Assert1(MD.isResolved(), "All nodes should be resolved!", &MD); + Assert(!MD.isTemporary(), "Expected no forward declarations!", &MD); + Assert(MD.isResolved(), "All nodes should be resolved!", &MD); } void Verifier::visitValueAsMetadata(const ValueAsMetadata &MD, Function *F) { - Assert1(MD.getValue(), "Expected valid value", &MD); - Assert2(!MD.getValue()->getType()->isMetadataTy(), - "Unexpected metadata round-trip through values", &MD, MD.getValue()); + Assert(MD.getValue(), "Expected valid value", &MD); + Assert(!MD.getValue()->getType()->isMetadataTy(), + "Unexpected metadata round-trip through values", &MD, MD.getValue()); auto *L = dyn_cast(&MD); if (!L) return; - Assert1(F, "function-local metadata used outside a function", L); + Assert(F, "function-local metadata used outside a function", L); // If this was an instruction, bb, or argument, verify that it is in the // function that we expect. Function *ActualF = nullptr; if (Instruction *I = dyn_cast(L->getValue())) { - Assert2(I->getParent(), "function-local metadata not in basic block", L, I); + Assert(I->getParent(), "function-local metadata not in basic block", L, I); ActualF = I->getParent()->getParent(); } else if (BasicBlock *BB = dyn_cast(L->getValue())) ActualF = BB->getParent(); @@ -658,7 +639,7 @@ void Verifier::visitValueAsMetadata(const ValueAsMetadata &MD, Function *F) { ActualF = A->getParent(); assert(ActualF && "Unimplemented function local metadata case!"); - Assert1(ActualF == F, "function-local metadata used in wrong function", L); + Assert(ActualF == F, "function-local metadata used in wrong function", L); } void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { @@ -678,126 +659,126 @@ void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { } void Verifier::visitMDLocation(const MDLocation &N) { - Assert1(N.getScope(), "location requires a valid scope", &N); - if (auto *IA = N.getInlinedAt()) - Assert2(isa(IA), "inlined-at should be a location", &N, IA); + Assert(N.getRawScope() && isa(N.getRawScope()), + "location requires a valid scope", &N, N.getRawScope()); + if (auto *IA = N.getRawInlinedAt()) + Assert(isa(IA), "inlined-at should be a location", &N, IA); } void Verifier::visitGenericDebugNode(const GenericDebugNode &N) { - Assert1(N.getTag(), "invalid tag", &N); + Assert(N.getTag(), "invalid tag", &N); } void Verifier::visitMDSubrange(const MDSubrange &N) { - Assert1(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); } void Verifier::visitMDEnumerator(const MDEnumerator &N) { - Assert1(N.getTag() == dwarf::DW_TAG_enumerator, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_enumerator, "invalid tag", &N); } void Verifier::visitMDBasicType(const MDBasicType &N) { - Assert1(N.getTag() == dwarf::DW_TAG_base_type || - N.getTag() == dwarf::DW_TAG_unspecified_type, - "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_base_type || + N.getTag() == dwarf::DW_TAG_unspecified_type, + "invalid tag", &N); } void Verifier::visitMDDerivedType(const MDDerivedType &N) { - Assert1(N.getTag() == dwarf::DW_TAG_typedef || - N.getTag() == dwarf::DW_TAG_pointer_type || - N.getTag() == dwarf::DW_TAG_ptr_to_member_type || - N.getTag() == dwarf::DW_TAG_reference_type || - N.getTag() == dwarf::DW_TAG_rvalue_reference_type || - N.getTag() == dwarf::DW_TAG_const_type || - N.getTag() == dwarf::DW_TAG_volatile_type || - N.getTag() == dwarf::DW_TAG_restrict_type || - N.getTag() == dwarf::DW_TAG_member || - N.getTag() == dwarf::DW_TAG_inheritance || - N.getTag() == dwarf::DW_TAG_friend, - "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_typedef || + N.getTag() == dwarf::DW_TAG_pointer_type || + N.getTag() == dwarf::DW_TAG_ptr_to_member_type || + N.getTag() == dwarf::DW_TAG_reference_type || + N.getTag() == dwarf::DW_TAG_rvalue_reference_type || + N.getTag() == dwarf::DW_TAG_const_type || + N.getTag() == dwarf::DW_TAG_volatile_type || + N.getTag() == dwarf::DW_TAG_restrict_type || + N.getTag() == dwarf::DW_TAG_member || + N.getTag() == dwarf::DW_TAG_inheritance || + N.getTag() == dwarf::DW_TAG_friend, + "invalid tag", &N); } void Verifier::visitMDCompositeType(const MDCompositeType &N) { - Assert1(N.getTag() == dwarf::DW_TAG_array_type || - N.getTag() == dwarf::DW_TAG_structure_type || - N.getTag() == dwarf::DW_TAG_union_type || - N.getTag() == dwarf::DW_TAG_enumeration_type || - N.getTag() == dwarf::DW_TAG_subroutine_type || - N.getTag() == dwarf::DW_TAG_class_type, - "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_array_type || + N.getTag() == dwarf::DW_TAG_structure_type || + N.getTag() == dwarf::DW_TAG_union_type || + N.getTag() == dwarf::DW_TAG_enumeration_type || + N.getTag() == dwarf::DW_TAG_subroutine_type || + N.getTag() == dwarf::DW_TAG_class_type, + "invalid tag", &N); } void Verifier::visitMDSubroutineType(const MDSubroutineType &N) { - Assert1(N.getTag() == dwarf::DW_TAG_subroutine_type, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_subroutine_type, "invalid tag", &N); } void Verifier::visitMDFile(const MDFile &N) { - Assert1(N.getTag() == dwarf::DW_TAG_file_type, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_file_type, "invalid tag", &N); } void Verifier::visitMDCompileUnit(const MDCompileUnit &N) { - Assert1(N.getTag() == dwarf::DW_TAG_compile_unit, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_compile_unit, "invalid tag", &N); } void Verifier::visitMDSubprogram(const MDSubprogram &N) { - Assert1(N.getTag() == dwarf::DW_TAG_subprogram, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_subprogram, "invalid tag", &N); } void Verifier::visitMDLexicalBlock(const MDLexicalBlock &N) { - Assert1(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); } void Verifier::visitMDLexicalBlockFile(const MDLexicalBlockFile &N) { - Assert1(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); } void Verifier::visitMDNamespace(const MDNamespace &N) { - Assert1(N.getTag() == dwarf::DW_TAG_namespace, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_namespace, "invalid tag", &N); } void Verifier::visitMDTemplateTypeParameter(const MDTemplateTypeParameter &N) { - Assert1(N.getTag() == dwarf::DW_TAG_template_type_parameter, "invalid tag", - &N); + Assert(N.getTag() == dwarf::DW_TAG_template_type_parameter, "invalid tag", + &N); } void Verifier::visitMDTemplateValueParameter( const MDTemplateValueParameter &N) { - Assert1(N.getTag() == dwarf::DW_TAG_template_value_parameter || - N.getTag() == dwarf::DW_TAG_GNU_template_template_param || - N.getTag() == dwarf::DW_TAG_GNU_template_parameter_pack, - "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_template_value_parameter || + N.getTag() == dwarf::DW_TAG_GNU_template_template_param || + N.getTag() == dwarf::DW_TAG_GNU_template_parameter_pack, + "invalid tag", &N); } void Verifier::visitMDGlobalVariable(const MDGlobalVariable &N) { - Assert1(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); } void Verifier::visitMDLocalVariable(const MDLocalVariable &N) { - Assert1(N.getTag() == dwarf::DW_TAG_auto_variable || - N.getTag() == dwarf::DW_TAG_arg_variable, - "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_auto_variable || + N.getTag() == dwarf::DW_TAG_arg_variable, + "invalid tag", &N); } void Verifier::visitMDExpression(const MDExpression &N) { - Assert1(N.getTag() == dwarf::DW_TAG_expression, "invalid tag", &N); - Assert1(N.isValid(), "invalid expression", &N); + Assert(N.isValid(), "invalid expression", &N); } void Verifier::visitMDObjCProperty(const MDObjCProperty &N) { - Assert1(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N); } void Verifier::visitMDImportedEntity(const MDImportedEntity &N) { - Assert1(N.getTag() == dwarf::DW_TAG_imported_module || - N.getTag() == dwarf::DW_TAG_imported_declaration, - "invalid tag", &N); + Assert(N.getTag() == dwarf::DW_TAG_imported_module || + N.getTag() == dwarf::DW_TAG_imported_declaration, + "invalid tag", &N); } void Verifier::visitComdat(const Comdat &C) { // The Module is invalid if the GlobalValue has private linkage. Entities // with private linkage don't have entries in the symbol table. if (const GlobalValue *GV = M->getNamedValue(C.getName())) - Assert1(!GV->hasPrivateLinkage(), "comdat global value has private linkage", - GV); + Assert(!GV->hasPrivateLinkage(), "comdat global value has private linkage", + GV); } void Verifier::visitModuleIdents(const Module &M) { @@ -809,12 +790,12 @@ void Verifier::visitModuleIdents(const Module &M) { // Scan each llvm.ident entry and make sure that this requirement is met. for (unsigned i = 0, e = Idents->getNumOperands(); i != e; ++i) { const MDNode *N = Idents->getOperand(i); - Assert1(N->getNumOperands() == 1, - "incorrect number of operands in llvm.ident metadata", N); - Assert1(dyn_cast_or_null(N->getOperand(0)), - ("invalid value for llvm.ident metadata entry operand" - "(the operand should be a string)"), - N->getOperand(0)); + Assert(N->getNumOperands() == 1, + "incorrect number of operands in llvm.ident metadata", N); + Assert(dyn_cast_or_null(N->getOperand(0)), + ("invalid value for llvm.ident metadata entry operand" + "(the operand should be a string)"), + N->getOperand(0)); } } @@ -857,22 +838,21 @@ Verifier::visitModuleFlag(const MDNode *Op, SmallVectorImpl &Requirements) { // Each module flag should have three arguments, the merge behavior (a // constant int), the flag ID (an MDString), and the value. - Assert1(Op->getNumOperands() == 3, - "incorrect number of operands in module flag", Op); + Assert(Op->getNumOperands() == 3, + "incorrect number of operands in module flag", Op); Module::ModFlagBehavior MFB; if (!Module::isValidModFlagBehavior(Op->getOperand(0), MFB)) { - Assert1( + Assert( mdconst::dyn_extract_or_null(Op->getOperand(0)), "invalid behavior operand in module flag (expected constant integer)", Op->getOperand(0)); - Assert1(false, - "invalid behavior operand in module flag (unexpected constant)", - Op->getOperand(0)); + Assert(false, + "invalid behavior operand in module flag (unexpected constant)", + Op->getOperand(0)); } MDString *ID = dyn_cast_or_null(Op->getOperand(1)); - Assert1(ID, - "invalid ID operand in module flag (expected metadata string)", - Op->getOperand(1)); + Assert(ID, "invalid ID operand in module flag (expected metadata string)", + Op->getOperand(1)); // Sanity check the values for behaviors with additional requirements. switch (MFB) { @@ -886,13 +866,13 @@ Verifier::visitModuleFlag(const MDNode *Op, // The value should itself be an MDNode with two operands, a flag ID (an // MDString), and a value. MDNode *Value = dyn_cast(Op->getOperand(2)); - Assert1(Value && Value->getNumOperands() == 2, - "invalid value for 'require' module flag (expected metadata pair)", - Op->getOperand(2)); - Assert1(isa(Value->getOperand(0)), - ("invalid value for 'require' module flag " - "(first value operand should be a string)"), - Value->getOperand(0)); + Assert(Value && Value->getNumOperands() == 2, + "invalid value for 'require' module flag (expected metadata pair)", + Op->getOperand(2)); + Assert(isa(Value->getOperand(0)), + ("invalid value for 'require' module flag " + "(first value operand should be a string)"), + Value->getOperand(0)); // Append it to the list of requirements, to check once all module flags are // scanned. @@ -903,9 +883,10 @@ Verifier::visitModuleFlag(const MDNode *Op, case Module::Append: case Module::AppendUnique: { // These behavior types require the operand be an MDNode. - Assert1(isa(Op->getOperand(2)), - "invalid value for 'append'-type module flag " - "(expected a metadata node)", Op->getOperand(2)); + Assert(isa(Op->getOperand(2)), + "invalid value for 'append'-type module flag " + "(expected a metadata node)", + Op->getOperand(2)); break; } } @@ -913,9 +894,8 @@ Verifier::visitModuleFlag(const MDNode *Op, // Unless this is a "requires" flag, check the ID is unique. if (MFB != Module::Require) { bool Inserted = SeenIDs.insert(std::make_pair(ID, Op)).second; - Assert1(Inserted, - "module flag identifiers must be unique (or of 'require' type)", - ID); + Assert(Inserted, + "module flag identifiers must be unique (or of 'require' type)", ID); } } @@ -991,14 +971,15 @@ void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, VerifyAttributeTypes(Attrs, Idx, false, V); if (isReturnValue) - Assert1(!Attrs.hasAttribute(Idx, Attribute::ByVal) && - !Attrs.hasAttribute(Idx, Attribute::Nest) && - !Attrs.hasAttribute(Idx, Attribute::StructRet) && - !Attrs.hasAttribute(Idx, Attribute::NoCapture) && - !Attrs.hasAttribute(Idx, Attribute::Returned) && - !Attrs.hasAttribute(Idx, Attribute::InAlloca), - "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', and " - "'returned' do not apply to return values!", V); + Assert(!Attrs.hasAttribute(Idx, Attribute::ByVal) && + !Attrs.hasAttribute(Idx, Attribute::Nest) && + !Attrs.hasAttribute(Idx, Attribute::StructRet) && + !Attrs.hasAttribute(Idx, Attribute::NoCapture) && + !Attrs.hasAttribute(Idx, Attribute::Returned) && + !Attrs.hasAttribute(Idx, Attribute::InAlloca), + "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', and " + "'returned' do not apply to return values!", + V); // Check for mutually incompatible attributes. Only inreg is compatible with // sret. @@ -1008,45 +989,58 @@ void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, AttrCount += Attrs.hasAttribute(Idx, Attribute::StructRet) || Attrs.hasAttribute(Idx, Attribute::InReg); AttrCount += Attrs.hasAttribute(Idx, Attribute::Nest); - Assert1(AttrCount <= 1, "Attributes 'byval', 'inalloca', 'inreg', 'nest', " - "and 'sret' are incompatible!", V); - - Assert1(!(Attrs.hasAttribute(Idx, Attribute::InAlloca) && - Attrs.hasAttribute(Idx, Attribute::ReadOnly)), "Attributes " - "'inalloca and readonly' are incompatible!", V); - - Assert1(!(Attrs.hasAttribute(Idx, Attribute::StructRet) && - Attrs.hasAttribute(Idx, Attribute::Returned)), "Attributes " - "'sret and returned' are incompatible!", V); - - Assert1(!(Attrs.hasAttribute(Idx, Attribute::ZExt) && - Attrs.hasAttribute(Idx, Attribute::SExt)), "Attributes " - "'zeroext and signext' are incompatible!", V); - - Assert1(!(Attrs.hasAttribute(Idx, Attribute::ReadNone) && - Attrs.hasAttribute(Idx, Attribute::ReadOnly)), "Attributes " - "'readnone and readonly' are incompatible!", V); - - Assert1(!(Attrs.hasAttribute(Idx, Attribute::NoInline) && - Attrs.hasAttribute(Idx, Attribute::AlwaysInline)), "Attributes " - "'noinline and alwaysinline' are incompatible!", V); - - Assert1(!AttrBuilder(Attrs, Idx). - hasAttributes(AttributeFuncs::typeIncompatible(Ty, Idx), Idx), - "Wrong types for attribute: " + - AttributeFuncs::typeIncompatible(Ty, Idx).getAsString(Idx), V); + Assert(AttrCount <= 1, "Attributes 'byval', 'inalloca', 'inreg', 'nest', " + "and 'sret' are incompatible!", + V); + + Assert(!(Attrs.hasAttribute(Idx, Attribute::InAlloca) && + Attrs.hasAttribute(Idx, Attribute::ReadOnly)), + "Attributes " + "'inalloca and readonly' are incompatible!", + V); + + Assert(!(Attrs.hasAttribute(Idx, Attribute::StructRet) && + Attrs.hasAttribute(Idx, Attribute::Returned)), + "Attributes " + "'sret and returned' are incompatible!", + V); + + Assert(!(Attrs.hasAttribute(Idx, Attribute::ZExt) && + Attrs.hasAttribute(Idx, Attribute::SExt)), + "Attributes " + "'zeroext and signext' are incompatible!", + V); + + Assert(!(Attrs.hasAttribute(Idx, Attribute::ReadNone) && + Attrs.hasAttribute(Idx, Attribute::ReadOnly)), + "Attributes " + "'readnone and readonly' are incompatible!", + V); + + Assert(!(Attrs.hasAttribute(Idx, Attribute::NoInline) && + Attrs.hasAttribute(Idx, Attribute::AlwaysInline)), + "Attributes " + "'noinline and alwaysinline' are incompatible!", + V); + + Assert(!AttrBuilder(Attrs, Idx) + .hasAttributes(AttributeFuncs::typeIncompatible(Ty, Idx), Idx), + "Wrong types for attribute: " + + AttributeFuncs::typeIncompatible(Ty, Idx).getAsString(Idx), + V); if (PointerType *PTy = dyn_cast(Ty)) { - if (!PTy->getElementType()->isSized()) { - Assert1(!Attrs.hasAttribute(Idx, Attribute::ByVal) && - !Attrs.hasAttribute(Idx, Attribute::InAlloca), - "Attributes 'byval' and 'inalloca' do not support unsized types!", - V); + SmallPtrSet Visited; + if (!PTy->getElementType()->isSized(&Visited)) { + Assert(!Attrs.hasAttribute(Idx, Attribute::ByVal) && + !Attrs.hasAttribute(Idx, Attribute::InAlloca), + "Attributes 'byval' and 'inalloca' do not support unsized types!", + V); } } else { - Assert1(!Attrs.hasAttribute(Idx, Attribute::ByVal), - "Attribute 'byval' only applies to parameters with pointer type!", - V); + Assert(!Attrs.hasAttribute(Idx, Attribute::ByVal), + "Attribute 'byval' only applies to parameters with pointer type!", + V); } } @@ -1078,28 +1072,30 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, continue; if (Attrs.hasAttribute(Idx, Attribute::Nest)) { - Assert1(!SawNest, "More than one parameter has attribute nest!", V); + Assert(!SawNest, "More than one parameter has attribute nest!", V); SawNest = true; } if (Attrs.hasAttribute(Idx, Attribute::Returned)) { - Assert1(!SawReturned, "More than one parameter has attribute returned!", - V); - Assert1(Ty->canLosslesslyBitCastTo(FT->getReturnType()), "Incompatible " - "argument and return types for 'returned' attribute", V); + Assert(!SawReturned, "More than one parameter has attribute returned!", + V); + Assert(Ty->canLosslesslyBitCastTo(FT->getReturnType()), + "Incompatible " + "argument and return types for 'returned' attribute", + V); SawReturned = true; } if (Attrs.hasAttribute(Idx, Attribute::StructRet)) { - Assert1(!SawSRet, "Cannot have multiple 'sret' parameters!", V); - Assert1(Idx == 1 || Idx == 2, - "Attribute 'sret' is not on first or second parameter!", V); + Assert(!SawSRet, "Cannot have multiple 'sret' parameters!", V); + Assert(Idx == 1 || Idx == 2, + "Attribute 'sret' is not on first or second parameter!", V); SawSRet = true; } if (Attrs.hasAttribute(Idx, Attribute::InAlloca)) { - Assert1(Idx == FT->getNumParams(), - "inalloca isn't on the last parameter!", V); + Assert(Idx == FT->getNumParams(), "inalloca isn't on the last parameter!", + V); } } @@ -1108,39 +1104,35 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, VerifyAttributeTypes(Attrs, AttributeSet::FunctionIndex, true, V); - Assert1(!(Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::ReadNone) && - Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::ReadOnly)), - "Attributes 'readnone and readonly' are incompatible!", V); + Assert( + !(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone) && + Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly)), + "Attributes 'readnone and readonly' are incompatible!", V); - Assert1(!(Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::NoInline) && - Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::AlwaysInline)), - "Attributes 'noinline and alwaysinline' are incompatible!", V); + Assert( + !(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::NoInline) && + Attrs.hasAttribute(AttributeSet::FunctionIndex, + Attribute::AlwaysInline)), + "Attributes 'noinline and alwaysinline' are incompatible!", V); if (Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeNone)) { - Assert1(Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::NoInline), - "Attribute 'optnone' requires 'noinline'!", V); + Assert(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::NoInline), + "Attribute 'optnone' requires 'noinline'!", V); - Assert1(!Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::OptimizeForSize), - "Attributes 'optsize and optnone' are incompatible!", V); + Assert(!Attrs.hasAttribute(AttributeSet::FunctionIndex, + Attribute::OptimizeForSize), + "Attributes 'optsize and optnone' are incompatible!", V); - Assert1(!Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::MinSize), - "Attributes 'minsize and optnone' are incompatible!", V); + Assert(!Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize), + "Attributes 'minsize and optnone' are incompatible!", V); } if (Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::JumpTable)) { const GlobalValue *GV = cast(V); - Assert1(GV->hasUnnamedAddr(), - "Attribute 'jumptable' requires 'unnamed_addr'", V); - + Assert(GV->hasUnnamedAddr(), + "Attribute 'jumptable' requires 'unnamed_addr'", V); } } @@ -1148,9 +1140,9 @@ void Verifier::VerifyConstantExprBitcastType(const ConstantExpr *CE) { if (CE->getOpcode() != Instruction::BitCast) return; - Assert1(CastInst::castIsValid(Instruction::BitCast, CE->getOperand(0), - CE->getType()), - "Invalid bitcast", CE); + Assert(CastInst::castIsValid(Instruction::BitCast, CE->getOperand(0), + CE->getType()), + "Invalid bitcast", CE); } bool Verifier::VerifyAttributeCount(AttributeSet Attrs, unsigned Params) { @@ -1175,84 +1167,86 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) { const Instruction &CI = *CS.getInstruction(); - Assert1(!CS.doesNotAccessMemory() && - !CS.onlyReadsMemory(), - "gc.statepoint must read and write memory to preserve " - "reordering restrictions required by safepoint semantics", &CI); - + Assert(!CS.doesNotAccessMemory() && !CS.onlyReadsMemory(), + "gc.statepoint must read and write memory to preserve " + "reordering restrictions required by safepoint semantics", + &CI); + const Value *Target = CS.getArgument(0); const PointerType *PT = dyn_cast(Target->getType()); - Assert2(PT && PT->getElementType()->isFunctionTy(), - "gc.statepoint callee must be of function pointer type", - &CI, Target); + Assert(PT && PT->getElementType()->isFunctionTy(), + "gc.statepoint callee must be of function pointer type", &CI, Target); FunctionType *TargetFuncType = cast(PT->getElementType()); const Value *NumCallArgsV = CS.getArgument(1); - Assert1(isa(NumCallArgsV), - "gc.statepoint number of arguments to underlying call " - "must be constant integer", &CI); + Assert(isa(NumCallArgsV), + "gc.statepoint number of arguments to underlying call " + "must be constant integer", + &CI); const int NumCallArgs = cast(NumCallArgsV)->getZExtValue(); - Assert1(NumCallArgs >= 0, - "gc.statepoint number of arguments to underlying call " - "must be positive", &CI); + Assert(NumCallArgs >= 0, + "gc.statepoint number of arguments to underlying call " + "must be positive", + &CI); const int NumParams = (int)TargetFuncType->getNumParams(); if (TargetFuncType->isVarArg()) { - Assert1(NumCallArgs >= NumParams, - "gc.statepoint mismatch in number of vararg call args", &CI); + Assert(NumCallArgs >= NumParams, + "gc.statepoint mismatch in number of vararg call args", &CI); // TODO: Remove this limitation - Assert1(TargetFuncType->getReturnType()->isVoidTy(), - "gc.statepoint doesn't support wrapping non-void " - "vararg functions yet", &CI); + Assert(TargetFuncType->getReturnType()->isVoidTy(), + "gc.statepoint doesn't support wrapping non-void " + "vararg functions yet", + &CI); } else - Assert1(NumCallArgs == NumParams, - "gc.statepoint mismatch in number of call args", &CI); + Assert(NumCallArgs == NumParams, + "gc.statepoint mismatch in number of call args", &CI); const Value *Unused = CS.getArgument(2); - Assert1(isa(Unused) && - cast(Unused)->isNullValue(), - "gc.statepoint parameter #3 must be zero", &CI); + Assert(isa(Unused) && cast(Unused)->isNullValue(), + "gc.statepoint parameter #3 must be zero", &CI); // Verify that the types of the call parameter arguments match // the type of the wrapped callee. for (int i = 0; i < NumParams; i++) { Type *ParamType = TargetFuncType->getParamType(i); Type *ArgType = CS.getArgument(3+i)->getType(); - Assert1(ArgType == ParamType, - "gc.statepoint call argument does not match wrapped " - "function type", &CI); + Assert(ArgType == ParamType, + "gc.statepoint call argument does not match wrapped " + "function type", + &CI); } const int EndCallArgsInx = 2+NumCallArgs; const Value *NumDeoptArgsV = CS.getArgument(EndCallArgsInx+1); - Assert1(isa(NumDeoptArgsV), - "gc.statepoint number of deoptimization arguments " - "must be constant integer", &CI); + Assert(isa(NumDeoptArgsV), + "gc.statepoint number of deoptimization arguments " + "must be constant integer", + &CI); const int NumDeoptArgs = cast(NumDeoptArgsV)->getZExtValue(); - Assert1(NumDeoptArgs >= 0, - "gc.statepoint number of deoptimization arguments " - "must be positive", &CI); + Assert(NumDeoptArgs >= 0, "gc.statepoint number of deoptimization arguments " + "must be positive", + &CI); + + Assert(4 + NumCallArgs + NumDeoptArgs <= (int)CS.arg_size(), + "gc.statepoint too few arguments according to length fields", &CI); - Assert1(4 + NumCallArgs + NumDeoptArgs <= (int)CS.arg_size(), - "gc.statepoint too few arguments according to length fields", &CI); - // Check that the only uses of this gc.statepoint are gc.result or // gc.relocate calls which are tied to this statepoint and thus part // of the same statepoint sequence for (const User *U : CI.users()) { const CallInst *Call = dyn_cast(U); - Assert2(Call, "illegal use of statepoint token", &CI, U); + Assert(Call, "illegal use of statepoint token", &CI, U); if (!Call) continue; - Assert2(isGCRelocate(Call) || isGCResult(Call), - "gc.result or gc.relocate are the only value uses" - "of a gc.statepoint", &CI, U); + Assert(isGCRelocate(Call) || isGCResult(Call), + "gc.result or gc.relocate are the only value uses" + "of a gc.statepoint", + &CI, U); if (isGCResult(Call)) { - Assert2(Call->getArgOperand(0) == &CI, - "gc.result connected to wrong gc.statepoint", - &CI, Call); + Assert(Call->getArgOperand(0) == &CI, + "gc.result connected to wrong gc.statepoint", &CI, Call); } else if (isGCRelocate(Call)) { - Assert2(Call->getArgOperand(0) == &CI, - "gc.relocate connected to wrong gc.statepoint", - &CI, Call); + Assert(Call->getArgOperand(0) == &CI, + "gc.relocate connected to wrong gc.statepoint", &CI, Call); } } @@ -1266,6 +1260,19 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) { // about. See example statepoint.ll in the verifier subdirectory } +void Verifier::verifyFrameRecoverIndices() { + for (auto &Counts : FrameEscapeInfo) { + Function *F = Counts.first; + unsigned EscapedObjectCount = Counts.second.first; + unsigned MaxRecoveredIndex = Counts.second.second; + Assert(MaxRecoveredIndex <= EscapedObjectCount, + "all indices passed to llvm.framerecover must be less than the " + "number of arguments passed ot llvm.frameescape in the parent " + "function", + F); + } +} + // visitFunction - Verify that a function is ok. // void Verifier::visitFunction(const Function &F) { @@ -1273,25 +1280,24 @@ void Verifier::visitFunction(const Function &F) { FunctionType *FT = F.getFunctionType(); unsigned NumArgs = F.arg_size(); - Assert1(Context == &F.getContext(), - "Function context does not match Module context!", &F); + Assert(Context == &F.getContext(), + "Function context does not match Module context!", &F); - Assert1(!F.hasCommonLinkage(), "Functions may not have common linkage", &F); - Assert2(FT->getNumParams() == NumArgs, - "# formal arguments must match # of arguments for function type!", - &F, FT); - Assert1(F.getReturnType()->isFirstClassType() || - F.getReturnType()->isVoidTy() || - F.getReturnType()->isStructTy(), - "Functions cannot return aggregate values!", &F); + Assert(!F.hasCommonLinkage(), "Functions may not have common linkage", &F); + Assert(FT->getNumParams() == NumArgs, + "# formal arguments must match # of arguments for function type!", &F, + FT); + Assert(F.getReturnType()->isFirstClassType() || + F.getReturnType()->isVoidTy() || F.getReturnType()->isStructTy(), + "Functions cannot return aggregate values!", &F); - Assert1(!F.hasStructRetAttr() || F.getReturnType()->isVoidTy(), - "Invalid struct return type!", &F); + Assert(!F.hasStructRetAttr() || F.getReturnType()->isVoidTy(), + "Invalid struct return type!", &F); AttributeSet Attrs = F.getAttributes(); - Assert1(VerifyAttributeCount(Attrs, FT->getNumParams()), - "Attribute after last parameter!", &F); + Assert(VerifyAttributeCount(Attrs, FT->getNumParams()), + "Attribute after last parameter!", &F); // Check function attributes. VerifyFunctionAttrs(FT, Attrs, &F); @@ -1299,9 +1305,8 @@ void Verifier::visitFunction(const Function &F) { // On function declarations/definitions, we do not support the builtin // attribute. We do not check this in VerifyFunctionAttrs since that is // checking for Attributes that can/can not ever be on functions. - Assert1(!Attrs.hasAttribute(AttributeSet::FunctionIndex, - Attribute::Builtin), - "Attribute 'builtin' can only be applied to a callsite.", &F); + Assert(!Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::Builtin), + "Attribute 'builtin' can only be applied to a callsite.", &F); // Check that this function meets the restrictions on this calling convention. // Sometimes varargs is used for perfectly forwarding thunks, so some of these @@ -1315,8 +1320,9 @@ void Verifier::visitFunction(const Function &F) { case CallingConv::Intel_OCL_BI: case CallingConv::PTX_Kernel: case CallingConv::PTX_Device: - Assert1(!F.isVarArg(), "Calling convention does not support varargs or " - "perfect forwarding!", &F); + Assert(!F.isVarArg(), "Calling convention does not support varargs or " + "perfect forwarding!", + &F); break; } @@ -1327,35 +1333,35 @@ void Verifier::visitFunction(const Function &F) { unsigned i = 0; for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++i) { - Assert2(I->getType() == FT->getParamType(i), - "Argument value does not match function argument type!", - I, FT->getParamType(i)); - Assert1(I->getType()->isFirstClassType(), - "Function arguments must have first-class types!", I); + Assert(I->getType() == FT->getParamType(i), + "Argument value does not match function argument type!", I, + FT->getParamType(i)); + Assert(I->getType()->isFirstClassType(), + "Function arguments must have first-class types!", I); if (!isLLVMdotName) - Assert2(!I->getType()->isMetadataTy(), - "Function takes metadata but isn't an intrinsic", I, &F); + Assert(!I->getType()->isMetadataTy(), + "Function takes metadata but isn't an intrinsic", I, &F); } if (F.isMaterializable()) { // Function has a body somewhere we can't see. } else if (F.isDeclaration()) { - Assert1(F.hasExternalLinkage() || F.hasExternalWeakLinkage(), - "invalid linkage type for function declaration", &F); + Assert(F.hasExternalLinkage() || F.hasExternalWeakLinkage(), + "invalid linkage type for function declaration", &F); } else { // Verify that this function (which has a body) is not named "llvm.*". It // is not legal to define intrinsics. - Assert1(!isLLVMdotName, "llvm intrinsics cannot be defined!", &F); + Assert(!isLLVMdotName, "llvm intrinsics cannot be defined!", &F); // Check the entry node const BasicBlock *Entry = &F.getEntryBlock(); - Assert1(pred_empty(Entry), - "Entry block to function must not have predecessors!", Entry); + Assert(pred_empty(Entry), + "Entry block to function must not have predecessors!", Entry); // The address of the entry block cannot be taken, unless it is dead. if (Entry->hasAddressTaken()) { - Assert1(!BlockAddress::lookup(Entry)->isConstantUsed(), - "blockaddress may not be used with the entry block!", Entry); + Assert(!BlockAddress::lookup(Entry)->isConstantUsed(), + "blockaddress may not be used with the entry block!", Entry); } } @@ -1364,13 +1370,13 @@ void Verifier::visitFunction(const Function &F) { if (F.getIntrinsicID()) { const User *U; if (F.hasAddressTaken(&U)) - Assert1(0, "Invalid user of intrinsic instruction!", U); + Assert(0, "Invalid user of intrinsic instruction!", U); } - Assert1(!F.hasDLLImportStorageClass() || - (F.isDeclaration() && F.hasExternalLinkage()) || - F.hasAvailableExternallyLinkage(), - "Function is marked as dllimport, but not external.", &F); + Assert(!F.hasDLLImportStorageClass() || + (F.isDeclaration() && F.hasExternalLinkage()) || + F.hasAvailableExternallyLinkage(), + "Function is marked as dllimport, but not external.", &F); } // verifyBasicBlock - Verify that a basic block is well formed... @@ -1379,7 +1385,7 @@ void Verifier::visitBasicBlock(BasicBlock &BB) { InstsInThisBlock.clear(); // Ensure that basic blocks have terminators! - Assert1(BB.getTerminator(), "Basic Block does not have terminator!", &BB); + Assert(BB.getTerminator(), "Basic Block does not have terminator!", &BB); // Check constraints that this basic block imposes on all of the PHI nodes in // it. @@ -1390,12 +1396,14 @@ void Verifier::visitBasicBlock(BasicBlock &BB) { PHINode *PN; for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast(I));++I) { // Ensure that PHI nodes have at least one entry! - Assert1(PN->getNumIncomingValues() != 0, - "PHI nodes must have at least one entry. If the block is dead, " - "the PHI should be removed!", PN); - Assert1(PN->getNumIncomingValues() == Preds.size(), - "PHINode should have one entry for each predecessor of its " - "parent basic block!", PN); + Assert(PN->getNumIncomingValues() != 0, + "PHI nodes must have at least one entry. If the block is dead, " + "the PHI should be removed!", + PN); + Assert(PN->getNumIncomingValues() == Preds.size(), + "PHINode should have one entry for each predecessor of its " + "parent basic block!", + PN); // Get and sort all incoming values in the PHI node... Values.clear(); @@ -1410,17 +1418,17 @@ void Verifier::visitBasicBlock(BasicBlock &BB) { // particular basic block in this PHI node, that the incoming values are // all identical. // - Assert4(i == 0 || Values[i].first != Values[i-1].first || - Values[i].second == Values[i-1].second, - "PHI node has multiple entries for the same basic block with " - "different incoming values!", PN, Values[i].first, - Values[i].second, Values[i-1].second); + Assert(i == 0 || Values[i].first != Values[i - 1].first || + Values[i].second == Values[i - 1].second, + "PHI node has multiple entries for the same basic block with " + "different incoming values!", + PN, Values[i].first, Values[i].second, Values[i - 1].second); // Check to make sure that the predecessors and PHI node entries are // matched up. - Assert3(Values[i].first == Preds[i], - "PHI node entries do not match predecessors!", PN, - Values[i].first, Preds[i]); + Assert(Values[i].first == Preds[i], + "PHI node entries do not match predecessors!", PN, + Values[i].first, Preds[i]); } } } @@ -1434,15 +1442,15 @@ void Verifier::visitBasicBlock(BasicBlock &BB) { void Verifier::visitTerminatorInst(TerminatorInst &I) { // Ensure that terminators only exist at the end of the basic block. - Assert1(&I == I.getParent()->getTerminator(), - "Terminator found in the middle of a basic block!", I.getParent()); + Assert(&I == I.getParent()->getTerminator(), + "Terminator found in the middle of a basic block!", I.getParent()); visitInstruction(I); } void Verifier::visitBranchInst(BranchInst &BI) { if (BI.isConditional()) { - Assert2(BI.getCondition()->getType()->isIntegerTy(1), - "Branch condition is not 'i1' type!", &BI, BI.getCondition()); + Assert(BI.getCondition()->getType()->isIntegerTy(1), + "Branch condition is not 'i1' type!", &BI, BI.getCondition()); } visitTerminatorInst(BI); } @@ -1451,13 +1459,15 @@ void Verifier::visitReturnInst(ReturnInst &RI) { Function *F = RI.getParent()->getParent(); unsigned N = RI.getNumOperands(); if (F->getReturnType()->isVoidTy()) - Assert2(N == 0, - "Found return instr that returns non-void in Function of void " - "return type!", &RI, F->getReturnType()); + Assert(N == 0, + "Found return instr that returns non-void in Function of void " + "return type!", + &RI, F->getReturnType()); else - Assert2(N == 1 && F->getReturnType() == RI.getOperand(0)->getType(), - "Function return type does not match operand " - "type of return inst!", &RI, F->getReturnType()); + Assert(N == 1 && F->getReturnType() == RI.getOperand(0)->getType(), + "Function return type does not match operand " + "type of return inst!", + &RI, F->getReturnType()); // Check to make sure that the return value has necessary properties for // terminators... @@ -1470,32 +1480,32 @@ void Verifier::visitSwitchInst(SwitchInst &SI) { Type *SwitchTy = SI.getCondition()->getType(); SmallPtrSet Constants; for (SwitchInst::CaseIt i = SI.case_begin(), e = SI.case_end(); i != e; ++i) { - Assert1(i.getCaseValue()->getType() == SwitchTy, - "Switch constants must all be same type as switch value!", &SI); - Assert2(Constants.insert(i.getCaseValue()).second, - "Duplicate integer as switch case", &SI, i.getCaseValue()); + Assert(i.getCaseValue()->getType() == SwitchTy, + "Switch constants must all be same type as switch value!", &SI); + Assert(Constants.insert(i.getCaseValue()).second, + "Duplicate integer as switch case", &SI, i.getCaseValue()); } visitTerminatorInst(SI); } void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { - Assert1(BI.getAddress()->getType()->isPointerTy(), - "Indirectbr operand must have pointer type!", &BI); + Assert(BI.getAddress()->getType()->isPointerTy(), + "Indirectbr operand must have pointer type!", &BI); for (unsigned i = 0, e = BI.getNumDestinations(); i != e; ++i) - Assert1(BI.getDestination(i)->getType()->isLabelTy(), - "Indirectbr destinations must all have pointer type!", &BI); + Assert(BI.getDestination(i)->getType()->isLabelTy(), + "Indirectbr destinations must all have pointer type!", &BI); visitTerminatorInst(BI); } void Verifier::visitSelectInst(SelectInst &SI) { - Assert1(!SelectInst::areInvalidOperands(SI.getOperand(0), SI.getOperand(1), - SI.getOperand(2)), - "Invalid operands for select instruction!", &SI); + Assert(!SelectInst::areInvalidOperands(SI.getOperand(0), SI.getOperand(1), + SI.getOperand(2)), + "Invalid operands for select instruction!", &SI); - Assert1(SI.getTrueValue()->getType() == SI.getType(), - "Select values must have same type as select instruction!", &SI); + Assert(SI.getTrueValue()->getType() == SI.getType(), + "Select values must have same type as select instruction!", &SI); visitInstruction(SI); } @@ -1503,7 +1513,7 @@ void Verifier::visitSelectInst(SelectInst &SI) { /// a pass, if any exist, it's an error. /// void Verifier::visitUserOp1(Instruction &I) { - Assert1(0, "User-defined operators should not live outside of a pass!", &I); + Assert(0, "User-defined operators should not live outside of a pass!", &I); } void Verifier::visitTruncInst(TruncInst &I) { @@ -1515,11 +1525,11 @@ void Verifier::visitTruncInst(TruncInst &I) { unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); unsigned DestBitSize = DestTy->getScalarSizeInBits(); - Assert1(SrcTy->isIntOrIntVectorTy(), "Trunc only operates on integer", &I); - Assert1(DestTy->isIntOrIntVectorTy(), "Trunc only produces integer", &I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "trunc source and destination must both be a vector or neither", &I); - Assert1(SrcBitSize > DestBitSize,"DestTy too big for Trunc", &I); + Assert(SrcTy->isIntOrIntVectorTy(), "Trunc only operates on integer", &I); + Assert(DestTy->isIntOrIntVectorTy(), "Trunc only produces integer", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), + "trunc source and destination must both be a vector or neither", &I); + Assert(SrcBitSize > DestBitSize, "DestTy too big for Trunc", &I); visitInstruction(I); } @@ -1530,14 +1540,14 @@ void Verifier::visitZExtInst(ZExtInst &I) { Type *DestTy = I.getType(); // Get the size of the types in bits, we'll need this later - Assert1(SrcTy->isIntOrIntVectorTy(), "ZExt only operates on integer", &I); - Assert1(DestTy->isIntOrIntVectorTy(), "ZExt only produces an integer", &I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "zext source and destination must both be a vector or neither", &I); + Assert(SrcTy->isIntOrIntVectorTy(), "ZExt only operates on integer", &I); + Assert(DestTy->isIntOrIntVectorTy(), "ZExt only produces an integer", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), + "zext source and destination must both be a vector or neither", &I); unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); unsigned DestBitSize = DestTy->getScalarSizeInBits(); - Assert1(SrcBitSize < DestBitSize,"Type too small for ZExt", &I); + Assert(SrcBitSize < DestBitSize, "Type too small for ZExt", &I); visitInstruction(I); } @@ -1551,11 +1561,11 @@ void Verifier::visitSExtInst(SExtInst &I) { unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); unsigned DestBitSize = DestTy->getScalarSizeInBits(); - Assert1(SrcTy->isIntOrIntVectorTy(), "SExt only operates on integer", &I); - Assert1(DestTy->isIntOrIntVectorTy(), "SExt only produces an integer", &I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "sext source and destination must both be a vector or neither", &I); - Assert1(SrcBitSize < DestBitSize,"Type too small for SExt", &I); + Assert(SrcTy->isIntOrIntVectorTy(), "SExt only operates on integer", &I); + Assert(DestTy->isIntOrIntVectorTy(), "SExt only produces an integer", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), + "sext source and destination must both be a vector or neither", &I); + Assert(SrcBitSize < DestBitSize, "Type too small for SExt", &I); visitInstruction(I); } @@ -1568,11 +1578,11 @@ void Verifier::visitFPTruncInst(FPTruncInst &I) { unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); unsigned DestBitSize = DestTy->getScalarSizeInBits(); - Assert1(SrcTy->isFPOrFPVectorTy(),"FPTrunc only operates on FP", &I); - Assert1(DestTy->isFPOrFPVectorTy(),"FPTrunc only produces an FP", &I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "fptrunc source and destination must both be a vector or neither",&I); - Assert1(SrcBitSize > DestBitSize,"DestTy too big for FPTrunc", &I); + Assert(SrcTy->isFPOrFPVectorTy(), "FPTrunc only operates on FP", &I); + Assert(DestTy->isFPOrFPVectorTy(), "FPTrunc only produces an FP", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), + "fptrunc source and destination must both be a vector or neither", &I); + Assert(SrcBitSize > DestBitSize, "DestTy too big for FPTrunc", &I); visitInstruction(I); } @@ -1586,11 +1596,11 @@ void Verifier::visitFPExtInst(FPExtInst &I) { unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); unsigned DestBitSize = DestTy->getScalarSizeInBits(); - Assert1(SrcTy->isFPOrFPVectorTy(),"FPExt only operates on FP", &I); - Assert1(DestTy->isFPOrFPVectorTy(),"FPExt only produces an FP", &I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "fpext source and destination must both be a vector or neither", &I); - Assert1(SrcBitSize < DestBitSize,"DestTy too small for FPExt", &I); + Assert(SrcTy->isFPOrFPVectorTy(), "FPExt only operates on FP", &I); + Assert(DestTy->isFPOrFPVectorTy(), "FPExt only produces an FP", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), + "fpext source and destination must both be a vector or neither", &I); + Assert(SrcBitSize < DestBitSize, "DestTy too small for FPExt", &I); visitInstruction(I); } @@ -1603,17 +1613,17 @@ void Verifier::visitUIToFPInst(UIToFPInst &I) { bool SrcVec = SrcTy->isVectorTy(); bool DstVec = DestTy->isVectorTy(); - Assert1(SrcVec == DstVec, - "UIToFP source and dest must both be vector or scalar", &I); - Assert1(SrcTy->isIntOrIntVectorTy(), - "UIToFP source must be integer or integer vector", &I); - Assert1(DestTy->isFPOrFPVectorTy(), - "UIToFP result must be FP or FP vector", &I); + Assert(SrcVec == DstVec, + "UIToFP source and dest must both be vector or scalar", &I); + Assert(SrcTy->isIntOrIntVectorTy(), + "UIToFP source must be integer or integer vector", &I); + Assert(DestTy->isFPOrFPVectorTy(), "UIToFP result must be FP or FP vector", + &I); if (SrcVec && DstVec) - Assert1(cast(SrcTy)->getNumElements() == - cast(DestTy)->getNumElements(), - "UIToFP source and dest vector length mismatch", &I); + Assert(cast(SrcTy)->getNumElements() == + cast(DestTy)->getNumElements(), + "UIToFP source and dest vector length mismatch", &I); visitInstruction(I); } @@ -1626,17 +1636,17 @@ void Verifier::visitSIToFPInst(SIToFPInst &I) { bool SrcVec = SrcTy->isVectorTy(); bool DstVec = DestTy->isVectorTy(); - Assert1(SrcVec == DstVec, - "SIToFP source and dest must both be vector or scalar", &I); - Assert1(SrcTy->isIntOrIntVectorTy(), - "SIToFP source must be integer or integer vector", &I); - Assert1(DestTy->isFPOrFPVectorTy(), - "SIToFP result must be FP or FP vector", &I); + Assert(SrcVec == DstVec, + "SIToFP source and dest must both be vector or scalar", &I); + Assert(SrcTy->isIntOrIntVectorTy(), + "SIToFP source must be integer or integer vector", &I); + Assert(DestTy->isFPOrFPVectorTy(), "SIToFP result must be FP or FP vector", + &I); if (SrcVec && DstVec) - Assert1(cast(SrcTy)->getNumElements() == - cast(DestTy)->getNumElements(), - "SIToFP source and dest vector length mismatch", &I); + Assert(cast(SrcTy)->getNumElements() == + cast(DestTy)->getNumElements(), + "SIToFP source and dest vector length mismatch", &I); visitInstruction(I); } @@ -1649,17 +1659,17 @@ void Verifier::visitFPToUIInst(FPToUIInst &I) { bool SrcVec = SrcTy->isVectorTy(); bool DstVec = DestTy->isVectorTy(); - Assert1(SrcVec == DstVec, - "FPToUI source and dest must both be vector or scalar", &I); - Assert1(SrcTy->isFPOrFPVectorTy(), "FPToUI source must be FP or FP vector", - &I); - Assert1(DestTy->isIntOrIntVectorTy(), - "FPToUI result must be integer or integer vector", &I); + Assert(SrcVec == DstVec, + "FPToUI source and dest must both be vector or scalar", &I); + Assert(SrcTy->isFPOrFPVectorTy(), "FPToUI source must be FP or FP vector", + &I); + Assert(DestTy->isIntOrIntVectorTy(), + "FPToUI result must be integer or integer vector", &I); if (SrcVec && DstVec) - Assert1(cast(SrcTy)->getNumElements() == - cast(DestTy)->getNumElements(), - "FPToUI source and dest vector length mismatch", &I); + Assert(cast(SrcTy)->getNumElements() == + cast(DestTy)->getNumElements(), + "FPToUI source and dest vector length mismatch", &I); visitInstruction(I); } @@ -1672,17 +1682,17 @@ void Verifier::visitFPToSIInst(FPToSIInst &I) { bool SrcVec = SrcTy->isVectorTy(); bool DstVec = DestTy->isVectorTy(); - Assert1(SrcVec == DstVec, - "FPToSI source and dest must both be vector or scalar", &I); - Assert1(SrcTy->isFPOrFPVectorTy(), - "FPToSI source must be FP or FP vector", &I); - Assert1(DestTy->isIntOrIntVectorTy(), - "FPToSI result must be integer or integer vector", &I); + Assert(SrcVec == DstVec, + "FPToSI source and dest must both be vector or scalar", &I); + Assert(SrcTy->isFPOrFPVectorTy(), "FPToSI source must be FP or FP vector", + &I); + Assert(DestTy->isIntOrIntVectorTy(), + "FPToSI result must be integer or integer vector", &I); if (SrcVec && DstVec) - Assert1(cast(SrcTy)->getNumElements() == - cast(DestTy)->getNumElements(), - "FPToSI source and dest vector length mismatch", &I); + Assert(cast(SrcTy)->getNumElements() == + cast(DestTy)->getNumElements(), + "FPToSI source and dest vector length mismatch", &I); visitInstruction(I); } @@ -1692,18 +1702,18 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) { Type *SrcTy = I.getOperand(0)->getType(); Type *DestTy = I.getType(); - Assert1(SrcTy->getScalarType()->isPointerTy(), - "PtrToInt source must be pointer", &I); - Assert1(DestTy->getScalarType()->isIntegerTy(), - "PtrToInt result must be integral", &I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "PtrToInt type mismatch", &I); + Assert(SrcTy->getScalarType()->isPointerTy(), + "PtrToInt source must be pointer", &I); + Assert(DestTy->getScalarType()->isIntegerTy(), + "PtrToInt result must be integral", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "PtrToInt type mismatch", + &I); if (SrcTy->isVectorTy()) { VectorType *VSrc = dyn_cast(SrcTy); VectorType *VDest = dyn_cast(DestTy); - Assert1(VSrc->getNumElements() == VDest->getNumElements(), - "PtrToInt Vector width mismatch", &I); + Assert(VSrc->getNumElements() == VDest->getNumElements(), + "PtrToInt Vector width mismatch", &I); } visitInstruction(I); @@ -1714,23 +1724,23 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) { Type *SrcTy = I.getOperand(0)->getType(); Type *DestTy = I.getType(); - Assert1(SrcTy->getScalarType()->isIntegerTy(), - "IntToPtr source must be an integral", &I); - Assert1(DestTy->getScalarType()->isPointerTy(), - "IntToPtr result must be a pointer",&I); - Assert1(SrcTy->isVectorTy() == DestTy->isVectorTy(), - "IntToPtr type mismatch", &I); + Assert(SrcTy->getScalarType()->isIntegerTy(), + "IntToPtr source must be an integral", &I); + Assert(DestTy->getScalarType()->isPointerTy(), + "IntToPtr result must be a pointer", &I); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "IntToPtr type mismatch", + &I); if (SrcTy->isVectorTy()) { VectorType *VSrc = dyn_cast(SrcTy); VectorType *VDest = dyn_cast(DestTy); - Assert1(VSrc->getNumElements() == VDest->getNumElements(), - "IntToPtr Vector width mismatch", &I); + Assert(VSrc->getNumElements() == VDest->getNumElements(), + "IntToPtr Vector width mismatch", &I); } visitInstruction(I); } void Verifier::visitBitCastInst(BitCastInst &I) { - Assert1( + Assert( CastInst::castIsValid(Instruction::BitCast, I.getOperand(0), I.getType()), "Invalid bitcast", &I); visitInstruction(I); @@ -1740,15 +1750,15 @@ void Verifier::visitAddrSpaceCastInst(AddrSpaceCastInst &I) { Type *SrcTy = I.getOperand(0)->getType(); Type *DestTy = I.getType(); - Assert1(SrcTy->isPtrOrPtrVectorTy(), - "AddrSpaceCast source must be a pointer", &I); - Assert1(DestTy->isPtrOrPtrVectorTy(), - "AddrSpaceCast result must be a pointer", &I); - Assert1(SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace(), - "AddrSpaceCast must be between different address spaces", &I); + Assert(SrcTy->isPtrOrPtrVectorTy(), "AddrSpaceCast source must be a pointer", + &I); + Assert(DestTy->isPtrOrPtrVectorTy(), "AddrSpaceCast result must be a pointer", + &I); + Assert(SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace(), + "AddrSpaceCast must be between different address spaces", &I); if (SrcTy->isVectorTy()) - Assert1(SrcTy->getVectorNumElements() == DestTy->getVectorNumElements(), - "AddrSpaceCast vector pointer number of elements mismatch", &I); + Assert(SrcTy->getVectorNumElements() == DestTy->getVectorNumElements(), + "AddrSpaceCast vector pointer number of elements mismatch", &I); visitInstruction(I); } @@ -1759,16 +1769,15 @@ void Verifier::visitPHINode(PHINode &PN) { // This can be tested by checking whether the instruction before this is // either nonexistent (because this is begin()) or is a PHI node. If not, // then there is some other instruction before a PHI. - Assert2(&PN == &PN.getParent()->front() || - isa(--BasicBlock::iterator(&PN)), - "PHI nodes not grouped at top of basic block!", - &PN, PN.getParent()); + Assert(&PN == &PN.getParent()->front() || + isa(--BasicBlock::iterator(&PN)), + "PHI nodes not grouped at top of basic block!", &PN, PN.getParent()); // Check that all of the values of the PHI node have the same type as the // result, and that the incoming blocks are really basic blocks. for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) { - Assert1(PN.getType() == PN.getIncomingValue(i)->getType(), - "PHI node operands are not the same type as the result!", &PN); + Assert(PN.getType() == PN.getIncomingValue(i)->getType(), + "PHI node operands are not the same type as the result!", &PN); } // All other PHI node constraints are checked in the visitBasicBlock method. @@ -1779,32 +1788,32 @@ void Verifier::visitPHINode(PHINode &PN) { void Verifier::VerifyCallSite(CallSite CS) { Instruction *I = CS.getInstruction(); - Assert1(CS.getCalledValue()->getType()->isPointerTy(), - "Called function must be a pointer!", I); + Assert(CS.getCalledValue()->getType()->isPointerTy(), + "Called function must be a pointer!", I); PointerType *FPTy = cast(CS.getCalledValue()->getType()); - Assert1(FPTy->getElementType()->isFunctionTy(), - "Called function is not pointer to function type!", I); + Assert(FPTy->getElementType()->isFunctionTy(), + "Called function is not pointer to function type!", I); FunctionType *FTy = cast(FPTy->getElementType()); // Verify that the correct number of arguments are being passed if (FTy->isVarArg()) - Assert1(CS.arg_size() >= FTy->getNumParams(), - "Called function requires more parameters than were provided!",I); + Assert(CS.arg_size() >= FTy->getNumParams(), + "Called function requires more parameters than were provided!", I); else - Assert1(CS.arg_size() == FTy->getNumParams(), - "Incorrect number of arguments passed to called function!", I); + Assert(CS.arg_size() == FTy->getNumParams(), + "Incorrect number of arguments passed to called function!", I); // Verify that all arguments to the call match the function type. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) - Assert3(CS.getArgument(i)->getType() == FTy->getParamType(i), - "Call parameter type does not match function signature!", - CS.getArgument(i), FTy->getParamType(i), I); + Assert(CS.getArgument(i)->getType() == FTy->getParamType(i), + "Call parameter type does not match function signature!", + CS.getArgument(i), FTy->getParamType(i), I); AttributeSet Attrs = CS.getAttributes(); - Assert1(VerifyAttributeCount(Attrs, CS.arg_size()), - "Attribute after last parameter!", I); + Assert(VerifyAttributeCount(Attrs, CS.arg_size()), + "Attribute after last parameter!", I); // Verify call attributes. VerifyFunctionAttrs(FTy, Attrs, I); @@ -1815,8 +1824,8 @@ void Verifier::VerifyCallSite(CallSite CS) { if (CS.hasInAllocaArgument()) { Value *InAllocaArg = CS.getArgument(FTy->getNumParams() - 1); if (auto AI = dyn_cast(InAllocaArg->stripInBoundsOffsets())) - Assert2(AI->isUsedWithInAlloca(), - "inalloca argument for call has mismatched alloca", AI, I); + Assert(AI->isUsedWithInAlloca(), + "inalloca argument for call has mismatched alloca", AI, I); } if (FTy->isVarArg()) { @@ -1837,25 +1846,25 @@ void Verifier::VerifyCallSite(CallSite CS) { VerifyParameterAttrs(Attrs, Idx, Ty, false, I); if (Attrs.hasAttribute(Idx, Attribute::Nest)) { - Assert1(!SawNest, "More than one parameter has attribute nest!", I); + Assert(!SawNest, "More than one parameter has attribute nest!", I); SawNest = true; } if (Attrs.hasAttribute(Idx, Attribute::Returned)) { - Assert1(!SawReturned, "More than one parameter has attribute returned!", - I); - Assert1(Ty->canLosslesslyBitCastTo(FTy->getReturnType()), - "Incompatible argument and return types for 'returned' " - "attribute", I); + Assert(!SawReturned, "More than one parameter has attribute returned!", + I); + Assert(Ty->canLosslesslyBitCastTo(FTy->getReturnType()), + "Incompatible argument and return types for 'returned' " + "attribute", + I); SawReturned = true; } - Assert1(!Attrs.hasAttribute(Idx, Attribute::StructRet), - "Attribute 'sret' cannot be used for vararg call arguments!", I); + Assert(!Attrs.hasAttribute(Idx, Attribute::StructRet), + "Attribute 'sret' cannot be used for vararg call arguments!", I); if (Attrs.hasAttribute(Idx, Attribute::InAlloca)) - Assert1(Idx == CS.arg_size(), "inalloca isn't on the last argument!", - I); + Assert(Idx == CS.arg_size(), "inalloca isn't on the last argument!", I); } } @@ -1864,8 +1873,8 @@ void Verifier::VerifyCallSite(CallSite CS) { !CS.getCalledFunction()->getName().startswith("llvm.")) { for (FunctionType::param_iterator PI = FTy->param_begin(), PE = FTy->param_end(); PI != PE; ++PI) - Assert1(!(*PI)->isMetadataTy(), - "Function has metadata parameter but isn't an intrinsic", I); + Assert(!(*PI)->isMetadataTy(), + "Function has metadata parameter but isn't an intrinsic", I); } visitInstruction(*I); @@ -1898,7 +1907,7 @@ static AttrBuilder getParameterABIAttributes(int I, AttributeSet Attrs) { } void Verifier::verifyMustTailCall(CallInst &CI) { - Assert1(!CI.isInlineAsm(), "cannot use musttail call with inline asm", &CI); + Assert(!CI.isInlineAsm(), "cannot use musttail call with inline asm", &CI); // - The caller and callee prototypes must match. Pointer types of // parameters or return types may differ in pointee type, but not @@ -1910,21 +1919,21 @@ void Verifier::verifyMustTailCall(CallInst &CI) { }; FunctionType *CallerTy = GetFnTy(F); FunctionType *CalleeTy = GetFnTy(CI.getCalledValue()); - Assert1(CallerTy->getNumParams() == CalleeTy->getNumParams(), - "cannot guarantee tail call due to mismatched parameter counts", &CI); - Assert1(CallerTy->isVarArg() == CalleeTy->isVarArg(), - "cannot guarantee tail call due to mismatched varargs", &CI); - Assert1(isTypeCongruent(CallerTy->getReturnType(), CalleeTy->getReturnType()), - "cannot guarantee tail call due to mismatched return types", &CI); + Assert(CallerTy->getNumParams() == CalleeTy->getNumParams(), + "cannot guarantee tail call due to mismatched parameter counts", &CI); + Assert(CallerTy->isVarArg() == CalleeTy->isVarArg(), + "cannot guarantee tail call due to mismatched varargs", &CI); + Assert(isTypeCongruent(CallerTy->getReturnType(), CalleeTy->getReturnType()), + "cannot guarantee tail call due to mismatched return types", &CI); for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) { - Assert1( + Assert( isTypeCongruent(CallerTy->getParamType(I), CalleeTy->getParamType(I)), "cannot guarantee tail call due to mismatched parameter types", &CI); } // - The calling conventions of the caller and callee must match. - Assert1(F->getCallingConv() == CI.getCallingConv(), - "cannot guarantee tail call due to mismatched calling conv", &CI); + Assert(F->getCallingConv() == CI.getCallingConv(), + "cannot guarantee tail call due to mismatched calling conv", &CI); // - All ABI-impacting function attributes, such as sret, byval, inreg, // returned, and inalloca, must match. @@ -1933,9 +1942,10 @@ void Verifier::verifyMustTailCall(CallInst &CI) { for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) { AttrBuilder CallerABIAttrs = getParameterABIAttributes(I, CallerAttrs); AttrBuilder CalleeABIAttrs = getParameterABIAttributes(I, CalleeAttrs); - Assert2(CallerABIAttrs == CalleeABIAttrs, - "cannot guarantee tail call due to mismatched ABI impacting " - "function attributes", &CI, CI.getOperand(I)); + Assert(CallerABIAttrs == CalleeABIAttrs, + "cannot guarantee tail call due to mismatched ABI impacting " + "function attributes", + &CI, CI.getOperand(I)); } // - The call must immediately precede a :ref:`ret ` instruction, @@ -1947,18 +1957,18 @@ void Verifier::verifyMustTailCall(CallInst &CI) { // Handle the optional bitcast. if (BitCastInst *BI = dyn_cast_or_null(Next)) { - Assert1(BI->getOperand(0) == RetVal, - "bitcast following musttail call must use the call", BI); + Assert(BI->getOperand(0) == RetVal, + "bitcast following musttail call must use the call", BI); RetVal = BI; Next = BI->getNextNode(); } // Check the return. ReturnInst *Ret = dyn_cast_or_null(Next); - Assert1(Ret, "musttail call must be precede a ret with an optional bitcast", - &CI); - Assert1(!Ret->getReturnValue() || Ret->getReturnValue() == RetVal, - "musttail call result must be returned", Ret); + Assert(Ret, "musttail call must be precede a ret with an optional bitcast", + &CI); + Assert(!Ret->getReturnValue() || Ret->getReturnValue() == RetVal, + "musttail call result must be returned", Ret); } void Verifier::visitCallInst(CallInst &CI) { @@ -1977,8 +1987,8 @@ void Verifier::visitInvokeInst(InvokeInst &II) { // Verify that there is a landingpad instruction as the first non-PHI // instruction of the 'unwind' destination. - Assert1(II.getUnwindDest()->isLandingPad(), - "The unwind destination does not have a landingpad instruction!",&II); + Assert(II.getUnwindDest()->isLandingPad(), + "The unwind destination does not have a landingpad instruction!", &II); if (Function *F = II.getCalledFunction()) // TODO: Ideally we should use visitIntrinsicFunction here. But it uses @@ -1994,8 +2004,8 @@ void Verifier::visitInvokeInst(InvokeInst &II) { /// of the same type! /// void Verifier::visitBinaryOperator(BinaryOperator &B) { - Assert1(B.getOperand(0)->getType() == B.getOperand(1)->getType(), - "Both operands to a binary operator are not of the same type!", &B); + Assert(B.getOperand(0)->getType() == B.getOperand(1)->getType(), + "Both operands to a binary operator are not of the same type!", &B); switch (B.getOpcode()) { // Check that integer arithmetic operators are only used with @@ -2007,11 +2017,12 @@ void Verifier::visitBinaryOperator(BinaryOperator &B) { case Instruction::UDiv: case Instruction::SRem: case Instruction::URem: - Assert1(B.getType()->isIntOrIntVectorTy(), - "Integer arithmetic operators only work with integral types!", &B); - Assert1(B.getType() == B.getOperand(0)->getType(), - "Integer arithmetic operators must have same type " - "for operands and result!", &B); + Assert(B.getType()->isIntOrIntVectorTy(), + "Integer arithmetic operators only work with integral types!", &B); + Assert(B.getType() == B.getOperand(0)->getType(), + "Integer arithmetic operators must have same type " + "for operands and result!", + &B); break; // Check that floating-point arithmetic operators are only used with // floating-point operands. @@ -2020,30 +2031,32 @@ void Verifier::visitBinaryOperator(BinaryOperator &B) { case Instruction::FMul: case Instruction::FDiv: case Instruction::FRem: - Assert1(B.getType()->isFPOrFPVectorTy(), - "Floating-point arithmetic operators only work with " - "floating-point types!", &B); - Assert1(B.getType() == B.getOperand(0)->getType(), - "Floating-point arithmetic operators must have same type " - "for operands and result!", &B); + Assert(B.getType()->isFPOrFPVectorTy(), + "Floating-point arithmetic operators only work with " + "floating-point types!", + &B); + Assert(B.getType() == B.getOperand(0)->getType(), + "Floating-point arithmetic operators must have same type " + "for operands and result!", + &B); break; // Check that logical operators are only used with integral operands. case Instruction::And: case Instruction::Or: case Instruction::Xor: - Assert1(B.getType()->isIntOrIntVectorTy(), - "Logical operators only work with integral types!", &B); - Assert1(B.getType() == B.getOperand(0)->getType(), - "Logical operators must have same type for operands and result!", - &B); + Assert(B.getType()->isIntOrIntVectorTy(), + "Logical operators only work with integral types!", &B); + Assert(B.getType() == B.getOperand(0)->getType(), + "Logical operators must have same type for operands and result!", + &B); break; case Instruction::Shl: case Instruction::LShr: case Instruction::AShr: - Assert1(B.getType()->isIntOrIntVectorTy(), - "Shifts only work with integral types!", &B); - Assert1(B.getType() == B.getOperand(0)->getType(), - "Shift return type must be same as operands!", &B); + Assert(B.getType()->isIntOrIntVectorTy(), + "Shifts only work with integral types!", &B); + Assert(B.getType() == B.getOperand(0)->getType(), + "Shift return type must be same as operands!", &B); break; default: llvm_unreachable("Unknown BinaryOperator opcode!"); @@ -2056,15 +2069,15 @@ void Verifier::visitICmpInst(ICmpInst &IC) { // Check that the operands are the same type Type *Op0Ty = IC.getOperand(0)->getType(); Type *Op1Ty = IC.getOperand(1)->getType(); - Assert1(Op0Ty == Op1Ty, - "Both operands to ICmp instruction are not of the same type!", &IC); + Assert(Op0Ty == Op1Ty, + "Both operands to ICmp instruction are not of the same type!", &IC); // Check that the operands are the right type - Assert1(Op0Ty->isIntOrIntVectorTy() || Op0Ty->getScalarType()->isPointerTy(), - "Invalid operand types for ICmp instruction", &IC); + Assert(Op0Ty->isIntOrIntVectorTy() || Op0Ty->getScalarType()->isPointerTy(), + "Invalid operand types for ICmp instruction", &IC); // Check that the predicate is valid. - Assert1(IC.getPredicate() >= CmpInst::FIRST_ICMP_PREDICATE && - IC.getPredicate() <= CmpInst::LAST_ICMP_PREDICATE, - "Invalid predicate in ICmp instruction!", &IC); + Assert(IC.getPredicate() >= CmpInst::FIRST_ICMP_PREDICATE && + IC.getPredicate() <= CmpInst::LAST_ICMP_PREDICATE, + "Invalid predicate in ICmp instruction!", &IC); visitInstruction(IC); } @@ -2073,72 +2086,72 @@ void Verifier::visitFCmpInst(FCmpInst &FC) { // Check that the operands are the same type Type *Op0Ty = FC.getOperand(0)->getType(); Type *Op1Ty = FC.getOperand(1)->getType(); - Assert1(Op0Ty == Op1Ty, - "Both operands to FCmp instruction are not of the same type!", &FC); + Assert(Op0Ty == Op1Ty, + "Both operands to FCmp instruction are not of the same type!", &FC); // Check that the operands are the right type - Assert1(Op0Ty->isFPOrFPVectorTy(), - "Invalid operand types for FCmp instruction", &FC); + Assert(Op0Ty->isFPOrFPVectorTy(), + "Invalid operand types for FCmp instruction", &FC); // Check that the predicate is valid. - Assert1(FC.getPredicate() >= CmpInst::FIRST_FCMP_PREDICATE && - FC.getPredicate() <= CmpInst::LAST_FCMP_PREDICATE, - "Invalid predicate in FCmp instruction!", &FC); + Assert(FC.getPredicate() >= CmpInst::FIRST_FCMP_PREDICATE && + FC.getPredicate() <= CmpInst::LAST_FCMP_PREDICATE, + "Invalid predicate in FCmp instruction!", &FC); visitInstruction(FC); } void Verifier::visitExtractElementInst(ExtractElementInst &EI) { - Assert1(ExtractElementInst::isValidOperands(EI.getOperand(0), - EI.getOperand(1)), - "Invalid extractelement operands!", &EI); + Assert( + ExtractElementInst::isValidOperands(EI.getOperand(0), EI.getOperand(1)), + "Invalid extractelement operands!", &EI); visitInstruction(EI); } void Verifier::visitInsertElementInst(InsertElementInst &IE) { - Assert1(InsertElementInst::isValidOperands(IE.getOperand(0), - IE.getOperand(1), - IE.getOperand(2)), - "Invalid insertelement operands!", &IE); + Assert(InsertElementInst::isValidOperands(IE.getOperand(0), IE.getOperand(1), + IE.getOperand(2)), + "Invalid insertelement operands!", &IE); visitInstruction(IE); } void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) { - Assert1(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1), - SV.getOperand(2)), - "Invalid shufflevector operands!", &SV); + Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1), + SV.getOperand(2)), + "Invalid shufflevector operands!", &SV); visitInstruction(SV); } void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { Type *TargetTy = GEP.getPointerOperandType()->getScalarType(); - Assert1(isa(TargetTy), - "GEP base pointer is not a vector or a vector of pointers", &GEP); - Assert1(cast(TargetTy)->getElementType()->isSized(), - "GEP into unsized type!", &GEP); - Assert1(GEP.getPointerOperandType()->isVectorTy() == - GEP.getType()->isVectorTy(), "Vector GEP must return a vector value", - &GEP); + Assert(isa(TargetTy), + "GEP base pointer is not a vector or a vector of pointers", &GEP); + Assert(cast(TargetTy)->getElementType()->isSized(), + "GEP into unsized type!", &GEP); + Assert(GEP.getPointerOperandType()->isVectorTy() == + GEP.getType()->isVectorTy(), + "Vector GEP must return a vector value", &GEP); SmallVector Idxs(GEP.idx_begin(), GEP.idx_end()); Type *ElTy = GetElementPtrInst::getIndexedType(GEP.getPointerOperandType(), Idxs); - Assert1(ElTy, "Invalid indices for GEP pointer type!", &GEP); + Assert(ElTy, "Invalid indices for GEP pointer type!", &GEP); - Assert2(GEP.getType()->getScalarType()->isPointerTy() && - cast(GEP.getType()->getScalarType())->getElementType() - == ElTy, "GEP is not of right type for indices!", &GEP, ElTy); + Assert(GEP.getType()->getScalarType()->isPointerTy() && + cast(GEP.getType()->getScalarType()) + ->getElementType() == ElTy, + "GEP is not of right type for indices!", &GEP, ElTy); if (GEP.getPointerOperandType()->isVectorTy()) { // Additional checks for vector GEPs. unsigned GepWidth = GEP.getPointerOperandType()->getVectorNumElements(); - Assert1(GepWidth == GEP.getType()->getVectorNumElements(), - "Vector GEP result width doesn't match operand's", &GEP); + Assert(GepWidth == GEP.getType()->getVectorNumElements(), + "Vector GEP result width doesn't match operand's", &GEP); for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { Type *IndexTy = Idxs[i]->getType(); - Assert1(IndexTy->isVectorTy(), - "Vector GEP must have vector indices!", &GEP); + Assert(IndexTy->isVectorTy(), "Vector GEP must have vector indices!", + &GEP); unsigned IndexWidth = IndexTy->getVectorNumElements(); - Assert1(IndexWidth == GepWidth, "Invalid GEP index vector width", &GEP); + Assert(IndexWidth == GepWidth, "Invalid GEP index vector width", &GEP); } } visitInstruction(GEP); @@ -2155,34 +2168,33 @@ void Verifier::visitRangeMetadata(Instruction& I, "precondition violation"); unsigned NumOperands = Range->getNumOperands(); - Assert1(NumOperands % 2 == 0, "Unfinished range!", Range); + Assert(NumOperands % 2 == 0, "Unfinished range!", Range); unsigned NumRanges = NumOperands / 2; - Assert1(NumRanges >= 1, "It should have at least one range!", Range); - + Assert(NumRanges >= 1, "It should have at least one range!", Range); + ConstantRange LastRange(1); // Dummy initial value for (unsigned i = 0; i < NumRanges; ++i) { ConstantInt *Low = mdconst::dyn_extract(Range->getOperand(2 * i)); - Assert1(Low, "The lower limit must be an integer!", Low); + Assert(Low, "The lower limit must be an integer!", Low); ConstantInt *High = mdconst::dyn_extract(Range->getOperand(2 * i + 1)); - Assert1(High, "The upper limit must be an integer!", High); - Assert1(High->getType() == Low->getType() && - High->getType() == Ty, "Range types must match instruction type!", - &I); - + Assert(High, "The upper limit must be an integer!", High); + Assert(High->getType() == Low->getType() && High->getType() == Ty, + "Range types must match instruction type!", &I); + APInt HighV = High->getValue(); APInt LowV = Low->getValue(); ConstantRange CurRange(LowV, HighV); - Assert1(!CurRange.isEmptySet() && !CurRange.isFullSet(), - "Range must not be empty!", Range); + Assert(!CurRange.isEmptySet() && !CurRange.isFullSet(), + "Range must not be empty!", Range); if (i != 0) { - Assert1(CurRange.intersectWith(LastRange).isEmptySet(), - "Intervals are overlapping", Range); - Assert1(LowV.sgt(LastRange.getLower()), "Intervals are not in order", - Range); - Assert1(!isContiguous(CurRange, LastRange), "Intervals are contiguous", - Range); + Assert(CurRange.intersectWith(LastRange).isEmptySet(), + "Intervals are overlapping", Range); + Assert(LowV.sgt(LastRange.getLower()), "Intervals are not in order", + Range); + Assert(!isContiguous(CurRange, LastRange), "Intervals are contiguous", + Range); } LastRange = ConstantRange(LowV, HighV); } @@ -2192,38 +2204,37 @@ void Verifier::visitRangeMetadata(Instruction& I, APInt FirstHigh = mdconst::dyn_extract(Range->getOperand(1))->getValue(); ConstantRange FirstRange(FirstLow, FirstHigh); - Assert1(FirstRange.intersectWith(LastRange).isEmptySet(), - "Intervals are overlapping", Range); - Assert1(!isContiguous(FirstRange, LastRange), "Intervals are contiguous", - Range); + Assert(FirstRange.intersectWith(LastRange).isEmptySet(), + "Intervals are overlapping", Range); + Assert(!isContiguous(FirstRange, LastRange), "Intervals are contiguous", + Range); } } void Verifier::visitLoadInst(LoadInst &LI) { PointerType *PTy = dyn_cast(LI.getOperand(0)->getType()); - Assert1(PTy, "Load operand must be a pointer.", &LI); + Assert(PTy, "Load operand must be a pointer.", &LI); Type *ElTy = PTy->getElementType(); - Assert2(ElTy == LI.getType(), - "Load result type does not match pointer operand type!", &LI, ElTy); - Assert1(LI.getAlignment() <= Value::MaximumAlignment, - "huge alignment values are unsupported", &LI); + Assert(ElTy == LI.getType(), + "Load result type does not match pointer operand type!", &LI, ElTy); + Assert(LI.getAlignment() <= Value::MaximumAlignment, + "huge alignment values are unsupported", &LI); if (LI.isAtomic()) { - Assert1(LI.getOrdering() != Release && LI.getOrdering() != AcquireRelease, - "Load cannot have Release ordering", &LI); - Assert1(LI.getAlignment() != 0, - "Atomic load must specify explicit alignment", &LI); + Assert(LI.getOrdering() != Release && LI.getOrdering() != AcquireRelease, + "Load cannot have Release ordering", &LI); + Assert(LI.getAlignment() != 0, + "Atomic load must specify explicit alignment", &LI); if (!ElTy->isPointerTy()) { - Assert2(ElTy->isIntegerTy(), - "atomic load operand must have integer type!", - &LI, ElTy); + Assert(ElTy->isIntegerTy(), "atomic load operand must have integer type!", + &LI, ElTy); unsigned Size = ElTy->getPrimitiveSizeInBits(); - Assert2(Size >= 8 && !(Size & (Size - 1)), - "atomic load operand must be power-of-two byte-sized integer", - &LI, ElTy); + Assert(Size >= 8 && !(Size & (Size - 1)), + "atomic load operand must be power-of-two byte-sized integer", &LI, + ElTy); } } else { - Assert1(LI.getSynchScope() == CrossThread, - "Non-atomic load cannot have SynchronizationScope specified", &LI); + Assert(LI.getSynchScope() == CrossThread, + "Non-atomic load cannot have SynchronizationScope specified", &LI); } visitInstruction(LI); @@ -2231,30 +2242,28 @@ void Verifier::visitLoadInst(LoadInst &LI) { void Verifier::visitStoreInst(StoreInst &SI) { PointerType *PTy = dyn_cast(SI.getOperand(1)->getType()); - Assert1(PTy, "Store operand must be a pointer.", &SI); + Assert(PTy, "Store operand must be a pointer.", &SI); Type *ElTy = PTy->getElementType(); - Assert2(ElTy == SI.getOperand(0)->getType(), - "Stored value type does not match pointer operand type!", - &SI, ElTy); - Assert1(SI.getAlignment() <= Value::MaximumAlignment, - "huge alignment values are unsupported", &SI); + Assert(ElTy == SI.getOperand(0)->getType(), + "Stored value type does not match pointer operand type!", &SI, ElTy); + Assert(SI.getAlignment() <= Value::MaximumAlignment, + "huge alignment values are unsupported", &SI); if (SI.isAtomic()) { - Assert1(SI.getOrdering() != Acquire && SI.getOrdering() != AcquireRelease, - "Store cannot have Acquire ordering", &SI); - Assert1(SI.getAlignment() != 0, - "Atomic store must specify explicit alignment", &SI); + Assert(SI.getOrdering() != Acquire && SI.getOrdering() != AcquireRelease, + "Store cannot have Acquire ordering", &SI); + Assert(SI.getAlignment() != 0, + "Atomic store must specify explicit alignment", &SI); if (!ElTy->isPointerTy()) { - Assert2(ElTy->isIntegerTy(), - "atomic store operand must have integer type!", - &SI, ElTy); + Assert(ElTy->isIntegerTy(), + "atomic store operand must have integer type!", &SI, ElTy); unsigned Size = ElTy->getPrimitiveSizeInBits(); - Assert2(Size >= 8 && !(Size & (Size - 1)), - "atomic store operand must be power-of-two byte-sized integer", - &SI, ElTy); + Assert(Size >= 8 && !(Size & (Size - 1)), + "atomic store operand must be power-of-two byte-sized integer", + &SI, ElTy); } } else { - Assert1(SI.getSynchScope() == CrossThread, - "Non-atomic store cannot have SynchronizationScope specified", &SI); + Assert(SI.getSynchScope() == CrossThread, + "Non-atomic store cannot have SynchronizationScope specified", &SI); } visitInstruction(SI); } @@ -2262,15 +2271,15 @@ void Verifier::visitStoreInst(StoreInst &SI) { void Verifier::visitAllocaInst(AllocaInst &AI) { SmallPtrSet Visited; PointerType *PTy = AI.getType(); - Assert1(PTy->getAddressSpace() == 0, - "Allocation instruction pointer not in the generic address space!", - &AI); - Assert1(PTy->getElementType()->isSized(&Visited), "Cannot allocate unsized type", - &AI); - Assert1(AI.getArraySize()->getType()->isIntegerTy(), - "Alloca array size must have integer type", &AI); - Assert1(AI.getAlignment() <= Value::MaximumAlignment, - "huge alignment values are unsupported", &AI); + Assert(PTy->getAddressSpace() == 0, + "Allocation instruction pointer not in the generic address space!", + &AI); + Assert(PTy->getElementType()->isSized(&Visited), + "Cannot allocate unsized type", &AI); + Assert(AI.getArraySize()->getType()->isIntegerTy(), + "Alloca array size must have integer type", &AI); + Assert(AI.getAlignment() <= Value::MaximumAlignment, + "huge alignment values are unsupported", &AI); visitInstruction(AI); } @@ -2278,87 +2287,83 @@ void Verifier::visitAllocaInst(AllocaInst &AI) { void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) { // FIXME: more conditions??? - Assert1(CXI.getSuccessOrdering() != NotAtomic, - "cmpxchg instructions must be atomic.", &CXI); - Assert1(CXI.getFailureOrdering() != NotAtomic, - "cmpxchg instructions must be atomic.", &CXI); - Assert1(CXI.getSuccessOrdering() != Unordered, - "cmpxchg instructions cannot be unordered.", &CXI); - Assert1(CXI.getFailureOrdering() != Unordered, - "cmpxchg instructions cannot be unordered.", &CXI); - Assert1(CXI.getSuccessOrdering() >= CXI.getFailureOrdering(), - "cmpxchg instructions be at least as constrained on success as fail", - &CXI); - Assert1(CXI.getFailureOrdering() != Release && - CXI.getFailureOrdering() != AcquireRelease, - "cmpxchg failure ordering cannot include release semantics", &CXI); + Assert(CXI.getSuccessOrdering() != NotAtomic, + "cmpxchg instructions must be atomic.", &CXI); + Assert(CXI.getFailureOrdering() != NotAtomic, + "cmpxchg instructions must be atomic.", &CXI); + Assert(CXI.getSuccessOrdering() != Unordered, + "cmpxchg instructions cannot be unordered.", &CXI); + Assert(CXI.getFailureOrdering() != Unordered, + "cmpxchg instructions cannot be unordered.", &CXI); + Assert(CXI.getSuccessOrdering() >= CXI.getFailureOrdering(), + "cmpxchg instructions be at least as constrained on success as fail", + &CXI); + Assert(CXI.getFailureOrdering() != Release && + CXI.getFailureOrdering() != AcquireRelease, + "cmpxchg failure ordering cannot include release semantics", &CXI); PointerType *PTy = dyn_cast(CXI.getOperand(0)->getType()); - Assert1(PTy, "First cmpxchg operand must be a pointer.", &CXI); + Assert(PTy, "First cmpxchg operand must be a pointer.", &CXI); Type *ElTy = PTy->getElementType(); - Assert2(ElTy->isIntegerTy(), - "cmpxchg operand must have integer type!", - &CXI, ElTy); + Assert(ElTy->isIntegerTy(), "cmpxchg operand must have integer type!", &CXI, + ElTy); unsigned Size = ElTy->getPrimitiveSizeInBits(); - Assert2(Size >= 8 && !(Size & (Size - 1)), - "cmpxchg operand must be power-of-two byte-sized integer", - &CXI, ElTy); - Assert2(ElTy == CXI.getOperand(1)->getType(), - "Expected value type does not match pointer operand type!", - &CXI, ElTy); - Assert2(ElTy == CXI.getOperand(2)->getType(), - "Stored value type does not match pointer operand type!", - &CXI, ElTy); + Assert(Size >= 8 && !(Size & (Size - 1)), + "cmpxchg operand must be power-of-two byte-sized integer", &CXI, ElTy); + Assert(ElTy == CXI.getOperand(1)->getType(), + "Expected value type does not match pointer operand type!", &CXI, + ElTy); + Assert(ElTy == CXI.getOperand(2)->getType(), + "Stored value type does not match pointer operand type!", &CXI, ElTy); visitInstruction(CXI); } void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) { - Assert1(RMWI.getOrdering() != NotAtomic, - "atomicrmw instructions must be atomic.", &RMWI); - Assert1(RMWI.getOrdering() != Unordered, - "atomicrmw instructions cannot be unordered.", &RMWI); + Assert(RMWI.getOrdering() != NotAtomic, + "atomicrmw instructions must be atomic.", &RMWI); + Assert(RMWI.getOrdering() != Unordered, + "atomicrmw instructions cannot be unordered.", &RMWI); PointerType *PTy = dyn_cast(RMWI.getOperand(0)->getType()); - Assert1(PTy, "First atomicrmw operand must be a pointer.", &RMWI); + Assert(PTy, "First atomicrmw operand must be a pointer.", &RMWI); Type *ElTy = PTy->getElementType(); - Assert2(ElTy->isIntegerTy(), - "atomicrmw operand must have integer type!", - &RMWI, ElTy); + Assert(ElTy->isIntegerTy(), "atomicrmw operand must have integer type!", + &RMWI, ElTy); unsigned Size = ElTy->getPrimitiveSizeInBits(); - Assert2(Size >= 8 && !(Size & (Size - 1)), - "atomicrmw operand must be power-of-two byte-sized integer", - &RMWI, ElTy); - Assert2(ElTy == RMWI.getOperand(1)->getType(), - "Argument value type does not match pointer operand type!", - &RMWI, ElTy); - Assert1(AtomicRMWInst::FIRST_BINOP <= RMWI.getOperation() && - RMWI.getOperation() <= AtomicRMWInst::LAST_BINOP, - "Invalid binary operation!", &RMWI); + Assert(Size >= 8 && !(Size & (Size - 1)), + "atomicrmw operand must be power-of-two byte-sized integer", &RMWI, + ElTy); + Assert(ElTy == RMWI.getOperand(1)->getType(), + "Argument value type does not match pointer operand type!", &RMWI, + ElTy); + Assert(AtomicRMWInst::FIRST_BINOP <= RMWI.getOperation() && + RMWI.getOperation() <= AtomicRMWInst::LAST_BINOP, + "Invalid binary operation!", &RMWI); visitInstruction(RMWI); } void Verifier::visitFenceInst(FenceInst &FI) { const AtomicOrdering Ordering = FI.getOrdering(); - Assert1(Ordering == Acquire || Ordering == Release || - Ordering == AcquireRelease || Ordering == SequentiallyConsistent, - "fence instructions may only have " - "acquire, release, acq_rel, or seq_cst ordering.", &FI); + Assert(Ordering == Acquire || Ordering == Release || + Ordering == AcquireRelease || Ordering == SequentiallyConsistent, + "fence instructions may only have " + "acquire, release, acq_rel, or seq_cst ordering.", + &FI); visitInstruction(FI); } void Verifier::visitExtractValueInst(ExtractValueInst &EVI) { - Assert1(ExtractValueInst::getIndexedType(EVI.getAggregateOperand()->getType(), - EVI.getIndices()) == - EVI.getType(), - "Invalid ExtractValueInst operands!", &EVI); + Assert(ExtractValueInst::getIndexedType(EVI.getAggregateOperand()->getType(), + EVI.getIndices()) == EVI.getType(), + "Invalid ExtractValueInst operands!", &EVI); visitInstruction(EVI); } void Verifier::visitInsertValueInst(InsertValueInst &IVI) { - Assert1(ExtractValueInst::getIndexedType(IVI.getAggregateOperand()->getType(), - IVI.getIndices()) == - IVI.getOperand(1)->getType(), - "Invalid InsertValueInst operands!", &IVI); + Assert(ExtractValueInst::getIndexedType(IVI.getAggregateOperand()->getType(), + IVI.getIndices()) == + IVI.getOperand(1)->getType(), + "Invalid InsertValueInst operands!", &IVI); visitInstruction(IVI); } @@ -2368,43 +2373,44 @@ void Verifier::visitLandingPadInst(LandingPadInst &LPI) { // The landingpad instruction is ill-formed if it doesn't have any clauses and // isn't a cleanup. - Assert1(LPI.getNumClauses() > 0 || LPI.isCleanup(), - "LandingPadInst needs at least one clause or to be a cleanup.", &LPI); + Assert(LPI.getNumClauses() > 0 || LPI.isCleanup(), + "LandingPadInst needs at least one clause or to be a cleanup.", &LPI); // The landingpad instruction defines its parent as a landing pad block. The // landing pad block may be branched to only by the unwind edge of an invoke. for (pred_iterator I = pred_begin(BB), E = pred_end(BB); I != E; ++I) { const InvokeInst *II = dyn_cast((*I)->getTerminator()); - Assert1(II && II->getUnwindDest() == BB && II->getNormalDest() != BB, - "Block containing LandingPadInst must be jumped to " - "only by the unwind edge of an invoke.", &LPI); + Assert(II && II->getUnwindDest() == BB && II->getNormalDest() != BB, + "Block containing LandingPadInst must be jumped to " + "only by the unwind edge of an invoke.", + &LPI); } // The landingpad instruction must be the first non-PHI instruction in the // block. - Assert1(LPI.getParent()->getLandingPadInst() == &LPI, - "LandingPadInst not the first non-PHI instruction in the block.", - &LPI); + Assert(LPI.getParent()->getLandingPadInst() == &LPI, + "LandingPadInst not the first non-PHI instruction in the block.", + &LPI); // The personality functions for all landingpad instructions within the same // function should match. if (PersonalityFn) - Assert1(LPI.getPersonalityFn() == PersonalityFn, - "Personality function doesn't match others in function", &LPI); + Assert(LPI.getPersonalityFn() == PersonalityFn, + "Personality function doesn't match others in function", &LPI); PersonalityFn = LPI.getPersonalityFn(); // All operands must be constants. - Assert1(isa(PersonalityFn), "Personality function is not constant!", - &LPI); + Assert(isa(PersonalityFn), "Personality function is not constant!", + &LPI); for (unsigned i = 0, e = LPI.getNumClauses(); i < e; ++i) { Constant *Clause = LPI.getClause(i); if (LPI.isCatch(i)) { - Assert1(isa(Clause->getType()), - "Catch operand does not have pointer type!", &LPI); + Assert(isa(Clause->getType()), + "Catch operand does not have pointer type!", &LPI); } else { - Assert1(LPI.isFilter(i), "Clause is neither catch nor filter!", &LPI); - Assert1(isa(Clause) || isa(Clause), - "Filter operand is not an array of constants!", &LPI); + Assert(LPI.isFilter(i), "Clause is neither catch nor filter!", &LPI); + Assert(isa(Clause) || isa(Clause), + "Filter operand is not an array of constants!", &LPI); } } @@ -2422,46 +2428,46 @@ void Verifier::verifyDominatesUse(Instruction &I, unsigned i) { } const Use &U = I.getOperandUse(i); - Assert2(InstsInThisBlock.count(Op) || DT.dominates(Op, U), - "Instruction does not dominate all uses!", Op, &I); + Assert(InstsInThisBlock.count(Op) || DT.dominates(Op, U), + "Instruction does not dominate all uses!", Op, &I); } /// verifyInstruction - Verify that an instruction is well formed. /// void Verifier::visitInstruction(Instruction &I) { BasicBlock *BB = I.getParent(); - Assert1(BB, "Instruction not embedded in basic block!", &I); + Assert(BB, "Instruction not embedded in basic block!", &I); if (!isa(I)) { // Check that non-phi nodes are not self referential for (User *U : I.users()) { - Assert1(U != (User*)&I || !DT.isReachableFromEntry(BB), - "Only PHI nodes may reference their own value!", &I); + Assert(U != (User *)&I || !DT.isReachableFromEntry(BB), + "Only PHI nodes may reference their own value!", &I); } } // Check that void typed values don't have names - Assert1(!I.getType()->isVoidTy() || !I.hasName(), - "Instruction has a name, but provides a void value!", &I); + Assert(!I.getType()->isVoidTy() || !I.hasName(), + "Instruction has a name, but provides a void value!", &I); // Check that the return value of the instruction is either void or a legal // value type. - Assert1(I.getType()->isVoidTy() || - I.getType()->isFirstClassType(), - "Instruction returns a non-scalar type!", &I); + Assert(I.getType()->isVoidTy() || I.getType()->isFirstClassType(), + "Instruction returns a non-scalar type!", &I); // Check that the instruction doesn't produce metadata. Calls are already // checked against the callee type. - Assert1(!I.getType()->isMetadataTy() || - isa(I) || isa(I), - "Invalid use of metadata!", &I); + Assert(!I.getType()->isMetadataTy() || isa(I) || isa(I), + "Invalid use of metadata!", &I); // Check that all uses of the instruction, if they are instructions // themselves, actually have parent basic blocks. If the use is not an // instruction, it is an error! for (Use &U : I.uses()) { if (Instruction *Used = dyn_cast(U.getUser())) - Assert2(Used->getParent() != nullptr, "Instruction referencing" - " instruction not embedded in a basic block!", &I, Used); + Assert(Used->getParent() != nullptr, + "Instruction referencing" + " instruction not embedded in a basic block!", + &I, Used); else { CheckFailed("Use of instruction is not an instruction!", U); return; @@ -2469,44 +2475,46 @@ void Verifier::visitInstruction(Instruction &I) { } for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { - Assert1(I.getOperand(i) != nullptr, "Instruction has null operand!", &I); + Assert(I.getOperand(i) != nullptr, "Instruction has null operand!", &I); // Check to make sure that only first-class-values are operands to // instructions. if (!I.getOperand(i)->getType()->isFirstClassType()) { - Assert1(0, "Instruction operands must be first-class values!", &I); + Assert(0, "Instruction operands must be first-class values!", &I); } if (Function *F = dyn_cast(I.getOperand(i))) { // Check to make sure that the "address of" an intrinsic function is never // taken. - Assert1(!F->isIntrinsic() || i == (isa(I) ? e-1 : - isa(I) ? e-3 : 0), - "Cannot take the address of an intrinsic!", &I); - Assert1(!F->isIntrinsic() || isa(I) || + Assert( + !F->isIntrinsic() || + i == (isa(I) ? e - 1 : isa(I) ? e - 3 : 0), + "Cannot take the address of an intrinsic!", &I); + Assert( + !F->isIntrinsic() || isa(I) || F->getIntrinsicID() == Intrinsic::donothing || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 || F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint, - "Cannot invoke an intrinsinc other than" - " donothing or patchpoint", &I); - Assert1(F->getParent() == M, "Referencing function in another module!", - &I); + "Cannot invoke an intrinsinc other than" + " donothing or patchpoint", + &I); + Assert(F->getParent() == M, "Referencing function in another module!", + &I); } else if (BasicBlock *OpBB = dyn_cast(I.getOperand(i))) { - Assert1(OpBB->getParent() == BB->getParent(), - "Referring to a basic block in another function!", &I); + Assert(OpBB->getParent() == BB->getParent(), + "Referring to a basic block in another function!", &I); } else if (Argument *OpArg = dyn_cast(I.getOperand(i))) { - Assert1(OpArg->getParent() == BB->getParent(), - "Referring to an argument in another function!", &I); + Assert(OpArg->getParent() == BB->getParent(), + "Referring to an argument in another function!", &I); } else if (GlobalValue *GV = dyn_cast(I.getOperand(i))) { - Assert1(GV->getParent() == M, "Referencing global in another module!", - &I); + Assert(GV->getParent() == M, "Referencing global in another module!", &I); } else if (isa(I.getOperand(i))) { verifyDominatesUse(I, i); } else if (isa(I.getOperand(i))) { - Assert1((i + 1 == e && isa(I)) || - (i + 3 == e && isa(I)), - "Cannot take the address of an inline asm!", &I); + Assert((i + 1 == e && isa(I)) || + (i + 3 == e && isa(I)), + "Cannot take the address of an inline asm!", &I); } else if (ConstantExpr *CE = dyn_cast(I.getOperand(i))) { if (CE->getType()->isPtrOrPtrVectorTy()) { // If we have a ConstantExpr pointer, we need to see if it came from an @@ -2532,31 +2540,37 @@ void Verifier::visitInstruction(Instruction &I) { } if (MDNode *MD = I.getMetadata(LLVMContext::MD_fpmath)) { - Assert1(I.getType()->isFPOrFPVectorTy(), - "fpmath requires a floating point result!", &I); - Assert1(MD->getNumOperands() == 1, "fpmath takes one operand!", &I); + Assert(I.getType()->isFPOrFPVectorTy(), + "fpmath requires a floating point result!", &I); + Assert(MD->getNumOperands() == 1, "fpmath takes one operand!", &I); if (ConstantFP *CFP0 = mdconst::dyn_extract_or_null(MD->getOperand(0))) { APFloat Accuracy = CFP0->getValueAPF(); - Assert1(Accuracy.isFiniteNonZero() && !Accuracy.isNegative(), - "fpmath accuracy not a positive number!", &I); + Assert(Accuracy.isFiniteNonZero() && !Accuracy.isNegative(), + "fpmath accuracy not a positive number!", &I); } else { - Assert1(false, "invalid fpmath accuracy!", &I); + Assert(false, "invalid fpmath accuracy!", &I); } } if (MDNode *Range = I.getMetadata(LLVMContext::MD_range)) { - Assert1(isa(I) || isa(I) || isa(I), - "Ranges are only for loads, calls and invokes!", &I); + Assert(isa(I) || isa(I) || isa(I), + "Ranges are only for loads, calls and invokes!", &I); visitRangeMetadata(I, Range, I.getType()); } if (I.getMetadata(LLVMContext::MD_nonnull)) { - Assert1(I.getType()->isPointerTy(), - "nonnull applies only to pointer types", &I); - Assert1(isa(I), - "nonnull applies only to load instructions, use attributes" - " for calls or invokes", &I); + Assert(I.getType()->isPointerTy(), "nonnull applies only to pointer types", + &I); + Assert(isa(I), + "nonnull applies only to load instructions, use attributes" + " for calls or invokes", + &I); + } + + if (MDNode *N = I.getDebugLoc().getAsMDNode()) { + Assert(isa(N), "invalid !dbg metadata attachment", &I, N); + visitMDNode(*N); } InstsInThisBlock.insert(&I); @@ -2717,7 +2731,7 @@ Verifier::VerifyIntrinsicIsVarArg(bool isVarArg, // If there are no descriptors left, then it can't be a vararg. if (Infos.empty()) - return isVarArg ? true : false; + return isVarArg; // There should be only one descriptor remaining at this point. if (Infos.size() != 1) @@ -2727,7 +2741,7 @@ Verifier::VerifyIntrinsicIsVarArg(bool isVarArg, IITDescriptor D = Infos.front(); Infos = Infos.slice(1); if (D.Kind == IITDescriptor::VarArg) - return isVarArg ? false : true; + return !isVarArg; return true; } @@ -2736,8 +2750,8 @@ Verifier::VerifyIntrinsicIsVarArg(bool isVarArg, /// void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { Function *IF = CI.getCalledFunction(); - Assert1(IF->isDeclaration(), "Intrinsic functions should never be defined!", - IF); + Assert(IF->isDeclaration(), "Intrinsic functions should never be defined!", + IF); // Verify that the intrinsic prototype lines up with what the .td files // describe. @@ -2749,31 +2763,33 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { ArrayRef TableRef = Table; SmallVector ArgTys; - Assert1(!VerifyIntrinsicType(IFTy->getReturnType(), TableRef, ArgTys), - "Intrinsic has incorrect return type!", IF); + Assert(!VerifyIntrinsicType(IFTy->getReturnType(), TableRef, ArgTys), + "Intrinsic has incorrect return type!", IF); for (unsigned i = 0, e = IFTy->getNumParams(); i != e; ++i) - Assert1(!VerifyIntrinsicType(IFTy->getParamType(i), TableRef, ArgTys), - "Intrinsic has incorrect argument type!", IF); + Assert(!VerifyIntrinsicType(IFTy->getParamType(i), TableRef, ArgTys), + "Intrinsic has incorrect argument type!", IF); // Verify if the intrinsic call matches the vararg property. if (IsVarArg) - Assert1(!VerifyIntrinsicIsVarArg(IsVarArg, TableRef), - "Intrinsic was not defined with variable arguments!", IF); + Assert(!VerifyIntrinsicIsVarArg(IsVarArg, TableRef), + "Intrinsic was not defined with variable arguments!", IF); else - Assert1(!VerifyIntrinsicIsVarArg(IsVarArg, TableRef), - "Callsite was not defined with variable arguments!", IF); + Assert(!VerifyIntrinsicIsVarArg(IsVarArg, TableRef), + "Callsite was not defined with variable arguments!", IF); // All descriptors should be absorbed by now. - Assert1(TableRef.empty(), "Intrinsic has too few arguments!", IF); + Assert(TableRef.empty(), "Intrinsic has too few arguments!", IF); // Now that we have the intrinsic ID and the actual argument types (and we // know they are legal for the intrinsic!) get the intrinsic name through the // usual means. This allows us to verify the mangling of argument types into // the name. const std::string ExpectedName = Intrinsic::getName(ID, ArgTys); - Assert1(ExpectedName == IF->getName(), - "Intrinsic name not mangled correctly for type arguments! " - "Should be: " + ExpectedName, IF); + Assert(ExpectedName == IF->getName(), + "Intrinsic name not mangled correctly for type arguments! " + "Should be: " + + ExpectedName, + IF); // If the intrinsic takes MDNode arguments, verify that they are either global // or are local to *this* function. @@ -2786,95 +2802,123 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { break; case Intrinsic::ctlz: // llvm.ctlz case Intrinsic::cttz: // llvm.cttz - Assert1(isa(CI.getArgOperand(1)), - "is_zero_undef argument of bit counting intrinsics must be a " - "constant int", &CI); + Assert(isa(CI.getArgOperand(1)), + "is_zero_undef argument of bit counting intrinsics must be a " + "constant int", + &CI); + break; + case Intrinsic::dbg_declare: // llvm.dbg.declare + Assert(isa(CI.getArgOperand(0)), + "invalid llvm.dbg.declare intrinsic call 1", &CI); + visitDbgIntrinsic("declare", cast(CI)); + break; + case Intrinsic::dbg_value: // llvm.dbg.value + visitDbgIntrinsic("value", cast(CI)); break; - case Intrinsic::dbg_declare: { // llvm.dbg.declare - Assert1(CI.getArgOperand(0) && isa(CI.getArgOperand(0)), - "invalid llvm.dbg.declare intrinsic call 1", &CI); - } break; case Intrinsic::memcpy: case Intrinsic::memmove: - case Intrinsic::memset: - Assert1(isa(CI.getArgOperand(3)), - "alignment argument of memory intrinsics must be a constant int", - &CI); - Assert1(isa(CI.getArgOperand(4)), - "isvolatile argument of memory intrinsics must be a constant int", - &CI); + case Intrinsic::memset: { + ConstantInt *AlignCI = dyn_cast(CI.getArgOperand(3)); + Assert(AlignCI, + "alignment argument of memory intrinsics must be a constant int", + &CI); + const APInt &AlignVal = AlignCI->getValue(); + Assert(AlignCI->isZero() || AlignVal.isPowerOf2(), + "alignment argument of memory intrinsics must be a power of 2", &CI); + Assert(isa(CI.getArgOperand(4)), + "isvolatile argument of memory intrinsics must be a constant int", + &CI); break; + } case Intrinsic::gcroot: case Intrinsic::gcwrite: case Intrinsic::gcread: if (ID == Intrinsic::gcroot) { AllocaInst *AI = dyn_cast(CI.getArgOperand(0)->stripPointerCasts()); - Assert1(AI, "llvm.gcroot parameter #1 must be an alloca.", &CI); - Assert1(isa(CI.getArgOperand(1)), - "llvm.gcroot parameter #2 must be a constant.", &CI); + Assert(AI, "llvm.gcroot parameter #1 must be an alloca.", &CI); + Assert(isa(CI.getArgOperand(1)), + "llvm.gcroot parameter #2 must be a constant.", &CI); if (!AI->getType()->getElementType()->isPointerTy()) { - Assert1(!isa(CI.getArgOperand(1)), - "llvm.gcroot parameter #1 must either be a pointer alloca, " - "or argument #2 must be a non-null constant.", &CI); + Assert(!isa(CI.getArgOperand(1)), + "llvm.gcroot parameter #1 must either be a pointer alloca, " + "or argument #2 must be a non-null constant.", + &CI); } } - Assert1(CI.getParent()->getParent()->hasGC(), - "Enclosing function does not use GC.", &CI); + Assert(CI.getParent()->getParent()->hasGC(), + "Enclosing function does not use GC.", &CI); break; case Intrinsic::init_trampoline: - Assert1(isa(CI.getArgOperand(1)->stripPointerCasts()), - "llvm.init_trampoline parameter #2 must resolve to a function.", - &CI); + Assert(isa(CI.getArgOperand(1)->stripPointerCasts()), + "llvm.init_trampoline parameter #2 must resolve to a function.", + &CI); break; case Intrinsic::prefetch: - Assert1(isa(CI.getArgOperand(1)) && - isa(CI.getArgOperand(2)) && - cast(CI.getArgOperand(1))->getZExtValue() < 2 && - cast(CI.getArgOperand(2))->getZExtValue() < 4, - "invalid arguments to llvm.prefetch", - &CI); + Assert(isa(CI.getArgOperand(1)) && + isa(CI.getArgOperand(2)) && + cast(CI.getArgOperand(1))->getZExtValue() < 2 && + cast(CI.getArgOperand(2))->getZExtValue() < 4, + "invalid arguments to llvm.prefetch", &CI); break; case Intrinsic::stackprotector: - Assert1(isa(CI.getArgOperand(1)->stripPointerCasts()), - "llvm.stackprotector parameter #2 must resolve to an alloca.", - &CI); + Assert(isa(CI.getArgOperand(1)->stripPointerCasts()), + "llvm.stackprotector parameter #2 must resolve to an alloca.", &CI); break; case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: case Intrinsic::invariant_start: - Assert1(isa(CI.getArgOperand(0)), - "size argument of memory use markers must be a constant integer", - &CI); + Assert(isa(CI.getArgOperand(0)), + "size argument of memory use markers must be a constant integer", + &CI); break; case Intrinsic::invariant_end: - Assert1(isa(CI.getArgOperand(1)), - "llvm.invariant.end parameter #2 must be a constant integer", &CI); + Assert(isa(CI.getArgOperand(1)), + "llvm.invariant.end parameter #2 must be a constant integer", &CI); break; - case Intrinsic::frameallocate: { + case Intrinsic::frameescape: { BasicBlock *BB = CI.getParent(); - Assert1(BB == &BB->getParent()->front(), - "llvm.frameallocate used outside of entry block", &CI); - Assert1(!SawFrameAllocate, - "multiple calls to llvm.frameallocate in one function", &CI); - SawFrameAllocate = true; - Assert1(isa(CI.getArgOperand(0)), - "llvm.frameallocate argument must be constant integer size", &CI); + Assert(BB == &BB->getParent()->front(), + "llvm.frameescape used outside of entry block", &CI); + Assert(!SawFrameEscape, + "multiple calls to llvm.frameescape in one function", &CI); + for (Value *Arg : CI.arg_operands()) { + auto *AI = dyn_cast(Arg->stripPointerCasts()); + Assert(AI && AI->isStaticAlloca(), + "llvm.frameescape only accepts static allocas", &CI); + } + FrameEscapeInfo[BB->getParent()].first = CI.getNumArgOperands(); + SawFrameEscape = true; break; } case Intrinsic::framerecover: { Value *FnArg = CI.getArgOperand(0)->stripPointerCasts(); Function *Fn = dyn_cast(FnArg); - Assert1(Fn && !Fn->isDeclaration(), "llvm.framerecover first " - "argument must be function defined in this module", &CI); + Assert(Fn && !Fn->isDeclaration(), + "llvm.framerecover first " + "argument must be function defined in this module", + &CI); + auto *IdxArg = dyn_cast(CI.getArgOperand(2)); + Assert(IdxArg, "idx argument of llvm.framerecover must be a constant int", + &CI); + auto &Entry = FrameEscapeInfo[Fn]; + Entry.second = unsigned( + std::max(uint64_t(Entry.second), IdxArg->getLimitedValue(~0U) + 1)); + break; + } + + case Intrinsic::eh_unwindhelp: { + auto *AI = dyn_cast(CI.getArgOperand(0)->stripPointerCasts()); + Assert(AI && AI->isStaticAlloca(), + "llvm.eh.unwindhelp requires a static alloca", &CI); break; } case Intrinsic::experimental_gc_statepoint: - Assert1(!CI.isInlineAsm(), - "gc.statepoint support for inline assembly unimplemented", &CI); + Assert(!CI.isInlineAsm(), + "gc.statepoint support for inline assembly unimplemented", &CI); VerifyStatepoint(ImmutableCallSite(&CI)); break; @@ -2886,56 +2930,52 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { CallSite StatepointCS(CI.getArgOperand(0)); const Function *StatepointFn = StatepointCS.getInstruction() ? StatepointCS.getCalledFunction() : nullptr; - Assert2(StatepointFn && StatepointFn->isDeclaration() && - StatepointFn->getIntrinsicID() == Intrinsic::experimental_gc_statepoint, - "gc.result operand #1 must be from a statepoint", - &CI, CI.getArgOperand(0)); + Assert(StatepointFn && StatepointFn->isDeclaration() && + StatepointFn->getIntrinsicID() == + Intrinsic::experimental_gc_statepoint, + "gc.result operand #1 must be from a statepoint", &CI, + CI.getArgOperand(0)); // Assert that result type matches wrapped callee. const Value *Target = StatepointCS.getArgument(0); const PointerType *PT = cast(Target->getType()); const FunctionType *TargetFuncType = cast(PT->getElementType()); - Assert1(CI.getType() == TargetFuncType->getReturnType(), - "gc.result result type does not match wrapped callee", - &CI); + Assert(CI.getType() == TargetFuncType->getReturnType(), + "gc.result result type does not match wrapped callee", &CI); break; } case Intrinsic::experimental_gc_relocate: { - Assert1(CI.getNumArgOperands() == 3, "wrong number of arguments", &CI); + Assert(CI.getNumArgOperands() == 3, "wrong number of arguments", &CI); // Check that this relocate is correctly tied to the statepoint // This is case for relocate on the unwinding path of an invoke statepoint if (ExtractValueInst *ExtractValue = dyn_cast(CI.getArgOperand(0))) { - Assert1(isa(ExtractValue->getAggregateOperand()), - "gc relocate on unwind path incorrectly linked to the statepoint", - &CI); + Assert(isa(ExtractValue->getAggregateOperand()), + "gc relocate on unwind path incorrectly linked to the statepoint", + &CI); const BasicBlock *invokeBB = ExtractValue->getParent()->getUniquePredecessor(); // Landingpad relocates should have only one predecessor with invoke // statepoint terminator - Assert1(invokeBB, - "safepoints should have unique landingpads", - ExtractValue->getParent()); - Assert1(invokeBB->getTerminator(), - "safepoint block should be well formed", - invokeBB); - Assert1(isStatepoint(invokeBB->getTerminator()), - "gc relocate should be linked to a statepoint", - invokeBB); + Assert(invokeBB, "safepoints should have unique landingpads", + ExtractValue->getParent()); + Assert(invokeBB->getTerminator(), "safepoint block should be well formed", + invokeBB); + Assert(isStatepoint(invokeBB->getTerminator()), + "gc relocate should be linked to a statepoint", invokeBB); } else { // In all other cases relocate should be tied to the statepoint directly. // This covers relocates on a normal return path of invoke statepoint and // relocates of a call statepoint auto Token = CI.getArgOperand(0); - Assert2(isa(Token) && isStatepoint(cast(Token)), - "gc relocate is incorrectly tied to the statepoint", - &CI, Token); + Assert(isa(Token) && isStatepoint(cast(Token)), + "gc relocate is incorrectly tied to the statepoint", &CI, Token); } // Verify rest of the relocate arguments @@ -2945,53 +2985,74 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { // Both the base and derived must be piped through the safepoint Value* Base = CI.getArgOperand(1); - Assert1(isa(Base), - "gc.relocate operand #2 must be integer offset", &CI); - + Assert(isa(Base), + "gc.relocate operand #2 must be integer offset", &CI); + Value* Derived = CI.getArgOperand(2); - Assert1(isa(Derived), - "gc.relocate operand #3 must be integer offset", &CI); + Assert(isa(Derived), + "gc.relocate operand #3 must be integer offset", &CI); const int BaseIndex = cast(Base)->getZExtValue(); const int DerivedIndex = cast(Derived)->getZExtValue(); // Check the bounds - Assert1(0 <= BaseIndex && - BaseIndex < (int)StatepointCS.arg_size(), - "gc.relocate: statepoint base index out of bounds", &CI); - Assert1(0 <= DerivedIndex && - DerivedIndex < (int)StatepointCS.arg_size(), - "gc.relocate: statepoint derived index out of bounds", &CI); + Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCS.arg_size(), + "gc.relocate: statepoint base index out of bounds", &CI); + Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCS.arg_size(), + "gc.relocate: statepoint derived index out of bounds", &CI); // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' // section of the statepoint's argument - const int NumCallArgs = + Assert(StatepointCS.arg_size() > 0, + "gc.statepoint: insufficient arguments"); + Assert(isa(StatepointCS.getArgument(1)), + "gc.statement: number of call arguments must be constant integer"); + const unsigned NumCallArgs = cast(StatepointCS.getArgument(1))->getZExtValue(); + Assert(StatepointCS.arg_size() > NumCallArgs+3, + "gc.statepoint: mismatch in number of call arguments"); + Assert(isa(StatepointCS.getArgument(NumCallArgs+3)), + "gc.statepoint: number of deoptimization arguments must be " + "a constant integer"); const int NumDeoptArgs = cast(StatepointCS.getArgument(NumCallArgs + 3))->getZExtValue(); const int GCParamArgsStart = NumCallArgs + NumDeoptArgs + 4; const int GCParamArgsEnd = StatepointCS.arg_size(); - Assert1(GCParamArgsStart <= BaseIndex && - BaseIndex < GCParamArgsEnd, - "gc.relocate: statepoint base index doesn't fall within the " - "'gc parameters' section of the statepoint call", &CI); - Assert1(GCParamArgsStart <= DerivedIndex && - DerivedIndex < GCParamArgsEnd, - "gc.relocate: statepoint derived index doesn't fall within the " - "'gc parameters' section of the statepoint call", &CI); - + Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, + "gc.relocate: statepoint base index doesn't fall within the " + "'gc parameters' section of the statepoint call", + &CI); + Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, + "gc.relocate: statepoint derived index doesn't fall within the " + "'gc parameters' section of the statepoint call", + &CI); // Assert that the result type matches the type of the relocated pointer GCRelocateOperands Operands(&CI); - Assert1(Operands.derivedPtr()->getType() == CI.getType(), - "gc.relocate: relocating a pointer shouldn't change its type", - &CI); + Assert(Operands.derivedPtr()->getType() == CI.getType(), + "gc.relocate: relocating a pointer shouldn't change its type", &CI); break; } }; } -void DebugInfoVerifier::verifyDebugInfo() { - if (!VerifyDebugInfo) +template +void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) { + auto *MD = cast(DII.getArgOperand(0))->getMetadata(); + Assert(isa(MD) || + (isa(MD) && !cast(MD)->getNumOperands()), + "invalid llvm.dbg." + Kind + " intrinsic address/value", &DII, MD); + Assert(isa(DII.getRawVariable()), + "invalid llvm.dbg." + Kind + " intrinsic variable", &DII, + DII.getRawVariable()); + Assert(isa(DII.getRawExpression()), + "invalid llvm.dbg." + Kind + " intrinsic expression", &DII, + DII.getRawExpression()); +} + +void Verifier::verifyDebugInfo() { + // Run the debug info verifier only if the regular verifier succeeds, since + // sometimes checks that have already failed will cause crashes here. + if (EverBroken || !VerifyDebugInfo) return; DebugInfoFinder Finder; @@ -3002,23 +3063,23 @@ void DebugInfoVerifier::verifyDebugInfo() { // // NOTE: The loud braces are necessary for MSVC compatibility. for (DICompileUnit CU : Finder.compile_units()) { - Assert1(CU.Verify(), "DICompileUnit does not Verify!", CU); + Assert(CU.Verify(), "DICompileUnit does not Verify!", CU); } for (DISubprogram S : Finder.subprograms()) { - Assert1(S.Verify(), "DISubprogram does not Verify!", S); + Assert(S.Verify(), "DISubprogram does not Verify!", S); } for (DIGlobalVariable GV : Finder.global_variables()) { - Assert1(GV.Verify(), "DIGlobalVariable does not Verify!", GV); + Assert(GV.Verify(), "DIGlobalVariable does not Verify!", GV); } for (DIType T : Finder.types()) { - Assert1(T.Verify(), "DIType does not Verify!", T); + Assert(T.Verify(), "DIType does not Verify!", T); } for (DIScope S : Finder.scopes()) { - Assert1(S.Verify(), "DIScope does not Verify!", S); + Assert(S.Verify(), "DIScope does not Verify!", S); } } -void DebugInfoVerifier::processInstructions(DebugInfoFinder &Finder) { +void Verifier::processInstructions(DebugInfoFinder &Finder) { for (const Function &F : *M) for (auto I = inst_begin(&F), E = inst_end(&F); I != E; ++I) { if (MDNode *MD = I->getMetadata(LLVMContext::MD_dbg)) @@ -3028,25 +3089,16 @@ void DebugInfoVerifier::processInstructions(DebugInfoFinder &Finder) { } } -void DebugInfoVerifier::processCallInst(DebugInfoFinder &Finder, - const CallInst &CI) { +void Verifier::processCallInst(DebugInfoFinder &Finder, const CallInst &CI) { if (Function *F = CI.getCalledFunction()) if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID()) switch (ID) { - case Intrinsic::dbg_declare: { - auto *DDI = cast(&CI); - Finder.processDeclare(*M, DDI); - if (auto E = DDI->getExpression()) - Assert1(DIExpression(E).Verify(), "DIExpression does not Verify!", E); + case Intrinsic::dbg_declare: + Finder.processDeclare(*M, cast(&CI)); break; - } - case Intrinsic::dbg_value: { - auto *DVI = cast(&CI); - Finder.processValue(*M, DVI); - if (auto E = DVI->getExpression()) - Assert1(DIExpression(E).Verify(), "DIExpression does not Verify!", E); + case Intrinsic::dbg_value: + Finder.processValue(*M, cast(&CI)); break; - } default: break; } @@ -3079,8 +3131,7 @@ bool llvm::verifyModule(const Module &M, raw_ostream *OS) { // Note that this function's return value is inverted from what you would // expect of a function called "verify". - DebugInfoVerifier DIV(OS ? *OS : NullStr); - return !V.verify(M) || !DIV.verify(M) || Broken; + return !V.verify(M) || Broken; } namespace { @@ -3090,7 +3141,7 @@ struct VerifierLegacyPass : public FunctionPass { Verifier V; bool FatalErrors; - VerifierLegacyPass() : FunctionPass(ID), FatalErrors(true) { + VerifierLegacyPass() : FunctionPass(ID), V(dbgs()), FatalErrors(true) { initializeVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); } explicit VerifierLegacyPass(bool FatalErrors) @@ -3116,48 +3167,15 @@ struct VerifierLegacyPass : public FunctionPass { AU.setPreservesAll(); } }; -struct DebugInfoVerifierLegacyPass : public ModulePass { - static char ID; - - DebugInfoVerifier V; - bool FatalErrors; - - DebugInfoVerifierLegacyPass() : ModulePass(ID), FatalErrors(true) { - initializeDebugInfoVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); - } - explicit DebugInfoVerifierLegacyPass(bool FatalErrors) - : ModulePass(ID), V(dbgs()), FatalErrors(FatalErrors) { - initializeDebugInfoVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); - } - - bool runOnModule(Module &M) override { - if (!V.verify(M) && FatalErrors) - report_fatal_error("Broken debug info found, compilation aborted!"); - - return false; - } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesAll(); - } -}; } char VerifierLegacyPass::ID = 0; INITIALIZE_PASS(VerifierLegacyPass, "verify", "Module Verifier", false, false) -char DebugInfoVerifierLegacyPass::ID = 0; -INITIALIZE_PASS(DebugInfoVerifierLegacyPass, "verify-di", "Debug Info Verifier", - false, false) - FunctionPass *llvm::createVerifierPass(bool FatalErrors) { return new VerifierLegacyPass(FatalErrors); } -ModulePass *llvm::createDebugInfoVerifierPass(bool FatalErrors) { - return new DebugInfoVerifierLegacyPass(FatalErrors); -} - PreservedAnalyses VerifierPass::run(Module &M) { if (verifyModule(M, &dbgs()) && FatalErrors) report_fatal_error("Broken module found, compilation aborted!"); diff --git a/lib/LLVMBuild.txt b/lib/LLVMBuild.txt index ad5b22b..bc2448d 100644 --- a/lib/LLVMBuild.txt +++ b/lib/LLVMBuild.txt @@ -16,7 +16,7 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = Analysis AsmParser Bitcode CodeGen DebugInfo ExecutionEngine LineEditor Linker IR IRReader LTO MC Object Option ProfileData Support TableGen Target Transforms +subdirectories = Analysis AsmParser Bitcode CodeGen DebugInfo ExecutionEngine LineEditor Linker IR IRReader LTO MC Object Option Passes ProfileData Support TableGen Target Transforms [component_0] type = Group diff --git a/lib/LTO/LTOCodeGenerator.cpp b/lib/LTO/LTOCodeGenerator.cpp index 61c2749..a6f980b 100644 --- a/lib/LTO/LTOCodeGenerator.cpp +++ b/lib/LTO/LTOCodeGenerator.cpp @@ -71,7 +71,7 @@ LTOCodeGenerator::LTOCodeGenerator() LTOCodeGenerator::LTOCodeGenerator(std::unique_ptr Context) : OwnedContext(std::move(Context)), Context(*OwnedContext), - IRLinker(new Module("ld-temp.o", *OwnedContext)) { + IRLinker(new Module("ld-temp.o", *OwnedContext)), OptLevel(2) { initialize(); } @@ -291,12 +291,11 @@ const void *LTOCodeGenerator::compileOptimized(size_t *length, bool LTOCodeGenerator::compile_to_file(const char **name, - bool disableOpt, bool disableInline, bool disableGVNLoadPRE, bool disableVectorization, std::string &errMsg) { - if (!optimize(disableOpt, disableInline, disableGVNLoadPRE, + if (!optimize(disableInline, disableGVNLoadPRE, disableVectorization, errMsg)) return false; @@ -304,12 +303,11 @@ bool LTOCodeGenerator::compile_to_file(const char **name, } const void* LTOCodeGenerator::compile(size_t *length, - bool disableOpt, bool disableInline, bool disableGVNLoadPRE, bool disableVectorization, std::string &errMsg) { - if (!optimize(disableOpt, disableInline, disableGVNLoadPRE, + if (!optimize(disableInline, disableGVNLoadPRE, disableVectorization, errMsg)) return nullptr; @@ -363,9 +361,25 @@ bool LTOCodeGenerator::determineTarget(std::string &errMsg) { MCpu = "cyclone"; } + CodeGenOpt::Level CGOptLevel; + switch (OptLevel) { + case 0: + CGOptLevel = CodeGenOpt::None; + break; + case 1: + CGOptLevel = CodeGenOpt::Less; + break; + case 2: + CGOptLevel = CodeGenOpt::Default; + break; + case 3: + CGOptLevel = CodeGenOpt::Aggressive; + break; + } + TargetMach = march->createTargetMachine(TripleStr, MCpu, FeatureStr, Options, RelocModel, CodeModel::Default, - CodeGenOpt::Aggressive); + CGOptLevel); return true; } @@ -457,7 +471,6 @@ void LTOCodeGenerator::applyScopeRestrictions() { // Start off with a verification pass. legacy::PassManager passes; passes.add(createVerifierPass()); - passes.add(createDebugInfoVerifierPass()); // mark which symbols can not be internalized Mangler Mangler(TargetMach->getDataLayout()); @@ -512,8 +525,7 @@ void LTOCodeGenerator::applyScopeRestrictions() { } /// Optimize merged modules using various IPO passes -bool LTOCodeGenerator::optimize(bool DisableOpt, - bool DisableInline, +bool LTOCodeGenerator::optimize(bool DisableInline, bool DisableGVNLoadPRE, bool DisableVectorization, std::string &errMsg) { @@ -529,9 +541,8 @@ bool LTOCodeGenerator::optimize(bool DisableOpt, legacy::PassManager passes; // Add an appropriate DataLayout instance for this module... - mergedModule->setDataLayout(TargetMach->getDataLayout()); + mergedModule->setDataLayout(*TargetMach->getDataLayout()); - passes.add(new DataLayoutPass()); passes.add( createTargetTransformInfoWrapperPass(TargetMach->getTargetIRAnalysis())); @@ -543,8 +554,7 @@ bool LTOCodeGenerator::optimize(bool DisableOpt, if (!DisableInline) PMB.Inliner = createFunctionInliningPass(); PMB.LibraryInfo = new TargetLibraryInfoImpl(TargetTriple); - if (DisableOpt) - PMB.OptLevel = 0; + PMB.OptLevel = OptLevel; PMB.VerifyInput = true; PMB.VerifyOutput = true; @@ -567,8 +577,6 @@ bool LTOCodeGenerator::compileOptimized(raw_ostream &out, std::string &errMsg) { legacy::PassManager codeGenPasses; - codeGenPasses.add(new DataLayoutPass()); - formatted_raw_ostream Out(out); // If the bitcode files contain ARC code and were compiled with optimization, diff --git a/lib/LTO/LTOModule.cpp b/lib/LTO/LTOModule.cpp index 0d07791..49aa97d 100644 --- a/lib/LTO/LTOModule.cpp +++ b/lib/LTO/LTOModule.cpp @@ -179,7 +179,8 @@ static Module *parseBitcodeFileImpl(MemoryBufferRef Buffer, std::unique_ptr LightweightBuf = MemoryBuffer::getMemBuffer(*MBOrErr, false); ErrorOr M = getLazyBitcodeModule(std::move(LightweightBuf), Context, - DiagnosticHandler); + DiagnosticHandler, + true/*ShouldLazyLoadMetadata*/); if (!M) return nullptr; return *M; @@ -229,7 +230,7 @@ LTOModule *LTOModule::makeLTOModule(MemoryBufferRef Buffer, TargetMachine *target = march->createTargetMachine(TripleStr, CPU, FeatureStr, options); - M->setDataLayout(target->getDataLayout()); + M->setDataLayout(*target->getDataLayout()); std::unique_ptr IRObj( new object::IRObjectFile(Buffer, std::move(M))); diff --git a/lib/Linker/LinkModules.cpp b/lib/Linker/LinkModules.cpp index e6d9acc..21edc50 100644 --- a/lib/Linker/LinkModules.cpp +++ b/lib/Linker/LinkModules.cpp @@ -226,6 +226,7 @@ void TypeMapTy::linkDefinedTypeBodies() { Elements[I] = get(SrcSTy->getElementType(I)); DstSTy->setBody(Elements, SrcSTy->isPacked()); + DstStructTypesSet.switchToNonOpaque(DstSTy); } SrcDefinitionsToResolve.clear(); DstResolvedOpaqueTypes.clear(); @@ -672,17 +673,12 @@ bool ModuleLinker::computeResultingSelectionKind(StringRef ComdatName, getComdatLeader(SrcM, ComdatName, SrcGV)) return true; - const DataLayout *DstDL = DstM->getDataLayout(); - const DataLayout *SrcDL = SrcM->getDataLayout(); - if (!DstDL || !SrcDL) { - return emitError( - "Linking COMDATs named '" + ComdatName + - "': can't do size dependent selection without DataLayout!"); - } + const DataLayout &DstDL = DstM->getDataLayout(); + const DataLayout &SrcDL = SrcM->getDataLayout(); uint64_t DstSize = - DstDL->getTypeAllocSize(DstGV->getType()->getPointerElementType()); + DstDL.getTypeAllocSize(DstGV->getType()->getPointerElementType()); uint64_t SrcSize = - SrcDL->getTypeAllocSize(SrcGV->getType()->getPointerElementType()); + SrcDL.getTypeAllocSize(SrcGV->getType()->getPointerElementType()); if (Result == Comdat::SelectionKind::ExactMatch) { if (SrcGV->getInitializer() != DstGV->getInitializer()) return emitError("Linking COMDATs named '" + ComdatName + @@ -768,9 +764,7 @@ bool ModuleLinker::shouldLinkFromSource(bool &LinkFromSrc, return false; } - // FIXME: Make datalayout mandatory and just use getDataLayout(). - DataLayout DL(Dest.getParent()); - + const DataLayout &DL = Dest.getParent()->getDataLayout(); uint64_t DestSize = DL.getTypeAllocSize(Dest.getType()->getElementType()); uint64_t SrcSize = DL.getTypeAllocSize(Src.getType()->getElementType()); LinkFromSrc = SrcSize > DestSize; @@ -1256,9 +1250,10 @@ void ModuleLinker::linkNamedMDNodes() { /// Drop DISubprograms that have been superseded. /// -/// FIXME: this creates an asymmetric result: we strip losing subprograms from -/// DstM, but leave losing subprograms in SrcM. Instead we should also strip -/// losers from SrcM, but this requires extra plumbing in MapMetadata. +/// FIXME: this creates an asymmetric result: we strip functions from losing +/// subprograms in DstM, but leave losing subprograms in SrcM. +/// TODO: Remove this logic once the backend can correctly determine canonical +/// subprograms. void ModuleLinker::stripReplacedSubprograms() { // Avoid quadratic runtime by returning early when there's nothing to do. if (OverridingFunctions.empty()) @@ -1268,8 +1263,8 @@ void ModuleLinker::stripReplacedSubprograms() { auto Functions = std::move(OverridingFunctions); OverridingFunctions.clear(); - // Drop subprograms whose functions have been overridden by the new compile - // unit. + // Drop functions from subprograms if they've been overridden by the new + // compile unit. NamedMDNode *CompileUnits = DstM->getNamedMetadata("llvm.dbg.cu"); if (!CompileUnits) return; @@ -1280,19 +1275,15 @@ void ModuleLinker::stripReplacedSubprograms() { DITypedArray SPs(CU.getSubprograms()); assert(SPs && "Expected valid subprogram array"); - SmallVector NewSPs; - NewSPs.reserve(SPs.getNumElements()); for (unsigned S = 0, SE = SPs.getNumElements(); S != SE; ++S) { DISubprogram SP = SPs.getElement(S); - if (SP && SP.getFunction() && Functions.count(SP.getFunction())) + if (!SP || !SP.getFunction() || !Functions.count(SP.getFunction())) continue; - NewSPs.push_back(SP); + // Prevent DebugInfoFinder from tagging this as the canonical subprogram, + // since the canonical one is in the incoming module. + SP->replaceFunction(nullptr); } - - // Redirect operand to the overriding subprogram. - if (NewSPs.size() != SPs.getNumElements()) - CU.replaceSubprograms(DIArray(MDNode::get(DstM->getContext(), NewSPs))); } } @@ -1482,11 +1473,10 @@ bool ModuleLinker::run() { // Inherit the target data from the source module if the destination module // doesn't have one already. - if (!DstM->getDataLayout() && SrcM->getDataLayout()) + if (DstM->getDataLayout().isDefault()) DstM->setDataLayout(SrcM->getDataLayout()); - if (SrcM->getDataLayout() && DstM->getDataLayout() && - *SrcM->getDataLayout() != *DstM->getDataLayout()) { + if (SrcM->getDataLayout() != DstM->getDataLayout()) { emitWarning("Linking two modules of different data layouts: '" + SrcM->getModuleIdentifier() + "' is '" + SrcM->getDataLayoutStr() + "' whereas '" + @@ -1570,6 +1560,13 @@ bool ModuleLinker::run() { MapValue(GV, ValueMap, RF_None, &TypeMap, &ValMaterializer); } + // Strip replaced subprograms before mapping any metadata -- so that we're + // not changing metadata from the source module (note that + // linkGlobalValueBody() eventually calls RemapInstruction() and therefore + // MapMetadata()) -- but after linking global value protocols -- so that + // OverridingFunctions has been built. + stripReplacedSubprograms(); + // Link in the function bodies that are defined in the source module into // DstM. for (Function &SF : *SrcM) { @@ -1592,9 +1589,6 @@ bool ModuleLinker::run() { linkGlobalValueBody(Src); } - // Strip replaced subprograms before linking together compile units. - stripReplacedSubprograms(); - // Remap all of the named MDNodes in Src into the DstM module. We do this // after linking GlobalValues so that MDNodes that reference GlobalValues // are properly remapped. @@ -1684,6 +1678,14 @@ void Linker::IdentifiedStructTypeSet::addNonOpaque(StructType *Ty) { NonOpaqueStructTypes.insert(Ty); } +void Linker::IdentifiedStructTypeSet::switchToNonOpaque(StructType *Ty) { + assert(!Ty->isOpaque()); + NonOpaqueStructTypes.insert(Ty); + bool Removed = OpaqueStructTypes.erase(Ty); + (void)Removed; + assert(Removed); +} + void Linker::IdentifiedStructTypeSet::addOpaque(StructType *Ty) { assert(Ty->isOpaque()); OpaqueStructTypes.insert(Ty); @@ -1777,7 +1779,7 @@ bool Linker::LinkModules(Module *Dest, Module *Src) { //===----------------------------------------------------------------------===// LLVMBool LLVMLinkModules(LLVMModuleRef Dest, LLVMModuleRef Src, - unsigned Unused, char **OutMessages) { + LLVMLinkerMode Unused, char **OutMessages) { Module *D = unwrap(Dest); std::string Message; raw_string_ostream Stream(Message); diff --git a/lib/MC/ELFObjectWriter.cpp b/lib/MC/ELFObjectWriter.cpp index 4819905..c99a3ee 100644 --- a/lib/MC/ELFObjectWriter.cpp +++ b/lib/MC/ELFObjectWriter.cpp @@ -185,11 +185,25 @@ class ELFObjectWriter : public MCObjectWriter { } public: - ELFObjectWriter(MCELFObjectTargetWriter *MOTW, raw_ostream &_OS, + ELFObjectWriter(MCELFObjectTargetWriter *MOTW, raw_ostream &OS, bool IsLittleEndian) - : MCObjectWriter(_OS, IsLittleEndian), FWriter(IsLittleEndian), + : MCObjectWriter(OS, IsLittleEndian), FWriter(IsLittleEndian), TargetObjectWriter(MOTW), NeedsGOT(false) {} + void reset() override { + UsedInReloc.clear(); + WeakrefUsedInReloc.clear(); + Renames.clear(); + Relocations.clear(); + ShStrTabBuilder.clear(); + StrTabBuilder.clear(); + FileSymbolData.clear(); + LocalSymbolData.clear(); + ExternalSymbolData.clear(); + UndefinedSymbolData.clear(); + MCObjectWriter::reset(); + } + virtual ~ELFObjectWriter(); void WriteWord(uint64_t W) { @@ -298,6 +312,8 @@ class ELFObjectWriter : public MCObjectWriter { bool InSet, bool IsPCRel) const override; + bool isWeak(const MCSymbolData &SD) const override; + void WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) override; void writeSection(MCAssembler &Asm, const SectionIndexMapTy &SectionIndexMap, @@ -789,6 +805,10 @@ static const MCSymbol *getWeakRef(const MCSymbolRefExpr &Ref) { return nullptr; } +static bool isWeak(const MCSymbolData &D) { + return D.getFlags() & ELF_STB_Weak || MCELF::GetType(D) == ELF::STT_GNU_IFUNC; +} + void ELFObjectWriter::RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, @@ -829,6 +849,10 @@ void ELFObjectWriter::RecordRelocation(MCAssembler &Asm, Fixup.getLoc(), "Cannot represent a difference across sections"); const MCSymbolData &SymBD = Asm.getSymbolData(SymB); + if (::isWeak(SymBD)) + Asm.getContext().FatalError( + Fixup.getLoc(), "Cannot represent a subtraction with a weak symbol"); + uint64_t SymBOffset = Layout.getSymbolOffset(&SymBD); uint64_t K = SymBOffset - FixupOffset; IsPCRel = true; @@ -1186,7 +1210,7 @@ getUncompressedData(MCAsmLayout &Layout, static bool prependCompressionHeader(uint64_t Size, SmallVectorImpl &CompressedContents) { - static const StringRef Magic = "ZLIB"; + const StringRef Magic = "ZLIB"; if (Size <= Magic.size() + sizeof(Size) + CompressedContents.size()) return false; if (sys::IsLittleEndianHost) @@ -1348,7 +1372,8 @@ static int cmpRel(const ELFRelocationEntry *AP, const ELFRelocationEntry *BP) { return B.Offset - A.Offset; if (B.Type != A.Type) return A.Type - B.Type; - llvm_unreachable("ELFRelocs might be unstable!"); + //llvm_unreachable("ELFRelocs might be unstable!"); + return 0; } static void sortRelocs(const MCAssembler &Asm, @@ -1794,12 +1819,16 @@ ELFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, const MCFragment &FB, bool InSet, bool IsPCRel) const { - if (DataA.getFlags() & ELF_STB_Weak || MCELF::GetType(DataA) == ELF::STT_GNU_IFUNC) + if (::isWeak(DataA)) return false; return MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( Asm, DataA, FB,InSet, IsPCRel); } +bool ELFObjectWriter::isWeak(const MCSymbolData &SD) const { + return ::isWeak(SD); +} + MCObjectWriter *llvm::createELFObjectWriter(MCELFObjectTargetWriter *MOTW, raw_ostream &OS, bool IsLittleEndian) { diff --git a/lib/MC/MCAsmInfo.cpp b/lib/MC/MCAsmInfo.cpp index 04b8042..bad257a 100644 --- a/lib/MC/MCAsmInfo.cpp +++ b/lib/MC/MCAsmInfo.cpp @@ -39,6 +39,7 @@ MCAsmInfo::MCAsmInfo() { CommentString = "#"; LabelSuffix = ":"; UseAssignmentForEHBegin = false; + NeedsLocalForSize = false; PrivateGlobalPrefix = "L"; PrivateLabelPrefix = PrivateGlobalPrefix; LinkerPrivateGlobalPrefix = ""; @@ -68,6 +69,7 @@ MCAsmInfo::MCAsmInfo() { HasAggressiveSymbolFolding = true; COMMDirectiveAlignmentIsInBytes = true; LCOMMDirectiveAlignmentType = LCOMM::NoAlignment; + HasFunctionAlignment = true; HasDotTypeDotSizeDirective = true; HasSingleParameterDotFile = true; HasIdentDirective = false; diff --git a/lib/MC/MCAsmInfoDarwin.cpp b/lib/MC/MCAsmInfoDarwin.cpp index a2a2504..ae9486d 100644 --- a/lib/MC/MCAsmInfoDarwin.cpp +++ b/lib/MC/MCAsmInfoDarwin.cpp @@ -16,7 +16,6 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSectionMachO.h" -#include "llvm/MC/MCStreamer.h" using namespace llvm; bool MCAsmInfoDarwin::isSectionAtomizableBySymbols( diff --git a/lib/MC/MCAsmStreamer.cpp b/lib/MC/MCAsmStreamer.cpp index 2312cd5..62f5279 100644 --- a/lib/MC/MCAsmStreamer.cpp +++ b/lib/MC/MCAsmStreamer.cpp @@ -267,7 +267,7 @@ void MCAsmStreamer::EmitCommentsAndEOL() { } CommentStream.flush(); - StringRef Comments = CommentToEmit.str(); + StringRef Comments = CommentToEmit; assert(Comments.back() == '\n' && "Comment array not newline terminated"); diff --git a/lib/MC/MCAssembler.cpp b/lib/MC/MCAssembler.cpp index 50ce845..857eafc 100644 --- a/lib/MC/MCAssembler.cpp +++ b/lib/MC/MCAssembler.cpp @@ -142,7 +142,7 @@ static bool getSymbolOffsetImpl(const MCAsmLayout &Layout, // If SD is a variable, evaluate it. MCValue Target; - if (!S.getVariableValue()->EvaluateAsValue(Target, &Layout, nullptr)) + if (!S.getVariableValue()->EvaluateAsRelocatable(Target, &Layout, nullptr)) report_fatal_error("unable to evaluate offset for variable '" + S.getName() + "'"); @@ -188,7 +188,7 @@ const MCSymbol *MCAsmLayout::getBaseSymbol(const MCSymbol &Symbol) const { const MCExpr *Expr = Symbol.getVariableValue(); MCValue Value; - if (!Expr->EvaluateAsValue(Value, this, nullptr)) + if (!Expr->evaluateAsValue(Value, *this)) llvm_unreachable("Invalid Expression"); const MCSymbolRefExpr *RefB = Value.getSymB(); @@ -277,9 +277,8 @@ MCFragment::MCFragment() : Kind(FragmentType(~0)) { MCFragment::~MCFragment() { } -MCFragment::MCFragment(FragmentType _Kind, MCSectionData *_Parent) - : Kind(_Kind), Parent(_Parent), Atom(nullptr), Offset(~UINT64_C(0)) -{ +MCFragment::MCFragment(FragmentType Kind, MCSectionData *Parent) + : Kind(Kind), Parent(Parent), Atom(nullptr), Offset(~UINT64_C(0)) { if (Parent) Parent->getFragmentList().push_back(this); } @@ -298,15 +297,10 @@ MCEncodedFragmentWithFixups::~MCEncodedFragmentWithFixups() { MCSectionData::MCSectionData() : Section(nullptr) {} -MCSectionData::MCSectionData(const MCSection &_Section, MCAssembler *A) - : Section(&_Section), - Ordinal(~UINT32_C(0)), - Alignment(1), - BundleLockState(NotBundleLocked), - BundleLockNestingDepth(0), - BundleGroupBeforeFirstInst(false), - HasInstructions(false) -{ +MCSectionData::MCSectionData(const MCSection &Section, MCAssembler *A) + : Section(&Section), Ordinal(~UINT32_C(0)), Alignment(1), + BundleLockState(NotBundleLocked), BundleLockNestingDepth(0), + BundleGroupBeforeFirstInst(false), HasInstructions(false) { if (A) A->getSectionList().push_back(this); } @@ -364,10 +358,10 @@ void MCSectionData::setBundleLockState(BundleLockStateType NewState) { MCSymbolData::MCSymbolData() : Symbol(nullptr) {} -MCSymbolData::MCSymbolData(const MCSymbol &_Symbol, MCFragment *_Fragment, - uint64_t _Offset, MCAssembler *A) - : Symbol(&_Symbol), Fragment(_Fragment), Offset(_Offset), - SymbolSize(nullptr), CommonAlign(-1U), Flags(0), Index(0) { +MCSymbolData::MCSymbolData(const MCSymbol &Symbol, MCFragment *Fragment, + uint64_t Offset, MCAssembler *A) + : Symbol(&Symbol), Fragment(Fragment), Offset(Offset), SymbolSize(nullptr), + CommonAlign(-1U), Flags(0), Index(0) { if (A) A->getSymbolList().push_back(this); } @@ -479,18 +473,6 @@ const MCSymbolData *MCAssembler::getAtom(const MCSymbolData *SD) const { return SD->getFragment()->getAtom(); } -// Try to fully compute Expr to an absolute value and if that fails produce -// a relocatable expr. -// FIXME: Should this be the behavior of EvaluateAsRelocatable itself? -static bool evaluate(const MCExpr &Expr, const MCAsmLayout &Layout, - const MCFixup &Fixup, MCValue &Target) { - if (Expr.EvaluateAsValue(Target, &Layout, &Fixup)) { - if (Target.isAbsolute()) - return true; - } - return Expr.EvaluateAsRelocatable(Target, &Layout, &Fixup); -} - bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, MCValue &Target, uint64_t &Value) const { @@ -500,7 +482,7 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, // probably merge the two into a single callback that tries to evaluate a // fixup and records a relocation if one is needed. const MCExpr *Expr = Fixup.getValue(); - if (!evaluate(*Expr, Layout, Fixup, Target)) + if (!Expr->EvaluateAsRelocatable(Target, &Layout, &Fixup)) getContext().FatalError(Fixup.getLoc(), "expected relocatable expression"); bool IsPCRel = Backend.getFixupKindInfo( @@ -795,7 +777,7 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout, case MCFragment::FT_LEB: { const MCLEBFragment &LF = cast(F); - OW->WriteBytes(LF.getContents().str()); + OW->WriteBytes(LF.getContents()); break; } @@ -811,12 +793,12 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout, case MCFragment::FT_Dwarf: { const MCDwarfLineAddrFragment &OF = cast(F); - OW->WriteBytes(OF.getContents().str()); + OW->WriteBytes(OF.getContents()); break; } case MCFragment::FT_DwarfFrame: { const MCDwarfCallFrameFragment &CF = cast(F); - OW->WriteBytes(CF.getContents().str()); + OW->WriteBytes(CF.getContents()); break; } } @@ -1040,7 +1022,10 @@ bool MCAssembler::relaxInstruction(MCAsmLayout &Layout, bool MCAssembler::relaxLEB(MCAsmLayout &Layout, MCLEBFragment &LF) { uint64_t OldSize = LF.getContents().size(); - int64_t Value = LF.getValue().evaluateKnownAbsolute(Layout); + int64_t Value; + bool Abs = LF.getValue().evaluateKnownAbsolute(Value, Layout); + if (!Abs) + report_fatal_error("sleb128 and uleb128 expressions must be absolute"); SmallString<8> &Data = LF.getContents(); Data.clear(); raw_svector_ostream OSE(Data); @@ -1056,7 +1041,10 @@ bool MCAssembler::relaxDwarfLineAddr(MCAsmLayout &Layout, MCDwarfLineAddrFragment &DF) { MCContext &Context = Layout.getAssembler().getContext(); uint64_t OldSize = DF.getContents().size(); - int64_t AddrDelta = DF.getAddrDelta().evaluateKnownAbsolute(Layout); + int64_t AddrDelta; + bool Abs = DF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, Layout); + assert(Abs && "We created a line delta with an invalid expression"); + (void) Abs; int64_t LineDelta; LineDelta = DF.getLineDelta(); SmallString<8> &Data = DF.getContents(); @@ -1071,7 +1059,10 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCAsmLayout &Layout, MCDwarfCallFrameFragment &DF) { MCContext &Context = Layout.getAssembler().getContext(); uint64_t OldSize = DF.getContents().size(); - int64_t AddrDelta = DF.getAddrDelta().evaluateKnownAbsolute(Layout); + int64_t AddrDelta; + bool Abs = DF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, Layout); + assert(Abs && "We created call frame with an invalid expression"); + (void) Abs; SmallString<8> &Data = DF.getContents(); Data.clear(); raw_svector_ostream OSE(Data); diff --git a/lib/MC/MCContext.cpp b/lib/MC/MCContext.cpp index 721edd4..3cb3ea1 100644 --- a/lib/MC/MCContext.cpp +++ b/lib/MC/MCContext.cpp @@ -18,6 +18,7 @@ #include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionMachO.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/ELF.h" #include "llvm/Support/ErrorHandling.h" @@ -33,7 +34,7 @@ MCContext::MCContext(const MCAsmInfo *mai, const MCRegisterInfo *mri, const MCObjectFileInfo *mofi, const SourceMgr *mgr, bool DoAutoReset) : SrcMgr(mgr), MAI(mai), MRI(mri), MOFI(mofi), Allocator(), - Symbols(Allocator), UsedNames(Allocator), NextUniqueID(0), + Symbols(Allocator), UsedNames(Allocator), CurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_IS_STMT, 0, 0), DwarfLocSeen(false), GenDwarfForAssembly(false), GenDwarfFileNumber(0), DwarfVersion(4), AllowTemporaryLabels(true), DwarfCompileUnitID(0), @@ -86,7 +87,7 @@ void MCContext::reset() { ELFUniquingMap.clear(); COFFUniquingMap.clear(); - NextUniqueID = 0; + NextID.clear(); AllowTemporaryLabels = true; DwarfLocSeen = false; GenDwarfForAssembly = false; @@ -97,13 +98,15 @@ void MCContext::reset() { // Symbol Manipulation //===----------------------------------------------------------------------===// -MCSymbol *MCContext::GetOrCreateSymbol(StringRef Name) { - assert(!Name.empty() && "Normal symbols cannot be unnamed!"); +MCSymbol *MCContext::GetOrCreateSymbol(const Twine &Name) { + SmallString<128> NameSV; + StringRef NameRef = Name.toStringRef(NameSV); - MCSymbol *&Sym = Symbols[Name]; + assert(!NameRef.empty() && "Normal symbols cannot be unnamed!"); + MCSymbol *&Sym = Symbols[NameRef]; if (!Sym) - Sym = CreateSymbol(Name); + Sym = CreateSymbol(NameRef, false); return Sym; } @@ -130,53 +133,54 @@ MCSymbol *MCContext::getOrCreateSectionSymbol(const MCSectionELF &Section) { return Sym; } -MCSymbol *MCContext::getOrCreateFrameAllocSymbol(StringRef FuncName) { - return GetOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + - "frameallocation_" + FuncName); +MCSymbol *MCContext::getOrCreateFrameAllocSymbol(StringRef FuncName, + unsigned Idx) { + return GetOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + FuncName + + "$frame_escape_" + Twine(Idx)); } -MCSymbol *MCContext::CreateSymbol(StringRef Name) { +MCSymbol *MCContext::CreateSymbol(StringRef Name, bool AlwaysAddSuffix) { // Determine whether this is an assembler temporary or normal label, if used. - bool isTemporary = false; + bool IsTemporary = false; if (AllowTemporaryLabels) - isTemporary = Name.startswith(MAI->getPrivateGlobalPrefix()); + IsTemporary = Name.startswith(MAI->getPrivateGlobalPrefix()); - auto NameEntry = UsedNames.insert(std::make_pair(Name, true)); - if (!NameEntry.second) { - assert(isTemporary && "Cannot rename non-temporary symbols"); - SmallString<128> NewName = Name; - do { + SmallString<128> NewName = Name; + bool AddSuffix = AlwaysAddSuffix; + unsigned &NextUniqueID = NextID[Name]; + for (;;) { + if (AddSuffix) { NewName.resize(Name.size()); raw_svector_ostream(NewName) << NextUniqueID++; - NameEntry = UsedNames.insert(std::make_pair(NewName, true)); - } while (!NameEntry.second); + } + auto NameEntry = UsedNames.insert(std::make_pair(NewName, true)); + if (NameEntry.second) { + // Ok, we found a name. Have the MCSymbol object itself refer to the copy + // of the string that is embedded in the UsedNames entry. + MCSymbol *Result = + new (*this) MCSymbol(NameEntry.first->getKey(), IsTemporary); + return Result; + } + assert(IsTemporary && "Cannot rename non-temporary symbols"); + AddSuffix = true; } - - // Ok, the entry doesn't already exist. Have the MCSymbol object itself refer - // to the copy of the string that is embedded in the UsedNames entry. - MCSymbol *Result = - new (*this) MCSymbol(NameEntry.first->getKey(), isTemporary); - - return Result; + llvm_unreachable("Infinite loop"); } -MCSymbol *MCContext::GetOrCreateSymbol(const Twine &Name) { +MCSymbol *MCContext::createTempSymbol(const Twine &Name, bool AlwaysAddSuffix) { SmallString<128> NameSV; - return GetOrCreateSymbol(Name.toStringRef(NameSV)); + raw_svector_ostream(NameSV) << MAI->getPrivateGlobalPrefix() << Name; + return CreateSymbol(NameSV, AlwaysAddSuffix); } MCSymbol *MCContext::CreateLinkerPrivateTempSymbol() { SmallString<128> NameSV; - raw_svector_ostream(NameSV) - << MAI->getLinkerPrivateGlobalPrefix() << "tmp" << NextUniqueID++; - return CreateSymbol(NameSV); + raw_svector_ostream(NameSV) << MAI->getLinkerPrivateGlobalPrefix() << "tmp"; + return CreateSymbol(NameSV, true); } MCSymbol *MCContext::CreateTempSymbol() { - SmallString<128> NameSV; - raw_svector_ostream(NameSV) - << MAI->getPrivateGlobalPrefix() << "tmp" << NextUniqueID++; - return CreateSymbol(NameSV); + return createTempSymbol("tmp", true); } unsigned MCContext::NextInstance(unsigned LocalLabelVal) { @@ -214,24 +218,20 @@ MCSymbol *MCContext::GetDirectionalLocalSymbol(unsigned LocalLabelVal, return getOrCreateDirectionalLocalSymbol(LocalLabelVal, Instance); } -MCSymbol *MCContext::LookupSymbol(StringRef Name) const { - return Symbols.lookup(Name); -} - MCSymbol *MCContext::LookupSymbol(const Twine &Name) const { SmallString<128> NameSV; - Name.toVector(NameSV); - return LookupSymbol(NameSV.str()); + StringRef NameRef = Name.toStringRef(NameSV); + return Symbols.lookup(NameRef); } //===----------------------------------------------------------------------===// // Section Management //===----------------------------------------------------------------------===// -const MCSectionMachO *MCContext:: -getMachOSection(StringRef Segment, StringRef Section, - unsigned TypeAndAttributes, - unsigned Reserved2, SectionKind Kind) { +const MCSectionMachO * +MCContext::getMachOSection(StringRef Segment, StringRef Section, + unsigned TypeAndAttributes, unsigned Reserved2, + SectionKind Kind, const char *BeginSymName) { // We unique sections by their segment/section pair. The returned section // may not have the same flags as the requested section, if so this should be @@ -244,17 +244,23 @@ getMachOSection(StringRef Segment, StringRef Section, Name += Section; // Do the lookup, if we have a hit, return it. - const MCSectionMachO *&Entry = MachOUniquingMap[Name.str()]; - if (Entry) return Entry; + const MCSectionMachO *&Entry = MachOUniquingMap[Name]; + if (Entry) + return Entry; + + MCSymbol *Begin = nullptr; + if (BeginSymName) + Begin = createTempSymbol(BeginSymName, false); // Otherwise, return a new section. return Entry = new (*this) MCSectionMachO(Segment, Section, TypeAndAttributes, - Reserved2, Kind); + Reserved2, Kind, Begin); } const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, - unsigned Flags) { - return getELFSection(Section, Type, Flags, 0, ""); + unsigned Flags, + const char *BeginSymName) { + return getELFSection(Section, Type, Flags, 0, "", BeginSymName); } void MCContext::renameELFSection(const MCSectionELF *Section, StringRef Name) { @@ -272,7 +278,8 @@ void MCContext::renameELFSection(const MCSectionELF *Section, StringRef Name) { const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, unsigned Flags, unsigned EntrySize, - StringRef Group, bool Unique) { + StringRef Group, bool Unique, + const char *BeginSymName) { // Do the lookup, if we have a hit, return it. auto IterBool = ELFUniquingMap.insert( std::make_pair(SectionGroupPair(Section, Group), nullptr)); @@ -292,8 +299,12 @@ const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, else Kind = SectionKind::getReadOnly(); - MCSectionELF *Result = new (*this) - MCSectionELF(CachedName, Type, Flags, Kind, EntrySize, GroupSym, Unique); + MCSymbol *Begin = nullptr; + if (BeginSymName) + Begin = createTempSymbol(BeginSymName, false); + + MCSectionELF *Result = new (*this) MCSectionELF( + CachedName, Type, Flags, Kind, EntrySize, GroupSym, Unique, Begin); if (!Unique) Entry.second = Result; return Result; @@ -301,22 +312,23 @@ const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, unsigned Flags, unsigned EntrySize, - StringRef Group) { - return getELFSection(Section, Type, Flags, EntrySize, Group, false); + StringRef Group, + const char *BeginSymName) { + return getELFSection(Section, Type, Flags, EntrySize, Group, false, + BeginSymName); } const MCSectionELF *MCContext::CreateELFGroupSection() { - MCSectionELF *Result = - new (*this) MCSectionELF(".group", ELF::SHT_GROUP, 0, - SectionKind::getReadOnly(), 4, nullptr, false); + MCSectionELF *Result = new (*this) + MCSectionELF(".group", ELF::SHT_GROUP, 0, SectionKind::getReadOnly(), 4, + nullptr, false, nullptr); return Result; } -const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section, - unsigned Characteristics, - SectionKind Kind, - StringRef COMDATSymName, - int Selection) { +const MCSectionCOFF * +MCContext::getCOFFSection(StringRef Section, unsigned Characteristics, + SectionKind Kind, StringRef COMDATSymName, + int Selection, const char *BeginSymName) { // Do the lookup, if we have a hit, return it. SectionGroupTriple T(Section, COMDATSymName, Selection); @@ -329,18 +341,23 @@ const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section, if (!COMDATSymName.empty()) COMDATSymbol = GetOrCreateSymbol(COMDATSymName); + MCSymbol *Begin = nullptr; + if (BeginSymName) + Begin = createTempSymbol(BeginSymName, false); + StringRef CachedName = std::get<0>(Iter->first); - MCSectionCOFF *Result = new (*this) - MCSectionCOFF(CachedName, Characteristics, COMDATSymbol, Selection, Kind); + MCSectionCOFF *Result = new (*this) MCSectionCOFF( + CachedName, Characteristics, COMDATSymbol, Selection, Kind, Begin); Iter->second = Result; return Result; } -const MCSectionCOFF * -MCContext::getCOFFSection(StringRef Section, unsigned Characteristics, - SectionKind Kind) { - return getCOFFSection(Section, Characteristics, Kind, "", 0); +const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section, + unsigned Characteristics, + SectionKind Kind, + const char *BeginSymName) { + return getCOFFSection(Section, Characteristics, Kind, "", 0, BeginSymName); } const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section) { diff --git a/lib/MC/MCDwarf.cpp b/lib/MC/MCDwarf.cpp index 5d96914..87e7ed1 100644 --- a/lib/MC/MCDwarf.cpp +++ b/lib/MC/MCDwarf.cpp @@ -179,28 +179,19 @@ EmitDwarfLineTable(MCObjectStreamer *MCOS, const MCSection *Section, } // Emit a DW_LNE_end_sequence for the end of the section. - // Using the pointer Section create a temporary label at the end of the - // section and use that and the LastLabel to compute the address delta - // and use INT64_MAX as the line delta which is the signal that this is - // actually a DW_LNE_end_sequence. + // Use the section end label to compute the address delta and use INT64_MAX + // as the line delta which is the signal that this is actually a + // DW_LNE_end_sequence. + MCSymbol *SectionEnd = MCOS->endSection(Section); - // Switch to the section to be able to create a symbol at its end. - // TODO: keep track of the last subsection so that this symbol appears in the - // correct place. - MCOS->SwitchSection(Section); + // Switch back the dwarf line section, in case endSection had to switch the + // section. + MCContext &Ctx = MCOS->getContext(); + MCOS->SwitchSection(Ctx.getObjectFileInfo()->getDwarfLineSection()); - MCContext &context = MCOS->getContext(); - // Create a symbol at the end of the section. - MCSymbol *SectionEnd = context.CreateTempSymbol(); - // Set the value of the symbol, as we are at the end of the section. - MCOS->EmitLabel(SectionEnd); - - // Switch back the dwarf line section. - MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfLineSection()); - - const MCAsmInfo *asmInfo = MCOS->getContext().getAsmInfo(); + const MCAsmInfo *AsmInfo = Ctx.getAsmInfo(); MCOS->EmitDwarfAdvanceLineAddr(INT64_MAX, LastLabel, SectionEnd, - asmInfo->getPointerSize()); + AsmInfo->getPointerSize()); } // @@ -243,7 +234,8 @@ std::pair MCDwarfLineTableHeader::Emit(MCStreamer *MCOS) 0, // length of DW_LNS_set_epilogue_begin 1 // DW_LNS_set_isa }; - assert(array_lengthof(StandardOpcodeLengths) == (DWARF2_LINE_OPCODE_BASE - 1)); + assert(array_lengthof(StandardOpcodeLengths) == + (DWARF2_LINE_OPCODE_BASE - 1)); return Emit(MCOS, StandardOpcodeLengths); } @@ -446,7 +438,7 @@ void MCDwarfLineAddr::Encode(MCContext &Context, int64_t LineDelta, if (LineDelta == INT64_MAX) { if (AddrDelta == MAX_SPECIAL_ADDR_DELTA) OS << char(dwarf::DW_LNS_const_add_pc); - else { + else if (AddrDelta) { OS << char(dwarf::DW_LNS_advance_pc); encodeULEB128(AddrDelta, OS); } @@ -1007,11 +999,13 @@ static void EmitPersonality(MCStreamer &streamer, const MCSymbol &symbol, namespace { class FrameEmitterImpl { int CFAOffset; + int InitialCFAOffset; bool IsEH; const MCSymbol *SectionStart; public: FrameEmitterImpl(bool isEH) - : CFAOffset(0), IsEH(isEH), SectionStart(nullptr) {} + : CFAOffset(0), InitialCFAOffset(0), IsEH(isEH), SectionStart(nullptr) { + } void setSectionStart(const MCSymbol *Label) { SectionStart = Label; } @@ -1292,7 +1286,7 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCObjectStreamer &streamer, Augmentation += "R"; if (IsSignalFrame) Augmentation += "S"; - streamer.EmitBytes(Augmentation.str()); + streamer.EmitBytes(Augmentation); } streamer.EmitIntValue(0, 1); @@ -1353,6 +1347,8 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCObjectStreamer &streamer, EmitCFIInstructions(streamer, Instructions, nullptr); } + InitialCFAOffset = CFAOffset; + // Padding streamer.EmitValueToAlignment(IsEH ? 4 : MAI->getPointerSize()); @@ -1368,6 +1364,8 @@ MCSymbol *FrameEmitterImpl::EmitFDE(MCObjectStreamer &streamer, MCSymbol *fdeEnd = context.CreateTempSymbol(); const MCObjectFileInfo *MOFI = context.getObjectFileInfo(); + CFAOffset = InitialCFAOffset; + // Length const MCExpr *Length = MakeStartMinusEndExpr(streamer, *fdeStart, *fdeEnd, 0); emitAbsValue(streamer, Length, 4); diff --git a/lib/MC/MCELFStreamer.cpp b/lib/MC/MCELFStreamer.cpp index 199825e..cdf5033 100644 --- a/lib/MC/MCELFStreamer.cpp +++ b/lib/MC/MCELFStreamer.cpp @@ -32,6 +32,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ELF.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -122,12 +123,11 @@ void MCELFStreamer::EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) { // If neither T1 < T2 nor T2 < T1 according to this ordering, use T2 (the user // provided type). static unsigned CombineSymbolTypes(unsigned T1, unsigned T2) { - unsigned TypeOrdering[] = {ELF::STT_NOTYPE, ELF::STT_OBJECT, ELF::STT_FUNC, - ELF::STT_GNU_IFUNC, ELF::STT_TLS}; - for (unsigned i = 0; i != array_lengthof(TypeOrdering); ++i) { - if (T1 == TypeOrdering[i]) + for (unsigned Type : {ELF::STT_NOTYPE, ELF::STT_OBJECT, ELF::STT_FUNC, + ELF::STT_GNU_IFUNC, ELF::STT_TLS}) { + if (T1 == Type) return T2; - if (T2 == TypeOrdering[i]) + if (T2 == Type) return T1; } diff --git a/lib/MC/MCExpr.cpp b/lib/MC/MCExpr.cpp index 709dc6b..8a64403 100644 --- a/lib/MC/MCExpr.cpp +++ b/lib/MC/MCExpr.cpp @@ -192,6 +192,7 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) { case VK_GOTPAGE: return "GOTPAGE"; case VK_GOTPAGEOFF: return "GOTPAGEOFF"; case VK_SECREL: return "SECREL32"; + case VK_SIZE: return "SIZE"; case VK_WEAKREF: return "WEAKREF"; case VK_ARM_NONE: return "none"; case VK_ARM_TARGET1: return "target1"; @@ -311,6 +312,7 @@ MCSymbolRefExpr::getVariantKindForName(StringRef Name) { .Case("gotpageoff", VK_GOTPAGEOFF) .Case("imgrel", VK_COFF_IMGREL32) .Case("secrel32", VK_SECREL) + .Case("size", VK_SIZE) .Case("l", VK_PPC_LO) .Case("h", VK_PPC_HI) .Case("ha", VK_PPC_HA) @@ -404,13 +406,10 @@ bool MCExpr::EvaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const { return EvaluateAsAbsolute(Res, &Asm, nullptr, nullptr); } -int64_t MCExpr::evaluateKnownAbsolute(const MCAsmLayout &Layout) const { - int64_t Res; - bool Abs = - evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, nullptr, true); - (void)Abs; - assert(Abs && "Not actually absolute"); - return Res; +bool MCExpr::evaluateKnownAbsolute(int64_t &Res, + const MCAsmLayout &Layout) const { + return evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, nullptr, + true); } bool MCExpr::EvaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, @@ -433,8 +432,8 @@ bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, return true; } - bool IsRelocatable = EvaluateAsRelocatableImpl( - Value, Asm, Layout, nullptr, Addrs, InSet, /*ForceVarExpansion*/ false); + bool IsRelocatable = + EvaluateAsRelocatableImpl(Value, Asm, Layout, nullptr, Addrs, InSet); // Record the current value. Res = Value.getConstant(); @@ -443,13 +442,10 @@ bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, } /// \brief Helper method for \see EvaluateSymbolAdd(). -static void AttemptToFoldSymbolOffsetDifference(const MCAssembler *Asm, - const MCAsmLayout *Layout, - const SectionAddrMap *Addrs, - bool InSet, - const MCSymbolRefExpr *&A, - const MCSymbolRefExpr *&B, - int64_t &Addend) { +static void AttemptToFoldSymbolOffsetDifference( + const MCAssembler *Asm, const MCAsmLayout *Layout, + const SectionAddrMap *Addrs, bool InSet, const MCSymbolRefExpr *&A, + const MCSymbolRefExpr *&B, int64_t &Addend) { if (!A || !B) return; @@ -523,13 +519,11 @@ static void AttemptToFoldSymbolOffsetDifference(const MCAssembler *Asm, /// They might look redundant, but this function can be used before layout /// is done (see the object streamer for example) and having the Asm argument /// lets us avoid relaxations early. -static bool EvaluateSymbolicAdd(const MCAssembler *Asm, - const MCAsmLayout *Layout, - const SectionAddrMap *Addrs, - bool InSet, - const MCValue &LHS,const MCSymbolRefExpr *RHS_A, - const MCSymbolRefExpr *RHS_B, int64_t RHS_Cst, - MCValue &Res) { +static bool +EvaluateSymbolicAdd(const MCAssembler *Asm, const MCAsmLayout *Layout, + const SectionAddrMap *Addrs, bool InSet, const MCValue &LHS, + const MCSymbolRefExpr *RHS_A, const MCSymbolRefExpr *RHS_B, + int64_t RHS_Cst, MCValue &Res) { // FIXME: This routine (and other evaluation parts) are *incredibly* sloppy // about dealing with modifiers. This will ultimately bite us, one day. const MCSymbolRefExpr *LHS_A = LHS.getSymA(); @@ -587,21 +581,29 @@ bool MCExpr::EvaluateAsRelocatable(MCValue &Res, const MCFixup *Fixup) const { MCAssembler *Assembler = Layout ? &Layout->getAssembler() : nullptr; return EvaluateAsRelocatableImpl(Res, Assembler, Layout, Fixup, nullptr, - false, /*ForceVarExpansion*/ false); + false); } -bool MCExpr::EvaluateAsValue(MCValue &Res, const MCAsmLayout *Layout, - const MCFixup *Fixup) const { - MCAssembler *Assembler = Layout ? &Layout->getAssembler() : nullptr; - return EvaluateAsRelocatableImpl(Res, Assembler, Layout, Fixup, nullptr, - false, /*ForceVarExpansion*/ true); +bool MCExpr::evaluateAsValue(MCValue &Res, const MCAsmLayout &Layout) const { + MCAssembler *Assembler = &Layout.getAssembler(); + return EvaluateAsRelocatableImpl(Res, Assembler, &Layout, nullptr, nullptr, + true); +} + +static bool canExpand(const MCSymbol &Sym, const MCAssembler *Asm, bool InSet) { + if (InSet) + return true; + if (!Asm) + return false; + const MCSymbolData &SD = Asm->getSymbolData(Sym); + return !Asm->getWriter().isWeak(SD); } bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, const MCFixup *Fixup, - const SectionAddrMap *Addrs, bool InSet, - bool ForceVarExpansion) const { + const SectionAddrMap *Addrs, + bool InSet) const { ++stats::MCExprEvaluate; switch (getKind()) { @@ -618,28 +620,24 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCSymbol &Sym = SRE->getSymbol(); // Evaluate recursively if this is a variable. - if (Sym.isVariable() && SRE->getKind() == MCSymbolRefExpr::VK_None) { + if (Sym.isVariable() && SRE->getKind() == MCSymbolRefExpr::VK_None && + canExpand(Sym, Asm, InSet)) { + bool IsMachO = SRE->hasSubsectionsViaSymbols(); if (Sym.getVariableValue()->EvaluateAsRelocatableImpl( - Res, Asm, Layout, Fixup, Addrs, true, ForceVarExpansion)) { + Res, Asm, Layout, Fixup, Addrs, InSet || IsMachO)) { + if (!IsMachO) + return true; + const MCSymbolRefExpr *A = Res.getSymA(); const MCSymbolRefExpr *B = Res.getSymB(); - - if (SRE->hasSubsectionsViaSymbols()) { - // FIXME: This is small hack. Given - // a = b + 4 - // .long a - // the OS X assembler will completely drop the 4. We should probably - // include it in the relocation or produce an error if that is not - // possible. - if (!A && !B) - return true; - } else { - if (ForceVarExpansion) - return true; - bool IsSymbol = A && A->getSymbol().isDefined(); - if (!IsSymbol) - return true; - } + // FIXME: This is small hack. Given + // a = b + 4 + // .long a + // the OS X assembler will completely drop the 4. We should probably + // include it in the relocation or produce an error if that is not + // possible. + if (!A && !B) + return true; } } @@ -651,9 +649,8 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCUnaryExpr *AUE = cast(this); MCValue Value; - if (!AUE->getSubExpr()->EvaluateAsRelocatableImpl(Value, Asm, Layout, - Fixup, Addrs, InSet, - ForceVarExpansion)) + if (!AUE->getSubExpr()->EvaluateAsRelocatableImpl(Value, Asm, Layout, Fixup, + Addrs, InSet)) return false; switch (AUE->getOpcode()) { @@ -686,12 +683,10 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCBinaryExpr *ABE = cast(this); MCValue LHSValue, RHSValue; - if (!ABE->getLHS()->EvaluateAsRelocatableImpl(LHSValue, Asm, Layout, - Fixup, Addrs, InSet, - ForceVarExpansion) || - !ABE->getRHS()->EvaluateAsRelocatableImpl(RHSValue, Asm, Layout, - Fixup, Addrs, InSet, - ForceVarExpansion)) + if (!ABE->getLHS()->EvaluateAsRelocatableImpl(LHSValue, Asm, Layout, Fixup, + Addrs, InSet) || + !ABE->getRHS()->EvaluateAsRelocatableImpl(RHSValue, Asm, Layout, Fixup, + Addrs, InSet)) return false; // We only support a few operations on non-constant expressions, handle @@ -704,14 +699,12 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, // Negate RHS and add. return EvaluateSymbolicAdd(Asm, Layout, Addrs, InSet, LHSValue, RHSValue.getSymB(), RHSValue.getSymA(), - -RHSValue.getConstant(), - Res); + -RHSValue.getConstant(), Res); case MCBinaryExpr::Add: return EvaluateSymbolicAdd(Asm, Layout, Addrs, InSet, LHSValue, RHSValue.getSymA(), RHSValue.getSymB(), - RHSValue.getConstant(), - Res); + RHSValue.getConstant(), Res); } } diff --git a/lib/MC/MCLinkerOptimizationHint.cpp b/lib/MC/MCLinkerOptimizationHint.cpp index 7739878..8db22dc 100644 --- a/lib/MC/MCLinkerOptimizationHint.cpp +++ b/lib/MC/MCLinkerOptimizationHint.cpp @@ -9,7 +9,7 @@ #include "llvm/MC/MCLinkerOptimizationHint.h" #include "llvm/MC/MCAsmLayout.h" -#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCAssembler.h" #include "llvm/Support/LEB128.h" using namespace llvm; diff --git a/lib/MC/MCMachOStreamer.cpp b/lib/MC/MCMachOStreamer.cpp index 79eab49..d5c7101 100644 --- a/lib/MC/MCMachOStreamer.cpp +++ b/lib/MC/MCMachOStreamer.cpp @@ -26,6 +26,7 @@ #include "llvm/MC/MCSymbol.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -39,6 +40,9 @@ private: /// need for local relocations. False by default. bool LabelSections; + bool DWARFMustBeAtTheEnd; + bool CreatedADWARFSection; + /// HasSectionLabel - map of which sections have already had a non-local /// label emitted to them. Used so we don't emit extraneous linker local /// labels in the middle of the section. @@ -51,9 +55,9 @@ private: public: MCMachOStreamer(MCContext &Context, MCAsmBackend &MAB, raw_ostream &OS, - MCCodeEmitter *Emitter, bool label) - : MCObjectStreamer(Context, MAB, OS, Emitter), - LabelSections(label) {} + MCCodeEmitter *Emitter, bool DWARFMustBeAtTheEnd, bool label) + : MCObjectStreamer(Context, MAB, OS, Emitter), LabelSections(label), + DWARFMustBeAtTheEnd(DWARFMustBeAtTheEnd), CreatedADWARFSection(false) {} /// state management void reset() override { @@ -119,10 +123,43 @@ public: } // end anonymous namespace. +static bool canGoAfterDWARF(const MCSectionMachO &MSec) { + // These sections are created by the assembler itself after the end of + // the .s file. + StringRef SegName = MSec.getSegmentName(); + StringRef SecName = MSec.getSectionName(); + + if (SegName == "__LD" && SecName == "__compact_unwind") + return true; + + if (SegName == "__IMPORT") { + if (SecName == "__jump_table") + return true; + + if (SecName == "__pointers") + return true; + } + + if (SegName == "__TEXT" && SecName == "__eh_frame") + return true; + + if (SegName == "__DATA" && SecName == "__nl_symbol_ptr") + return true; + + return false; +} + void MCMachOStreamer::ChangeSection(const MCSection *Section, const MCExpr *Subsection) { // Change the section normally. - MCObjectStreamer::ChangeSection(Section, Subsection); + bool Created = MCObjectStreamer::changeSectionImpl(Section, Subsection); + const MCSectionMachO &MSec = *cast(Section); + StringRef SegName = MSec.getSegmentName(); + if (SegName == "__DWARF") + CreatedADWARFSection = true; + else if (Created && DWARFMustBeAtTheEnd && !canGoAfterDWARF(MSec)) + assert(!CreatedADWARFSection && "Creating regular section after DWARF"); + // Output a linker-local symbol so we don't need section-relative local // relocations. The linker hates us when we do that. if (LabelSections && !HasSectionLabel[Section]) { @@ -455,9 +492,10 @@ void MCMachOStreamer::FinishImpl() { MCStreamer *llvm::createMachOStreamer(MCContext &Context, MCAsmBackend &MAB, raw_ostream &OS, MCCodeEmitter *CE, - bool RelaxAll, + bool RelaxAll, bool DWARFMustBeAtTheEnd, bool LabelSections) { - MCMachOStreamer *S = new MCMachOStreamer(Context, MAB, OS, CE, LabelSections); + MCMachOStreamer *S = new MCMachOStreamer(Context, MAB, OS, CE, + DWARFMustBeAtTheEnd, LabelSections); if (RelaxAll) S->getAssembler().setRelaxAll(true); return S; diff --git a/lib/MC/MCObjectFileInfo.cpp b/lib/MC/MCObjectFileInfo.cpp index 11c9cc2..cd70362 100644 --- a/lib/MC/MCObjectFileInfo.cpp +++ b/lib/MC/MCObjectFileInfo.cpp @@ -183,82 +183,60 @@ void MCObjectFileInfo::InitMachOMCObjectFileInfo(Triple T) { // Debug Information. DwarfAccelNamesSection = - Ctx->getMachOSection("__DWARF", "__apple_names", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__apple_names", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "names_begin"); DwarfAccelObjCSection = - Ctx->getMachOSection("__DWARF", "__apple_objc", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__apple_objc", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "objc_begin"); // 16 character section limit... DwarfAccelNamespaceSection = - Ctx->getMachOSection("__DWARF", "__apple_namespac", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__apple_namespac", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "namespac_begin"); DwarfAccelTypesSection = - Ctx->getMachOSection("__DWARF", "__apple_types", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__apple_types", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "types_begin"); DwarfAbbrevSection = - Ctx->getMachOSection("__DWARF", "__debug_abbrev", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_abbrev", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "section_abbrev"); DwarfInfoSection = - Ctx->getMachOSection("__DWARF", "__debug_info", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_info", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "section_info"); DwarfLineSection = - Ctx->getMachOSection("__DWARF", "__debug_line", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_line", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "section_line"); DwarfFrameSection = - Ctx->getMachOSection("__DWARF", "__debug_frame", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_frame", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); DwarfPubNamesSection = - Ctx->getMachOSection("__DWARF", "__debug_pubnames", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_pubnames", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); DwarfPubTypesSection = - Ctx->getMachOSection("__DWARF", "__debug_pubtypes", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_pubtypes", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); DwarfGnuPubNamesSection = - Ctx->getMachOSection("__DWARF", "__debug_gnu_pubn", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_gnu_pubn", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); DwarfGnuPubTypesSection = - Ctx->getMachOSection("__DWARF", "__debug_gnu_pubt", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_gnu_pubt", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); DwarfStrSection = - Ctx->getMachOSection("__DWARF", "__debug_str", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_str", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "info_string"); DwarfLocSection = - Ctx->getMachOSection("__DWARF", "__debug_loc", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_loc", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "section_debug_loc"); DwarfARangesSection = - Ctx->getMachOSection("__DWARF", "__debug_aranges", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_aranges", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); DwarfRangesSection = - Ctx->getMachOSection("__DWARF", "__debug_ranges", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfMacroInfoSection = - Ctx->getMachOSection("__DWARF", "__debug_macinfo", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_ranges", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata(), "debug_range"); DwarfDebugInlineSection = - Ctx->getMachOSection("__DWARF", "__debug_inlined", - MachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - StackMapSection = - Ctx->getMachOSection("__LLVM_STACKMAPS", "__llvm_stackmaps", 0, - SectionKind::getMetadata()); + Ctx->getMachOSection("__DWARF", "__debug_inlined", MachO::S_ATTR_DEBUG, + SectionKind::getMetadata()); + StackMapSection = Ctx->getMachOSection("__LLVM_STACKMAPS", "__llvm_stackmaps", + 0, SectionKind::getMetadata()); TLSExtraDataSection = TLSTLVSection; } @@ -477,9 +455,10 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) { COFFDebugSymbolsSection = nullptr; // Debug Info Sections. - DwarfAbbrevSection = - Ctx->getELFSection(".debug_abbrev", ELF::SHT_PROGBITS, 0); - DwarfInfoSection = Ctx->getELFSection(".debug_info", ELF::SHT_PROGBITS, 0); + DwarfAbbrevSection = Ctx->getELFSection(".debug_abbrev", ELF::SHT_PROGBITS, 0, + "section_abbrev"); + DwarfInfoSection = + Ctx->getELFSection(".debug_info", ELF::SHT_PROGBITS, 0, "section_info"); DwarfLineSection = Ctx->getELFSection(".debug_line", ELF::SHT_PROGBITS, 0); DwarfFrameSection = Ctx->getELFSection(".debug_frame", ELF::SHT_PROGBITS, 0); DwarfPubNamesSection = @@ -497,21 +476,19 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) { DwarfARangesSection = Ctx->getELFSection(".debug_aranges", ELF::SHT_PROGBITS, 0); DwarfRangesSection = - Ctx->getELFSection(".debug_ranges", ELF::SHT_PROGBITS, 0); - DwarfMacroInfoSection = - Ctx->getELFSection(".debug_macinfo", ELF::SHT_PROGBITS, 0); + Ctx->getELFSection(".debug_ranges", ELF::SHT_PROGBITS, 0, "debug_range"); // DWARF5 Experimental Debug Info // Accelerator Tables DwarfAccelNamesSection = - Ctx->getELFSection(".apple_names", ELF::SHT_PROGBITS, 0); + Ctx->getELFSection(".apple_names", ELF::SHT_PROGBITS, 0, "names_begin"); DwarfAccelObjCSection = - Ctx->getELFSection(".apple_objc", ELF::SHT_PROGBITS, 0); - DwarfAccelNamespaceSection = - Ctx->getELFSection(".apple_namespaces", ELF::SHT_PROGBITS, 0); + Ctx->getELFSection(".apple_objc", ELF::SHT_PROGBITS, 0, "objc_begin"); + DwarfAccelNamespaceSection = Ctx->getELFSection( + ".apple_namespaces", ELF::SHT_PROGBITS, 0, "namespac_begin"); DwarfAccelTypesSection = - Ctx->getELFSection(".apple_types", ELF::SHT_PROGBITS, 0); + Ctx->getELFSection(".apple_types", ELF::SHT_PROGBITS, 0, "types_begin"); // Fission Sections DwarfInfoDWOSection = @@ -526,72 +503,58 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) { DwarfLineDWOSection = Ctx->getELFSection(".debug_line.dwo", ELF::SHT_PROGBITS, 0); DwarfLocDWOSection = - Ctx->getELFSection(".debug_loc.dwo", ELF::SHT_PROGBITS, 0); + Ctx->getELFSection(".debug_loc.dwo", ELF::SHT_PROGBITS, 0, "skel_loc"); DwarfStrOffDWOSection = Ctx->getELFSection(".debug_str_offsets.dwo", ELF::SHT_PROGBITS, 0); - DwarfAddrSection = Ctx->getELFSection(".debug_addr", ELF::SHT_PROGBITS, 0); + DwarfAddrSection = + Ctx->getELFSection(".debug_addr", ELF::SHT_PROGBITS, 0, "addr_sec"); StackMapSection = Ctx->getELFSection(".llvm_stackmaps", ELF::SHT_PROGBITS, ELF::SHF_ALLOC); } - void MCObjectFileInfo::InitCOFFMCObjectFileInfo(Triple T) { bool IsWoA = T.getArch() == Triple::arm || T.getArch() == Triple::thumb; CommDirectiveSupportsAlignment = true; // COFF - BSSSection = - Ctx->getCOFFSection(".bss", - COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getBSS()); - TextSection = - Ctx->getCOFFSection(".text", - (IsWoA ? COFF::IMAGE_SCN_MEM_16BIT - : (COFF::SectionCharacteristics)0) | - COFF::IMAGE_SCN_CNT_CODE | - COFF::IMAGE_SCN_MEM_EXECUTE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getText()); - DataSection = - Ctx->getCOFFSection(".data", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); - ReadOnlySection = - Ctx->getCOFFSection(".rdata", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getReadOnly()); + BSSSection = Ctx->getCOFFSection( + ".bss", COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ | COFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getBSS()); + TextSection = Ctx->getCOFFSection( + ".text", + (IsWoA ? COFF::IMAGE_SCN_MEM_16BIT : (COFF::SectionCharacteristics)0) | + COFF::IMAGE_SCN_CNT_CODE | COFF::IMAGE_SCN_MEM_EXECUTE | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getText()); + DataSection = Ctx->getCOFFSection( + ".data", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ | + COFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); + ReadOnlySection = Ctx->getCOFFSection( + ".rdata", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ, + SectionKind::getReadOnly()); if (T.isKnownWindowsMSVCEnvironment() || T.isWindowsItaniumEnvironment()) { StaticCtorSection = - Ctx->getCOFFSection(".CRT$XCU", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getReadOnly()); + Ctx->getCOFFSection(".CRT$XCU", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getReadOnly()); StaticDtorSection = - Ctx->getCOFFSection(".CRT$XTX", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getReadOnly()); + Ctx->getCOFFSection(".CRT$XTX", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getReadOnly()); } else { - StaticCtorSection = - Ctx->getCOFFSection(".ctors", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); - StaticDtorSection = - Ctx->getCOFFSection(".dtors", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); + StaticCtorSection = Ctx->getCOFFSection( + ".ctors", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ | COFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); + StaticDtorSection = Ctx->getCOFFSection( + ".dtors", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ | COFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); } // FIXME: We're emitting LSDA info into a readonly section on COFF, even @@ -611,187 +574,149 @@ void MCObjectFileInfo::InitCOFFMCObjectFileInfo(Triple T) { // Debug info. COFFDebugSymbolsSection = - Ctx->getCOFFSection(".debug$S", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - - DwarfAbbrevSection = - Ctx->getCOFFSection(".debug_abbrev", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfInfoSection = - Ctx->getCOFFSection(".debug_info", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfLineSection = - Ctx->getCOFFSection(".debug_line", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfFrameSection = - Ctx->getCOFFSection(".debug_frame", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfPubNamesSection = - Ctx->getCOFFSection(".debug_pubnames", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfPubTypesSection = - Ctx->getCOFFSection(".debug_pubtypes", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfGnuPubNamesSection = - Ctx->getCOFFSection(".debug_gnu_pubnames", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfGnuPubTypesSection = - Ctx->getCOFFSection(".debug_gnu_pubtypes", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfStrSection = - Ctx->getCOFFSection(".debug_str", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfLocSection = - Ctx->getCOFFSection(".debug_loc", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfARangesSection = - Ctx->getCOFFSection(".debug_aranges", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfRangesSection = - Ctx->getCOFFSection(".debug_ranges", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfMacroInfoSection = - Ctx->getCOFFSection(".debug_macinfo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfInfoDWOSection = - Ctx->getCOFFSection(".debug_info.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfTypesDWOSection = - Ctx->getCOFFSection(".debug_types.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfAbbrevDWOSection = - Ctx->getCOFFSection(".debug_abbrev.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfStrDWOSection = - Ctx->getCOFFSection(".debug_str.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfLineDWOSection = - Ctx->getCOFFSection(".debug_line.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfLocDWOSection = - Ctx->getCOFFSection(".debug_loc.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfStrOffDWOSection = - Ctx->getCOFFSection(".debug_str_offsets.dwo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfAddrSection = - Ctx->getCOFFSection(".debug_addr", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfAccelNamesSection = - Ctx->getCOFFSection(".apple_names", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfAccelNamespaceSection = - Ctx->getCOFFSection(".apple_namespaces", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfAccelTypesSection = - Ctx->getCOFFSection(".apple_types", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfAccelObjCSection = - Ctx->getCOFFSection(".apple_objc", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - - DrectveSection = - Ctx->getCOFFSection(".drectve", - COFF::IMAGE_SCN_LNK_INFO | - COFF::IMAGE_SCN_LNK_REMOVE, - SectionKind::getMetadata()); - - PDataSection = - Ctx->getCOFFSection(".pdata", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getDataRel()); - - XDataSection = - Ctx->getCOFFSection(".xdata", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getDataRel()); - - TLSDataSection = - Ctx->getCOFFSection(".tls$", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); + Ctx->getCOFFSection(".debug$S", COFF::IMAGE_SCN_MEM_DISCARDABLE | + COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + + DwarfAbbrevSection = Ctx->getCOFFSection( + ".debug_abbrev", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_abbrev"); + DwarfInfoSection = Ctx->getCOFFSection( + ".debug_info", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_info"); + DwarfLineSection = Ctx->getCOFFSection( + ".debug_line", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_line"); + + DwarfFrameSection = Ctx->getCOFFSection( + ".debug_frame", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfPubNamesSection = Ctx->getCOFFSection( + ".debug_pubnames", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfPubTypesSection = Ctx->getCOFFSection( + ".debug_pubtypes", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfGnuPubNamesSection = Ctx->getCOFFSection( + ".debug_gnu_pubnames", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfGnuPubTypesSection = Ctx->getCOFFSection( + ".debug_gnu_pubtypes", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfStrSection = Ctx->getCOFFSection( + ".debug_str", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "info_string"); + DwarfLocSection = Ctx->getCOFFSection( + ".debug_loc", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_debug_loc"); + DwarfARangesSection = Ctx->getCOFFSection( + ".debug_aranges", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfRangesSection = Ctx->getCOFFSection( + ".debug_ranges", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "debug_range"); + DwarfInfoDWOSection = Ctx->getCOFFSection( + ".debug_info.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_info_dwo"); + DwarfTypesDWOSection = Ctx->getCOFFSection( + ".debug_types.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_types_dwo"); + DwarfAbbrevDWOSection = Ctx->getCOFFSection( + ".debug_abbrev.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "section_abbrev_dwo"); + DwarfStrDWOSection = Ctx->getCOFFSection( + ".debug_str.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "skel_string"); + DwarfLineDWOSection = Ctx->getCOFFSection( + ".debug_line.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfLocDWOSection = Ctx->getCOFFSection( + ".debug_loc.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "skel_loc"); + DwarfStrOffDWOSection = Ctx->getCOFFSection( + ".debug_str_offsets.dwo", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + DwarfAddrSection = Ctx->getCOFFSection( + ".debug_addr", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "addr_sec"); + DwarfAccelNamesSection = Ctx->getCOFFSection( + ".apple_names", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "names_begin"); + DwarfAccelNamespaceSection = Ctx->getCOFFSection( + ".apple_namespaces", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "namespac_begin"); + DwarfAccelTypesSection = Ctx->getCOFFSection( + ".apple_types", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "types_begin"); + DwarfAccelObjCSection = Ctx->getCOFFSection( + ".apple_objc", + COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata(), "objc_begin"); + + DrectveSection = Ctx->getCOFFSection( + ".drectve", COFF::IMAGE_SCN_LNK_INFO | COFF::IMAGE_SCN_LNK_REMOVE, + SectionKind::getMetadata()); + + PDataSection = Ctx->getCOFFSection( + ".pdata", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ, + SectionKind::getDataRel()); + + XDataSection = Ctx->getCOFFSection( + ".xdata", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ, + SectionKind::getDataRel()); + + TLSDataSection = Ctx->getCOFFSection( + ".tls$", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ | + COFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); } void MCObjectFileInfo::InitMCObjectFileInfo(StringRef T, Reloc::Model relocm, diff --git a/lib/MC/MCObjectStreamer.cpp b/lib/MC/MCObjectStreamer.cpp index 08fe501..6aa2de3 100644 --- a/lib/MC/MCObjectStreamer.cpp +++ b/lib/MC/MCObjectStreamer.cpp @@ -20,6 +20,7 @@ #include "llvm/MC/MCSection.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TargetRegistry.h" using namespace llvm; MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, @@ -31,8 +32,8 @@ MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, raw_ostream &OS, MCCodeEmitter *Emitter_, - MCAssembler *_Assembler) - : MCStreamer(Context), Assembler(_Assembler), CurSectionData(nullptr), + MCAssembler *Assembler) + : MCStreamer(Context), Assembler(Assembler), CurSectionData(nullptr), EmitEHFrame(true), EmitDebugFrame(false) {} MCObjectStreamer::~MCObjectStreamer() { @@ -181,10 +182,16 @@ void MCObjectStreamer::EmitWeakReference(MCSymbol *Alias, void MCObjectStreamer::ChangeSection(const MCSection *Section, const MCExpr *Subsection) { + changeSectionImpl(Section, Subsection); +} + +bool MCObjectStreamer::changeSectionImpl(const MCSection *Section, + const MCExpr *Subsection) { assert(Section && "Cannot switch to a null section!"); flushPendingLabels(nullptr); - CurSectionData = &getAssembler().getOrCreateSectionData(*Section); + bool Created; + CurSectionData = &getAssembler().getOrCreateSectionData(*Section, &Created); int64_t IntSubsection = 0; if (Subsection && @@ -194,6 +201,7 @@ void MCObjectStreamer::ChangeSection(const MCSection *Section, report_fatal_error("Subsection number out of range"); CurInsertionPoint = CurSectionData->getSubsectionInsertionPoint(unsigned(IntSubsection)); + return Created; } void MCObjectStreamer::EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) { diff --git a/lib/MC/MCObjectWriter.cpp b/lib/MC/MCObjectWriter.cpp index 94d7cd6..3c536ec 100644 --- a/lib/MC/MCObjectWriter.cpp +++ b/lib/MC/MCObjectWriter.cpp @@ -17,11 +17,9 @@ using namespace llvm; MCObjectWriter::~MCObjectWriter() { } -bool -MCObjectWriter::IsSymbolRefDifferenceFullyResolved(const MCAssembler &Asm, - const MCSymbolRefExpr *A, - const MCSymbolRefExpr *B, - bool InSet) const { +bool MCObjectWriter::IsSymbolRefDifferenceFullyResolved( + const MCAssembler &Asm, const MCSymbolRefExpr *A, const MCSymbolRefExpr *B, + bool InSet) const { // Modified symbol references cannot be resolved. if (A->getKind() != MCSymbolRefExpr::VK_None || B->getKind() != MCSymbolRefExpr::VK_None) @@ -54,3 +52,5 @@ MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, // On ELF and COFF A - B is absolute if A and B are in the same section. return &SecA == &SecB; } + +bool MCObjectWriter::isWeak(const MCSymbolData &SD) const { return false; } diff --git a/lib/MC/MCParser/AsmLexer.cpp b/lib/MC/MCParser/AsmLexer.cpp index 5c8ec66..b983d99 100644 --- a/lib/MC/MCParser/AsmLexer.cpp +++ b/lib/MC/MCParser/AsmLexer.cpp @@ -21,7 +21,7 @@ #include using namespace llvm; -AsmLexer::AsmLexer(const MCAsmInfo &_MAI) : MAI(_MAI) { +AsmLexer::AsmLexer(const MCAsmInfo &MAI) : MAI(MAI) { CurPtr = nullptr; isAtStartOfLine = true; AllowAtInIdentifier = !StringRef(MAI.getCommentString()).startswith("@"); diff --git a/lib/MC/MCParser/AsmParser.cpp b/lib/MC/MCParser/AsmParser.cpp index ef6a540..2bf980b 100644 --- a/lib/MC/MCParser/AsmParser.cpp +++ b/lib/MC/MCParser/AsmParser.cpp @@ -339,8 +339,8 @@ private: DK_WEAK_DEF_CAN_BE_HIDDEN, DK_COMM, DK_COMMON, DK_LCOMM, DK_ABORT, DK_INCLUDE, DK_INCBIN, DK_CODE16, DK_CODE16GCC, DK_REPT, DK_IRP, DK_IRPC, DK_IF, DK_IFEQ, DK_IFGE, DK_IFGT, DK_IFLE, DK_IFLT, DK_IFNE, DK_IFB, - DK_IFNB, DK_IFC, DK_IFEQS, DK_IFNC, DK_IFDEF, DK_IFNDEF, DK_IFNOTDEF, - DK_ELSEIF, DK_ELSE, DK_ENDIF, + DK_IFNB, DK_IFC, DK_IFEQS, DK_IFNC, DK_IFNES, DK_IFDEF, DK_IFNDEF, + DK_IFNOTDEF, DK_ELSEIF, DK_ELSE, DK_ENDIF, DK_SPACE, DK_SKIP, DK_FILE, DK_LINE, DK_LOC, DK_STABS, DK_CFI_SECTIONS, DK_CFI_STARTPROC, DK_CFI_ENDPROC, DK_CFI_DEF_CFA, DK_CFI_DEF_CFA_OFFSET, DK_CFI_ADJUST_CFA_OFFSET, DK_CFI_DEF_CFA_REGISTER, @@ -435,8 +435,8 @@ private: bool parseDirectiveIfb(SMLoc DirectiveLoc, bool ExpectBlank); // ".ifc" or ".ifnc", depending on ExpectEqual. bool parseDirectiveIfc(SMLoc DirectiveLoc, bool ExpectEqual); - // ".ifeqs" - bool parseDirectiveIfeqs(SMLoc DirectiveLoc); + // ".ifeqs" or ".ifnes", depending on ExpectEqual. + bool parseDirectiveIfeqs(SMLoc DirectiveLoc, bool ExpectEqual); // ".ifdef" or ".ifndef", depending on expect_defined bool parseDirectiveIfdef(SMLoc DirectiveLoc, bool expect_defined); bool parseDirectiveElseIf(SMLoc DirectiveLoc); // ".elseif" @@ -486,10 +486,10 @@ extern MCAsmParserExtension *createCOFFAsmParser(); enum { DEFAULT_ADDRSPACE = 0 }; -AsmParser::AsmParser(SourceMgr &_SM, MCContext &_Ctx, MCStreamer &_Out, - const MCAsmInfo &_MAI) - : Lexer(_MAI), Ctx(_Ctx), Out(_Out), MAI(_MAI), SrcMgr(_SM), - PlatformParser(nullptr), CurBuffer(_SM.getMainFileID()), +AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out, + const MCAsmInfo &MAI) + : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM), + PlatformParser(nullptr), CurBuffer(SM.getMainFileID()), MacrosEnabledFlag(true), HadError(false), CppHashLineNumber(0), AssemblerDialect(~0U), IsDarwin(false), ParsingInlineAsm(false) { // Save the old handler. @@ -500,7 +500,7 @@ AsmParser::AsmParser(SourceMgr &_SM, MCContext &_Ctx, MCStreamer &_Out, Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer()); // Initialize the platform / file format parser. - switch (_Ctx.getObjectFileInfo()->getObjectFileType()) { + switch (Ctx.getObjectFileInfo()->getObjectFileType()) { case MCObjectFileInfo::IsCOFF: PlatformParser.reset(createCOFFAsmParser()); break; @@ -1244,9 +1244,11 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info, case DK_IFC: return parseDirectiveIfc(IDLoc, true); case DK_IFEQS: - return parseDirectiveIfeqs(IDLoc); + return parseDirectiveIfeqs(IDLoc, true); case DK_IFNC: return parseDirectiveIfc(IDLoc, false); + case DK_IFNES: + return parseDirectiveIfeqs(IDLoc, false); case DK_IFDEF: return parseDirectiveIfdef(IDLoc, true); case DK_IFNDEF: @@ -2791,7 +2793,7 @@ bool AsmParser::parseDirectiveFile(SMLoc DirectiveLoc) { if (FileNumber == -1) getStreamer().EmitFileDirective(Filename); else { - if (getContext().getGenDwarfForAssembly() == true) + if (getContext().getGenDwarfForAssembly()) Error(DirectiveLoc, "input can't have .file dwarf directives when -g is " "used to generate dwarf debug info for assembly code"); @@ -3943,9 +3945,12 @@ bool AsmParser::parseDirectiveIfc(SMLoc DirectiveLoc, bool ExpectEqual) { /// parseDirectiveIfeqs /// ::= .ifeqs string1, string2 -bool AsmParser::parseDirectiveIfeqs(SMLoc DirectiveLoc) { +bool AsmParser::parseDirectiveIfeqs(SMLoc DirectiveLoc, bool ExpectEqual) { if (Lexer.isNot(AsmToken::String)) { - TokError("expected string parameter for '.ifeqs' directive"); + if (ExpectEqual) + TokError("expected string parameter for '.ifeqs' directive"); + else + TokError("expected string parameter for '.ifnes' directive"); eatToEndOfStatement(); return true; } @@ -3954,7 +3959,10 @@ bool AsmParser::parseDirectiveIfeqs(SMLoc DirectiveLoc) { Lex(); if (Lexer.isNot(AsmToken::Comma)) { - TokError("expected comma after first string for '.ifeqs' directive"); + if (ExpectEqual) + TokError("expected comma after first string for '.ifeqs' directive"); + else + TokError("expected comma after first string for '.ifnes' directive"); eatToEndOfStatement(); return true; } @@ -3962,7 +3970,10 @@ bool AsmParser::parseDirectiveIfeqs(SMLoc DirectiveLoc) { Lex(); if (Lexer.isNot(AsmToken::String)) { - TokError("expected string parameter for '.ifeqs' directive"); + if (ExpectEqual) + TokError("expected string parameter for '.ifeqs' directive"); + else + TokError("expected string parameter for '.ifnes' directive"); eatToEndOfStatement(); return true; } @@ -3972,7 +3983,7 @@ bool AsmParser::parseDirectiveIfeqs(SMLoc DirectiveLoc) { TheCondStack.push_back(TheCondState); TheCondState.TheCond = AsmCond::IfCond; - TheCondState.CondMet = String1 == String2; + TheCondState.CondMet = ExpectEqual == (String1 == String2); TheCondState.Ignore = !TheCondState.CondMet; return false; @@ -4219,6 +4230,7 @@ void AsmParser::initializeDirectiveKindMap() { DirectiveKindMap[".ifc"] = DK_IFC; DirectiveKindMap[".ifeqs"] = DK_IFEQS; DirectiveKindMap[".ifnc"] = DK_IFNC; + DirectiveKindMap[".ifnes"] = DK_IFNES; DirectiveKindMap[".ifdef"] = DK_IFDEF; DirectiveKindMap[".ifndef"] = DK_IFNDEF; DirectiveKindMap[".ifnotdef"] = DK_IFNOTDEF; diff --git a/lib/MC/MCParser/DarwinAsmParser.cpp b/lib/MC/MCParser/DarwinAsmParser.cpp index 3ea745e..9102dc3 100644 --- a/lib/MC/MCParser/DarwinAsmParser.cpp +++ b/lib/MC/MCParser/DarwinAsmParser.cpp @@ -626,7 +626,7 @@ bool DarwinAsmParser::parseDirectiveSecureLogUnique(StringRef, SMLoc IDLoc) { if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in '.secure_log_unique' directive"); - if (getContext().getSecureLogUsed() != false) + if (getContext().getSecureLogUsed()) return Error(IDLoc, ".secure_log_unique specified multiple times"); // Get the secure log path. diff --git a/lib/MC/MCSection.cpp b/lib/MC/MCSection.cpp index ccf4a7d..7889f83 100644 --- a/lib/MC/MCSection.cpp +++ b/lib/MC/MCSection.cpp @@ -10,6 +10,7 @@ #include "llvm/MC/MCSection.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -17,6 +18,14 @@ using namespace llvm; // MCSection //===----------------------------------------------------------------------===// +MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) const { + if (!End) + End = Ctx.createTempSymbol("sec_end", true); + return End; +} + +bool MCSection::hasEnded() const { return End && End->isInSection(); } + MCSection::~MCSection() { } diff --git a/lib/MC/MCSectionMachO.cpp b/lib/MC/MCSectionMachO.cpp index 46beda4..c9f1591 100644 --- a/lib/MC/MCSectionMachO.cpp +++ b/lib/MC/MCSectionMachO.cpp @@ -70,8 +70,10 @@ ENTRY(nullptr /*FIXME*/, S_ATTR_LOC_RELOC) }; MCSectionMachO::MCSectionMachO(StringRef Segment, StringRef Section, - unsigned TAA, unsigned reserved2, SectionKind K) - : MCSection(SV_MachO, K), TypeAndAttributes(TAA), Reserved2(reserved2) { + unsigned TAA, unsigned reserved2, SectionKind K, + MCSymbol *Begin) + : MCSection(SV_MachO, K, Begin), TypeAndAttributes(TAA), + Reserved2(reserved2) { assert(Segment.size() <= 16 && Section.size() <= 16 && "Segment or section string too long"); for (unsigned i = 0; i != 16; ++i) { diff --git a/lib/MC/MCStreamer.cpp b/lib/MC/MCStreamer.cpp index f11ee66..27d0355 100644 --- a/lib/MC/MCStreamer.cpp +++ b/lib/MC/MCStreamer.cpp @@ -16,6 +16,7 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/MCSection.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWin64EH.h" #include "llvm/Support/ErrorHandling.h" @@ -661,3 +662,30 @@ void MCStreamer::EmitBundleAlignMode(unsigned AlignPow2) {} void MCStreamer::EmitBundleLock(bool AlignToEnd) {} void MCStreamer::FinishImpl() {} void MCStreamer::EmitBundleUnlock() {} + +void MCStreamer::SwitchSection(const MCSection *Section, + const MCExpr *Subsection) { + assert(Section && "Cannot switch to a null section!"); + MCSectionSubPair curSection = SectionStack.back().first; + SectionStack.back().second = curSection; + if (MCSectionSubPair(Section, Subsection) != curSection) { + SectionStack.back().first = MCSectionSubPair(Section, Subsection); + assert(!Section->hasEnded() && "Section already ended"); + ChangeSection(Section, Subsection); + MCSymbol *Sym = Section->getBeginSymbol(); + if (Sym && !Sym->isInSection()) + EmitLabel(Sym); + } +} + +MCSymbol *MCStreamer::endSection(const MCSection *Section) { + // TODO: keep track of the last subsection so that this symbol appears in the + // correct place. + MCSymbol *Sym = Section->getEndSymbol(Context); + if (Sym->isInSection()) + return Sym; + + SwitchSection(Section); + EmitLabel(Sym); + return Sym; +} diff --git a/lib/MC/MCWinEH.cpp b/lib/MC/MCWinEH.cpp index 47eaf0f..b1c95f8 100644 --- a/lib/MC/MCWinEH.cpp +++ b/lib/MC/MCWinEH.cpp @@ -11,6 +11,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCSectionCOFF.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWinEH.h" #include "llvm/Support/COFF.h" diff --git a/lib/MC/MachObjectWriter.cpp b/lib/MC/MachObjectWriter.cpp index 588d424..5e9e86f 100644 --- a/lib/MC/MachObjectWriter.cpp +++ b/lib/MC/MachObjectWriter.cpp @@ -23,6 +23,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MachO.h" +#include "llvm/Support/raw_ostream.h" #include using namespace llvm; diff --git a/lib/MC/SubtargetFeature.cpp b/lib/MC/SubtargetFeature.cpp index 587be54..ec6c9cb 100644 --- a/lib/MC/SubtargetFeature.cpp +++ b/lib/MC/SubtargetFeature.cpp @@ -201,9 +201,13 @@ SubtargetFeatures::ToggleFeature(uint64_t Bits, StringRef Feature, SetImpliedBits(Bits, FeatureEntry, FeatureTable); } } else { - errs() << "'" << Feature - << "' is not a recognized feature for this target" - << " (ignoring feature)\n"; + // Bug: 20140355 + // Silence this warning for now + if (false) { + errs() << "'" << Feature + << "' is not a recognized feature for this target" + << " (ignoring feature)\n"; + } } return Bits; @@ -281,9 +285,13 @@ SubtargetFeatures::getFeatureBits(StringRef CPU, ClearImpliedBits(Bits, FeatureEntry, FeatureTable); } } else { - errs() << "'" << Feature - << "' is not a recognized feature for this target" - << " (ignoring feature)\n"; + // Bug: 20140355 + // Silence this warning for now + if (false) { + errs() << "'" << Feature + << "' is not a recognized feature for this target" + << " (ignoring feature)\n"; + } } } diff --git a/lib/MC/WinCOFFObjectWriter.cpp b/lib/MC/WinCOFFObjectWriter.cpp index c519a9d..c6bc81d 100644 --- a/lib/MC/WinCOFFObjectWriter.cpp +++ b/lib/MC/WinCOFFObjectWriter.cpp @@ -175,6 +175,8 @@ public: const MCFragment &FB, bool InSet, bool IsPCRel) const override; + bool isWeak(const MCSymbolData &SD) const override; + void RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, bool &IsPCRel, @@ -661,6 +663,12 @@ bool WinCOFFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( InSet, IsPCRel); } +bool WinCOFFObjectWriter::isWeak(const MCSymbolData &SD) const { + // FIXME: this is for PR23025. Write a good description on + // why this is needed. + return SD.isExternal(); +} + void WinCOFFObjectWriter::RecordRelocation( MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, bool &IsPCRel, uint64_t &FixedValue) { diff --git a/lib/MC/WinCOFFStreamer.cpp b/lib/MC/WinCOFFStreamer.cpp index 41a3da7..f902d2b 100644 --- a/lib/MC/WinCOFFStreamer.cpp +++ b/lib/MC/WinCOFFStreamer.cpp @@ -230,11 +230,11 @@ void MCWinCOFFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, AssignSection(Symbol, Section); if (ByteAlignment != 1) - new MCAlignFragment(ByteAlignment, /*_Value=*/0, /*_ValueSize=*/0, + new MCAlignFragment(ByteAlignment, /*Value=*/0, /*ValueSize=*/0, ByteAlignment, &SectionData); MCFillFragment *Fragment = - new MCFillFragment(/*_Value=*/0, /*_ValueSize=*/0, Size, &SectionData); + new MCFillFragment(/*Value=*/0, /*ValueSize=*/0, Size, &SectionData); SD.setFragment(Fragment); } diff --git a/lib/Makefile b/lib/Makefile index 52fdaaf..f75ca58 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -12,6 +12,6 @@ include $(LEVEL)/Makefile.config PARALLEL_DIRS := IR AsmParser Bitcode Analysis Transforms CodeGen Target \ ExecutionEngine Linker LTO MC Object Option DebugInfo \ - IRReader LineEditor ProfileData + IRReader LineEditor ProfileData Passes include $(LEVEL)/Makefile.common diff --git a/lib/Object/Archive.cpp b/lib/Object/Archive.cpp index 43b0771..7d43daa 100644 --- a/lib/Object/Archive.cpp +++ b/lib/Object/Archive.cpp @@ -20,6 +20,7 @@ using namespace llvm; using namespace object; +using namespace llvm::support::endian; static const char *const Magic = "!\n"; static const char *const ThinMagic = "!\n"; @@ -363,11 +364,9 @@ ErrorOr Archive::Symbol::getMember() const { Offsets += sizeof(uint32_t); uint32_t Offset = 0; if (Parent->kind() == K_GNU) { - Offset = - *(reinterpret_cast(Offsets) + SymbolIndex); + Offset = read32be(Offsets + SymbolIndex * 4); } else if (Parent->kind() == K_MIPS64) { - Offset = - *(reinterpret_cast(Offsets) + SymbolIndex); + Offset = read64be(Offsets + SymbolIndex * 8); } else if (Parent->kind() == K_BSD) { // The SymbolIndex is an index into the ranlib structs that start at // Offsets (the first uint32_t is the number of bytes of the ranlib @@ -375,36 +374,29 @@ ErrorOr Archive::Symbol::getMember() const { // being a string table offset and the second being the offset into // the archive of the member that defines the symbol. Which is what // is needed here. - Offset = *(reinterpret_cast(Offsets) + - (SymbolIndex * 2) + 1); + Offset = read32le(Offsets + SymbolIndex * 8 + 4); } else { - uint32_t MemberCount = *reinterpret_cast(Buf); - // Skip offsets. - Buf += sizeof(support::ulittle32_t) + - (MemberCount * sizeof(support::ulittle32_t)); - - uint32_t SymbolCount = *reinterpret_cast(Buf); + uint32_t MemberCount = read32le(Buf); + Buf += MemberCount * 4 + 4; + uint32_t SymbolCount = read32le(Buf); if (SymbolIndex >= SymbolCount) return object_error::parse_failed; // Skip SymbolCount to get to the indices table. - const char *Indices = Buf + sizeof(support::ulittle32_t); + const char *Indices = Buf + 4; // Get the index of the offset in the file member offset table for this // symbol. - uint16_t OffsetIndex = - *(reinterpret_cast(Indices) - + SymbolIndex); + uint16_t OffsetIndex = read16le(Indices + SymbolIndex * 2); // Subtract 1 since OffsetIndex is 1 based. --OffsetIndex; if (OffsetIndex >= MemberCount) return object_error::parse_failed; - Offset = *(reinterpret_cast(Offsets) - + OffsetIndex); + Offset = read32le(Offsets + OffsetIndex * 4); } const char *Loc = Parent->getData().begin() + Offset; @@ -430,8 +422,7 @@ Archive::Symbol Archive::Symbol::getNext() const { // the string table followed by the string table. const char *Buf = Parent->SymbolTable->getBuffer().begin(); uint32_t RanlibCount = 0; - RanlibCount = (*reinterpret_cast(Buf)) / - (sizeof(uint32_t) * 2); + RanlibCount = read32le(Buf) / 8; // If t.SymbolIndex + 1 will be past the count of symbols (the RanlibCount) // don't change the t.StringIndex as we don't want to reference a ranlib // past RanlibCount. @@ -439,10 +430,8 @@ Archive::Symbol Archive::Symbol::getNext() const { const char *Ranlibs = Buf + 4; uint32_t CurRanStrx = 0; uint32_t NextRanStrx = 0; - CurRanStrx = *(reinterpret_cast(Ranlibs) + - (t.SymbolIndex * 2)); - NextRanStrx = *(reinterpret_cast(Ranlibs) + - ((t.SymbolIndex + 1) * 2)); + CurRanStrx = read32le(Ranlibs + t.SymbolIndex * 8); + NextRanStrx = read32le(Ranlibs + (t.SymbolIndex + 1) * 8); t.StringIndex -= CurRanStrx; t.StringIndex += NextRanStrx; } @@ -462,10 +451,10 @@ Archive::symbol_iterator Archive::symbol_begin() const { const char *buf = SymbolTable->getBuffer().begin(); if (kind() == K_GNU) { uint32_t symbol_count = 0; - symbol_count = *reinterpret_cast(buf); + symbol_count = read32be(buf); buf += sizeof(uint32_t) + (symbol_count * (sizeof(uint32_t))); } else if (kind() == K_MIPS64) { - uint64_t symbol_count = *reinterpret_cast(buf); + uint64_t symbol_count = read64be(buf); buf += sizeof(uint64_t) + (symbol_count * (sizeof(uint64_t))); } else if (kind() == K_BSD) { // The __.SYMDEF or "__.SYMDEF SORTED" member starts with a uint32_t @@ -475,11 +464,10 @@ Archive::symbol_iterator Archive::symbol_begin() const { // define the symbol. After that the next uint32_t is the byte count of // the string table followed by the string table. uint32_t ranlib_count = 0; - ranlib_count = (*reinterpret_cast(buf)) / - (sizeof(uint32_t) * 2); + ranlib_count = read32le(buf) / 8; const char *ranlibs = buf + 4; uint32_t ran_strx = 0; - ran_strx = *(reinterpret_cast(ranlibs)); + ran_strx = read32le(ranlibs); buf += sizeof(uint32_t) + (ranlib_count * (2 * (sizeof(uint32_t)))); // Skip the byte count of the string table. buf += sizeof(uint32_t); @@ -487,9 +475,9 @@ Archive::symbol_iterator Archive::symbol_begin() const { } else { uint32_t member_count = 0; uint32_t symbol_count = 0; - member_count = *reinterpret_cast(buf); + member_count = read32le(buf); buf += 4 + (member_count * 4); // Skip offsets. - symbol_count = *reinterpret_cast(buf); + symbol_count = read32le(buf); buf += 4 + (symbol_count * 2); // Skip indices. } uint32_t string_start_offset = buf - SymbolTable->getBuffer().begin(); @@ -503,17 +491,16 @@ Archive::symbol_iterator Archive::symbol_end() const { const char *buf = SymbolTable->getBuffer().begin(); uint32_t symbol_count = 0; if (kind() == K_GNU) { - symbol_count = *reinterpret_cast(buf); + symbol_count = read32be(buf); } else if (kind() == K_MIPS64) { - symbol_count = *reinterpret_cast(buf); + symbol_count = read64be(buf); } else if (kind() == K_BSD) { - symbol_count = (*reinterpret_cast(buf)) / - (sizeof(uint32_t) * 2); + symbol_count = read32le(buf) / 8; } else { uint32_t member_count = 0; - member_count = *reinterpret_cast(buf); + member_count = read32le(buf); buf += 4 + (member_count * 4); // Skip offsets. - symbol_count = *reinterpret_cast(buf); + symbol_count = read32le(buf); } return symbol_iterator(Symbol(this, symbol_count, 0)); } diff --git a/lib/Object/COFFObjectFile.cpp b/lib/Object/COFFObjectFile.cpp index cde6fdc..ad278a4 100644 --- a/lib/Object/COFFObjectFile.cpp +++ b/lib/Object/COFFObjectFile.cpp @@ -190,7 +190,9 @@ std::error_code COFFObjectFile::getSymbolType(DataRefImpl Ref, Result = SymbolRef::ST_Data; } else if (Symb.isFileRecord()) { Result = SymbolRef::ST_File; - } else if (SectionNumber == COFF::IMAGE_SYM_DEBUG) { + } else if (SectionNumber == COFF::IMAGE_SYM_DEBUG || + Symb.isSectionDefinition()) { + // TODO: perhaps we need a new symbol type ST_Section. Result = SymbolRef::ST_Debug; } else if (!COFF::isReservedSectionNumber(SectionNumber)) { const coff_section *Section = nullptr; @@ -359,12 +361,17 @@ bool COFFObjectFile::isSectionData(DataRefImpl Ref) const { bool COFFObjectFile::isSectionBSS(DataRefImpl Ref) const { const coff_section *Sec = toSec(Ref); - return Sec->Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA; + const uint32_t BssFlags = COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA | + COFF::IMAGE_SCN_MEM_READ | + COFF::IMAGE_SCN_MEM_WRITE; + return (Sec->Characteristics & BssFlags) == BssFlags; } bool COFFObjectFile::isSectionVirtual(DataRefImpl Ref) const { const coff_section *Sec = toSec(Ref); - return Sec->Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA; + // In COFF, a virtual section won't have any in-file + // content, so the file pointer to the content will be zero. + return Sec->PointerToRawData == 0; } bool COFFObjectFile::sectionContainsSymbol(DataRefImpl SecRef, diff --git a/lib/Object/ELFYAML.cpp b/lib/Object/ELFYAML.cpp index cce05cf..19527e2 100644 --- a/lib/Object/ELFYAML.cpp +++ b/lib/Object/ELFYAML.cpp @@ -235,6 +235,7 @@ void ScalarEnumerationTraits::enumeration( ECase(ELFOSABI_NSK) ECase(ELFOSABI_AROS) ECase(ELFOSABI_FENIXOS) + ECase(ELFOSABI_CLOUDABI) ECase(ELFOSABI_C6000_ELFABI) ECase(ELFOSABI_C6000_LINUX) ECase(ELFOSABI_ARM) diff --git a/lib/Object/IRObjectFile.cpp b/lib/Object/IRObjectFile.cpp index a2cbdcd..58c4ca3 100644 --- a/lib/Object/IRObjectFile.cpp +++ b/lib/Object/IRObjectFile.cpp @@ -13,6 +13,7 @@ #include "llvm/Object/IRObjectFile.h" #include "RecordStreamer.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/GVMaterializer.h" #include "llvm/IR/LLVMContext.h" @@ -35,12 +36,9 @@ using namespace object; IRObjectFile::IRObjectFile(MemoryBufferRef Object, std::unique_ptr Mod) : SymbolicFile(Binary::ID_IR, Object), M(std::move(Mod)) { - // If we have a DataLayout, setup a mangler. - const DataLayout *DL = M->getDataLayout(); - if (!DL) - return; - - Mang.reset(new Mangler(DL)); + // Setup a mangler with the DataLayout. + const DataLayout &DL = M->getDataLayout(); + Mang.reset(new Mangler(&DL)); const std::string &InlineAsm = M->getModuleInlineAsm(); if (InlineAsm.empty()) @@ -302,7 +300,9 @@ llvm::object::IRObjectFile::create(MemoryBufferRef Object, std::unique_ptr Buff( MemoryBuffer::getMemBuffer(BCOrErr.get(), false)); - ErrorOr MOrErr = getLazyBitcodeModule(std::move(Buff), Context); + ErrorOr MOrErr = + getLazyBitcodeModule(std::move(Buff), Context, nullptr, + /*ShouldLazyLoadMetadata*/ true); if (std::error_code EC = MOrErr.getError()) return EC; diff --git a/lib/Option/Arg.cpp b/lib/Option/Arg.cpp index af632d6..ac00073 100644 --- a/lib/Option/Arg.cpp +++ b/lib/Option/Arg.cpp @@ -17,22 +17,21 @@ using namespace llvm; using namespace llvm::opt; -Arg::Arg(const Option _Opt, StringRef S, unsigned _Index, const Arg *_BaseArg) - : Opt(_Opt), BaseArg(_BaseArg), Spelling(S), Index(_Index), - Claimed(false), OwnsValues(false) { -} - -Arg::Arg(const Option _Opt, StringRef S, unsigned _Index, - const char *Value0, const Arg *_BaseArg) - : Opt(_Opt), BaseArg(_BaseArg), Spelling(S), Index(_Index), - Claimed(false), OwnsValues(false) { +Arg::Arg(const Option Opt, StringRef S, unsigned Index, const Arg *BaseArg) + : Opt(Opt), BaseArg(BaseArg), Spelling(S), Index(Index), Claimed(false), + OwnsValues(false) {} + +Arg::Arg(const Option Opt, StringRef S, unsigned Index, const char *Value0, + const Arg *BaseArg) + : Opt(Opt), BaseArg(BaseArg), Spelling(S), Index(Index), Claimed(false), + OwnsValues(false) { Values.push_back(Value0); } -Arg::Arg(const Option _Opt, StringRef S, unsigned _Index, - const char *Value0, const char *Value1, const Arg *_BaseArg) - : Opt(_Opt), BaseArg(_BaseArg), Spelling(S), Index(_Index), - Claimed(false), OwnsValues(false) { +Arg::Arg(const Option Opt, StringRef S, unsigned Index, const char *Value0, + const char *Value1, const Arg *BaseArg) + : Opt(Opt), BaseArg(BaseArg), Spelling(S), Index(Index), Claimed(false), + OwnsValues(false) { Values.push_back(Value0); Values.push_back(Value1); } diff --git a/lib/Option/ArgList.cpp b/lib/Option/ArgList.cpp index 85e956f..4bc8f92 100644 --- a/lib/Option/ArgList.cpp +++ b/lib/Option/ArgList.cpp @@ -63,6 +63,26 @@ Arg *ArgList::getLastArgNoClaim(OptSpecifier Id0, OptSpecifier Id1) const { return nullptr; } +Arg *ArgList::getLastArgNoClaim(OptSpecifier Id0, OptSpecifier Id1, + OptSpecifier Id2) const { + // FIXME: Make search efficient? + for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it) + if ((*it)->getOption().matches(Id0) || (*it)->getOption().matches(Id1) || + (*it)->getOption().matches(Id2)) + return *it; + return nullptr; +} + +Arg *ArgList::getLastArgNoClaim(OptSpecifier Id0, OptSpecifier Id1, + OptSpecifier Id2, OptSpecifier Id3) const { + // FIXME: Make search efficient? + for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it) + if ((*it)->getOption().matches(Id0) || (*it)->getOption().matches(Id1) || + (*it)->getOption().matches(Id2) || (*it)->getOption().matches(Id3)) + return *it; + return nullptr; +} + Arg *ArgList::getLastArg(OptSpecifier Id) const { Arg *Res = nullptr; for (const_iterator it = begin(), ie = end(); it != ie; ++it) { @@ -285,11 +305,6 @@ void ArgList::ClaimAllArgs() const { (*it)->claim(); } -const char *ArgList::MakeArgString(const Twine &T) const { - SmallString<256> Str; - return MakeArgString(T.toStringRef(Str)); -} - const char *ArgList::GetOrMakeJoinedArgString(unsigned Index, StringRef LHS, StringRef RHS) const { @@ -334,19 +349,18 @@ unsigned InputArgList::MakeIndex(StringRef String0, return Index0; } -const char *InputArgList::MakeArgString(StringRef Str) const { +const char *InputArgList::MakeArgStringRef(StringRef Str) const { return getArgString(MakeIndex(Str)); } // -DerivedArgList::DerivedArgList(const InputArgList &_BaseArgs) - : BaseArgs(_BaseArgs) { -} +DerivedArgList::DerivedArgList(const InputArgList &BaseArgs) + : BaseArgs(BaseArgs) {} DerivedArgList::~DerivedArgList() {} -const char *DerivedArgList::MakeArgString(StringRef Str) const { +const char *DerivedArgList::MakeArgStringRef(StringRef Str) const { return BaseArgs.MakeArgString(Str); } diff --git a/lib/Option/OptTable.cpp b/lib/Option/OptTable.cpp index dca02c1..96ba183 100644 --- a/lib/Option/OptTable.cpp +++ b/lib/Option/OptTable.cpp @@ -84,15 +84,11 @@ static inline bool operator<(const OptTable::Info &I, const char *Name) { OptSpecifier::OptSpecifier(const Option *Opt) : ID(Opt->getID()) {} -OptTable::OptTable(const Info *_OptionInfos, unsigned _NumOptionInfos, - bool _IgnoreCase) - : OptionInfos(_OptionInfos), - NumOptionInfos(_NumOptionInfos), - IgnoreCase(_IgnoreCase), - TheInputOptionID(0), - TheUnknownOptionID(0), - FirstSearchableIndex(0) -{ +OptTable::OptTable(const Info *OptionInfos, unsigned NumOptionInfos, + bool IgnoreCase) + : OptionInfos(OptionInfos), NumOptionInfos(NumOptionInfos), + IgnoreCase(IgnoreCase), TheInputOptionID(0), TheUnknownOptionID(0), + FirstSearchableIndex(0) { // Explicitly zero initialize the error to work around a bug in array // value-initialization on MinGW with gcc 4.3.5. diff --git a/lib/Option/Option.cpp b/lib/Option/Option.cpp index cdc63c3..e29d649 100644 --- a/lib/Option/Option.cpp +++ b/lib/Option/Option.cpp @@ -35,9 +35,6 @@ Option::Option(const OptTable::Info *info, const OptTable *owner) } } -Option::~Option() { -} - void Option::dump() const { llvm::errs() << "<"; switch (getKind()) { diff --git a/lib/Passes/Android.mk b/lib/Passes/Android.mk new file mode 100644 index 0000000..6e441d7 --- /dev/null +++ b/lib/Passes/Android.mk @@ -0,0 +1,30 @@ +LOCAL_PATH:= $(call my-dir) + +passes_SRC_FILES := \ + PassBuilder.cpp + +# For the host +# ===================================================== +include $(CLEAR_VARS) + +LOCAL_MODULE:= libLLVMPasses +LOCAL_MODULE_TAGS := optional +LOCAL_SRC_FILES := $(passes_SRC_FILES) + +include $(LLVM_HOST_BUILD_MK) +include $(LLVM_GEN_INTRINSICS_MK) +include $(BUILD_HOST_STATIC_LIBRARY) + +# For the device +# ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) +include $(CLEAR_VARS) + +LOCAL_MODULE:= libLLVMPasses +LOCAL_MODULE_TAGS := optional +LOCAL_SRC_FILES := $(passes_SRC_FILES) + +include $(LLVM_DEVICE_BUILD_MK) +include $(LLVM_GEN_INTRINSICS_MK) +include $(BUILD_STATIC_LIBRARY) +endif diff --git a/lib/Passes/CMakeLists.txt b/lib/Passes/CMakeLists.txt new file mode 100644 index 0000000..6ceac7b --- /dev/null +++ b/lib/Passes/CMakeLists.txt @@ -0,0 +1,8 @@ +add_llvm_library(LLVMPasses + PassBuilder.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/Passes + ) + +add_dependencies(LLVMPasses intrinsics_gen) diff --git a/lib/Passes/LLVMBuild.txt b/lib/Passes/LLVMBuild.txt new file mode 100644 index 0000000..3063fe3 --- /dev/null +++ b/lib/Passes/LLVMBuild.txt @@ -0,0 +1,22 @@ +;===- ./lib/Passes/LLVMBuild.txt -------------------------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = Passes +parent = Libraries +required_libraries = Analysis Core IPA IPO InstCombine Scalar Support TransformUtils Vectorize diff --git a/lib/Passes/Makefile b/lib/Passes/Makefile new file mode 100644 index 0000000..413dc5c --- /dev/null +++ b/lib/Passes/Makefile @@ -0,0 +1,14 @@ +##===- lib/Passes/Makefile ---------------------------------*- Makefile -*-===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===----------------------------------------------------------------------===## + +LEVEL = ../.. +LIBRARYNAME = LLVMPasses +BUILD_ARCHIVE := 1 + +include $(LEVEL)/Makefile.common diff --git a/lib/Passes/PassBuilder.cpp b/lib/Passes/PassBuilder.cpp new file mode 100644 index 0000000..ba71320 --- /dev/null +++ b/lib/Passes/PassBuilder.cpp @@ -0,0 +1,412 @@ +//===- Parsing, selection, and construction of pass pipelines -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file provides the implementation of the PassBuilder based on our +/// static pass registry as well as related functionality. It also provides +/// helpers to aid in analyzing, debugging, and testing passes and pass +/// pipelines. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/PassManager.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Support/Debug.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/InstCombine/InstCombine.h" +#include "llvm/Transforms/Scalar/EarlyCSE.h" +#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" +#include "llvm/Transforms/Scalar/SimplifyCFG.h" + +using namespace llvm; + +namespace { + +/// \brief No-op module pass which does nothing. +struct NoOpModulePass { + PreservedAnalyses run(Module &M) { return PreservedAnalyses::all(); } + static StringRef name() { return "NoOpModulePass"; } +}; + +/// \brief No-op module analysis. +struct NoOpModuleAnalysis { + struct Result {}; + Result run(Module &) { return Result(); } + static StringRef name() { return "NoOpModuleAnalysis"; } + static void *ID() { return (void *)&PassID; } +private: + static char PassID; +}; + +char NoOpModuleAnalysis::PassID; + +/// \brief No-op CGSCC pass which does nothing. +struct NoOpCGSCCPass { + PreservedAnalyses run(LazyCallGraph::SCC &C) { + return PreservedAnalyses::all(); + } + static StringRef name() { return "NoOpCGSCCPass"; } +}; + +/// \brief No-op CGSCC analysis. +struct NoOpCGSCCAnalysis { + struct Result {}; + Result run(LazyCallGraph::SCC &) { return Result(); } + static StringRef name() { return "NoOpCGSCCAnalysis"; } + static void *ID() { return (void *)&PassID; } +private: + static char PassID; +}; + +char NoOpCGSCCAnalysis::PassID; + +/// \brief No-op function pass which does nothing. +struct NoOpFunctionPass { + PreservedAnalyses run(Function &F) { return PreservedAnalyses::all(); } + static StringRef name() { return "NoOpFunctionPass"; } +}; + +/// \brief No-op function analysis. +struct NoOpFunctionAnalysis { + struct Result {}; + Result run(Function &) { return Result(); } + static StringRef name() { return "NoOpFunctionAnalysis"; } + static void *ID() { return (void *)&PassID; } +private: + static char PassID; +}; + +char NoOpFunctionAnalysis::PassID; + +} // End anonymous namespace. + +void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) { +#define MODULE_ANALYSIS(NAME, CREATE_PASS) \ + MAM.registerPass(CREATE_PASS); +#include "PassRegistry.def" +} + +void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) { +#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ + CGAM.registerPass(CREATE_PASS); +#include "PassRegistry.def" +} + +void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + FAM.registerPass(CREATE_PASS); +#include "PassRegistry.def" +} + +#ifndef NDEBUG +static bool isModulePassName(StringRef Name) { +#define MODULE_PASS(NAME, CREATE_PASS) if (Name == NAME) return true; +#define MODULE_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ + return true; +#include "PassRegistry.def" + + return false; +} +#endif + +static bool isCGSCCPassName(StringRef Name) { +#define CGSCC_PASS(NAME, CREATE_PASS) if (Name == NAME) return true; +#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ + return true; +#include "PassRegistry.def" + + return false; +} + +static bool isFunctionPassName(StringRef Name) { +#define FUNCTION_PASS(NAME, CREATE_PASS) if (Name == NAME) return true; +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ + return true; +#include "PassRegistry.def" + + return false; +} + +bool PassBuilder::parseModulePassName(ModulePassManager &MPM, StringRef Name) { +#define MODULE_PASS(NAME, CREATE_PASS) \ + if (Name == NAME) { \ + MPM.addPass(CREATE_PASS); \ + return true; \ + } +#define MODULE_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == "require<" NAME ">") { \ + MPM.addPass(RequireAnalysisPass()); \ + return true; \ + } \ + if (Name == "invalidate<" NAME ">") { \ + MPM.addPass(InvalidateAnalysisPass()); \ + return true; \ + } +#include "PassRegistry.def" + + return false; +} + +bool PassBuilder::parseCGSCCPassName(CGSCCPassManager &CGPM, StringRef Name) { +#define CGSCC_PASS(NAME, CREATE_PASS) \ + if (Name == NAME) { \ + CGPM.addPass(CREATE_PASS); \ + return true; \ + } +#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == "require<" NAME ">") { \ + CGPM.addPass(RequireAnalysisPass()); \ + return true; \ + } \ + if (Name == "invalidate<" NAME ">") { \ + CGPM.addPass(InvalidateAnalysisPass()); \ + return true; \ + } +#include "PassRegistry.def" + + return false; +} + +bool PassBuilder::parseFunctionPassName(FunctionPassManager &FPM, + StringRef Name) { +#define FUNCTION_PASS(NAME, CREATE_PASS) \ + if (Name == NAME) { \ + FPM.addPass(CREATE_PASS); \ + return true; \ + } +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == "require<" NAME ">") { \ + FPM.addPass(RequireAnalysisPass()); \ + return true; \ + } \ + if (Name == "invalidate<" NAME ">") { \ + FPM.addPass(InvalidateAnalysisPass()); \ + return true; \ + } +#include "PassRegistry.def" + + return false; +} + +bool PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM, + StringRef &PipelineText, + bool VerifyEachPass, + bool DebugLogging) { + for (;;) { + // Parse nested pass managers by recursing. + if (PipelineText.startswith("function(")) { + FunctionPassManager NestedFPM(DebugLogging); + + // Parse the inner pipeline inte the nested manager. + PipelineText = PipelineText.substr(strlen("function(")); + if (!parseFunctionPassPipeline(NestedFPM, PipelineText, VerifyEachPass, + DebugLogging) || + PipelineText.empty()) + return false; + assert(PipelineText[0] == ')'); + PipelineText = PipelineText.substr(1); + + // Add the nested pass manager with the appropriate adaptor. + FPM.addPass(std::move(NestedFPM)); + } else { + // Otherwise try to parse a pass name. + size_t End = PipelineText.find_first_of(",)"); + if (!parseFunctionPassName(FPM, PipelineText.substr(0, End))) + return false; + if (VerifyEachPass) + FPM.addPass(VerifierPass()); + + PipelineText = PipelineText.substr(End); + } + + if (PipelineText.empty() || PipelineText[0] == ')') + return true; + + assert(PipelineText[0] == ','); + PipelineText = PipelineText.substr(1); + } +} + +bool PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM, + StringRef &PipelineText, + bool VerifyEachPass, + bool DebugLogging) { + for (;;) { + // Parse nested pass managers by recursing. + if (PipelineText.startswith("cgscc(")) { + CGSCCPassManager NestedCGPM(DebugLogging); + + // Parse the inner pipeline into the nested manager. + PipelineText = PipelineText.substr(strlen("cgscc(")); + if (!parseCGSCCPassPipeline(NestedCGPM, PipelineText, VerifyEachPass, + DebugLogging) || + PipelineText.empty()) + return false; + assert(PipelineText[0] == ')'); + PipelineText = PipelineText.substr(1); + + // Add the nested pass manager with the appropriate adaptor. + CGPM.addPass(std::move(NestedCGPM)); + } else if (PipelineText.startswith("function(")) { + FunctionPassManager NestedFPM(DebugLogging); + + // Parse the inner pipeline inte the nested manager. + PipelineText = PipelineText.substr(strlen("function(")); + if (!parseFunctionPassPipeline(NestedFPM, PipelineText, VerifyEachPass, + DebugLogging) || + PipelineText.empty()) + return false; + assert(PipelineText[0] == ')'); + PipelineText = PipelineText.substr(1); + + // Add the nested pass manager with the appropriate adaptor. + CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(NestedFPM))); + } else { + // Otherwise try to parse a pass name. + size_t End = PipelineText.find_first_of(",)"); + if (!parseCGSCCPassName(CGPM, PipelineText.substr(0, End))) + return false; + // FIXME: No verifier support for CGSCC passes! + + PipelineText = PipelineText.substr(End); + } + + if (PipelineText.empty() || PipelineText[0] == ')') + return true; + + assert(PipelineText[0] == ','); + PipelineText = PipelineText.substr(1); + } +} + +bool PassBuilder::parseModulePassPipeline(ModulePassManager &MPM, + StringRef &PipelineText, + bool VerifyEachPass, + bool DebugLogging) { + for (;;) { + // Parse nested pass managers by recursing. + if (PipelineText.startswith("module(")) { + ModulePassManager NestedMPM(DebugLogging); + + // Parse the inner pipeline into the nested manager. + PipelineText = PipelineText.substr(strlen("module(")); + if (!parseModulePassPipeline(NestedMPM, PipelineText, VerifyEachPass, + DebugLogging) || + PipelineText.empty()) + return false; + assert(PipelineText[0] == ')'); + PipelineText = PipelineText.substr(1); + + // Now add the nested manager as a module pass. + MPM.addPass(std::move(NestedMPM)); + } else if (PipelineText.startswith("cgscc(")) { + CGSCCPassManager NestedCGPM(DebugLogging); + + // Parse the inner pipeline inte the nested manager. + PipelineText = PipelineText.substr(strlen("cgscc(")); + if (!parseCGSCCPassPipeline(NestedCGPM, PipelineText, VerifyEachPass, + DebugLogging) || + PipelineText.empty()) + return false; + assert(PipelineText[0] == ')'); + PipelineText = PipelineText.substr(1); + + // Add the nested pass manager with the appropriate adaptor. + MPM.addPass( + createModuleToPostOrderCGSCCPassAdaptor(std::move(NestedCGPM))); + } else if (PipelineText.startswith("function(")) { + FunctionPassManager NestedFPM(DebugLogging); + + // Parse the inner pipeline inte the nested manager. + PipelineText = PipelineText.substr(strlen("function(")); + if (!parseFunctionPassPipeline(NestedFPM, PipelineText, VerifyEachPass, + DebugLogging) || + PipelineText.empty()) + return false; + assert(PipelineText[0] == ')'); + PipelineText = PipelineText.substr(1); + + // Add the nested pass manager with the appropriate adaptor. + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(NestedFPM))); + } else { + // Otherwise try to parse a pass name. + size_t End = PipelineText.find_first_of(",)"); + if (!parseModulePassName(MPM, PipelineText.substr(0, End))) + return false; + if (VerifyEachPass) + MPM.addPass(VerifierPass()); + + PipelineText = PipelineText.substr(End); + } + + if (PipelineText.empty() || PipelineText[0] == ')') + return true; + + assert(PipelineText[0] == ','); + PipelineText = PipelineText.substr(1); + } +} + +// Primary pass pipeline description parsing routine. +// FIXME: Should this routine accept a TargetMachine or require the caller to +// pre-populate the analysis managers with target-specific stuff? +bool PassBuilder::parsePassPipeline(ModulePassManager &MPM, + StringRef PipelineText, bool VerifyEachPass, + bool DebugLogging) { + // By default, try to parse the pipeline as-if it were within an implicit + // 'module(...)' pass pipeline. If this will parse at all, it needs to + // consume the entire string. + if (parseModulePassPipeline(MPM, PipelineText, VerifyEachPass, DebugLogging)) + return PipelineText.empty(); + + // This isn't parsable as a module pipeline, look for the end of a pass name + // and directly drop down to that layer. + StringRef FirstName = + PipelineText.substr(0, PipelineText.find_first_of(",)")); + assert(!isModulePassName(FirstName) && + "Already handled all module pipeline options."); + + // If this looks like a CGSCC pass, parse the whole thing as a CGSCC + // pipeline. + if (isCGSCCPassName(FirstName)) { + CGSCCPassManager CGPM(DebugLogging); + if (!parseCGSCCPassPipeline(CGPM, PipelineText, VerifyEachPass, + DebugLogging) || + !PipelineText.empty()) + return false; + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); + return true; + } + + // Similarly, if this looks like a Function pass, parse the whole thing as + // a Function pipelien. + if (isFunctionPassName(FirstName)) { + FunctionPassManager FPM(DebugLogging); + if (!parseFunctionPassPipeline(FPM, PipelineText, VerifyEachPass, + DebugLogging) || + !PipelineText.empty()) + return false; + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + return true; + } + + return false; +} diff --git a/lib/Passes/PassRegistry.def b/lib/Passes/PassRegistry.def new file mode 100644 index 0000000..d768a3a --- /dev/null +++ b/lib/Passes/PassRegistry.def @@ -0,0 +1,77 @@ +//===- PassRegistry.def - Registry of passes --------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the core LLVM +// libraries. This file describes both transformation passes and analyses +// Analyses are registered while transformation passes have names registered +// that can be used when providing a textual pass pipeline. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_ANALYSIS +#define MODULE_ANALYSIS(NAME, CREATE_PASS) +#endif +MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis()) +MODULE_ANALYSIS("no-op-module", NoOpModuleAnalysis()) +MODULE_ANALYSIS("targetlibinfo", TargetLibraryAnalysis()) +#undef MODULE_ANALYSIS + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("invalidate", InvalidateAllAnalysesPass()) +MODULE_PASS("no-op-module", NoOpModulePass()) +MODULE_PASS("print", PrintModulePass(dbgs())) +MODULE_PASS("print-cg", LazyCallGraphPrinterPass(dbgs())) +MODULE_PASS("verify", VerifierPass()) +#undef MODULE_PASS + +#ifndef CGSCC_ANALYSIS +#define CGSCC_ANALYSIS(NAME, CREATE_PASS) +#endif +CGSCC_ANALYSIS("no-op-cgscc", NoOpCGSCCAnalysis()) +#undef CGSCC_ANALYSIS + +#ifndef CGSCC_PASS +#define CGSCC_PASS(NAME, CREATE_PASS) +#endif +CGSCC_PASS("invalidate", InvalidateAllAnalysesPass()) +CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass()) +#undef CGSCC_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis()) +FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis()) +FUNCTION_ANALYSIS("loops", LoopAnalysis()) +FUNCTION_ANALYSIS("no-op-function", NoOpFunctionAnalysis()) +FUNCTION_ANALYSIS("targetlibinfo", TargetLibraryAnalysis()) +FUNCTION_ANALYSIS("targetir", + TM ? TM->getTargetIRAnalysis() : TargetIRAnalysis()) +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("early-cse", EarlyCSEPass()) +FUNCTION_PASS("instcombine", InstCombinePass()) +FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) +FUNCTION_PASS("no-op-function", NoOpFunctionPass()) +FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass()) +FUNCTION_PASS("print", PrintFunctionPass(dbgs())) +FUNCTION_PASS("print", AssumptionPrinterPass(dbgs())) +FUNCTION_PASS("print", DominatorTreePrinterPass(dbgs())) +FUNCTION_PASS("print", LoopPrinterPass(dbgs())) +FUNCTION_PASS("simplify-cfg", SimplifyCFGPass()) +FUNCTION_PASS("verify", VerifierPass()) +FUNCTION_PASS("verify", DominatorTreeVerifierPass()) +#undef FUNCTION_PASS diff --git a/lib/ProfileData/CoverageMapping.cpp b/lib/ProfileData/CoverageMapping.cpp index 31213d7..46d494b 100644 --- a/lib/ProfileData/CoverageMapping.cpp +++ b/lib/ProfileData/CoverageMapping.cpp @@ -20,6 +20,7 @@ #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; using namespace coverage; @@ -217,12 +218,13 @@ CoverageMapping::load(CoverageMappingReader &CoverageReader, } ErrorOr> -CoverageMapping::load(StringRef ObjectFilename, StringRef ProfileFilename) { +CoverageMapping::load(StringRef ObjectFilename, StringRef ProfileFilename, + Triple::ArchType Arch) { auto CounterMappingBuff = MemoryBuffer::getFileOrSTDIN(ObjectFilename); if (std::error_code EC = CounterMappingBuff.getError()) return EC; auto CoverageReaderOrErr = - BinaryCoverageReader::create(CounterMappingBuff.get()); + BinaryCoverageReader::create(CounterMappingBuff.get(), Arch); if (std::error_code EC = CoverageReaderOrErr.getError()) return EC; auto CoverageReader = std::move(CoverageReaderOrErr.get()); diff --git a/lib/ProfileData/CoverageMappingReader.cpp b/lib/ProfileData/CoverageMappingReader.cpp index d32f1da..12e9e88 100644 --- a/lib/ProfileData/CoverageMappingReader.cpp +++ b/lib/ProfileData/CoverageMappingReader.cpp @@ -14,9 +14,12 @@ #include "llvm/ProfileData/CoverageMappingReader.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/Object/MachOUniversal.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/LEB128.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; using namespace coverage; @@ -287,24 +290,6 @@ std::error_code RawCoverageMappingReader::read() { } namespace { -/// \brief The coverage mapping data for a single function. -/// It points to the function's name. -template struct CoverageMappingFunctionRecord { - IntPtrT FunctionNamePtr; - uint32_t FunctionNameSize; - uint32_t CoverageMappingSize; - uint64_t FunctionHash; -}; - -/// \brief The coverage mapping data for a single translation unit. -/// It points to the array of function coverage mapping records and the encoded -/// filenames array. -template struct CoverageMappingTURecord { - uint32_t FunctionRecordsSize; - uint32_t FilenamesSize; - uint32_t CoverageMappingsSize; - uint32_t Version; -}; /// \brief A helper structure to access the data from a section /// in an object file. @@ -331,77 +316,76 @@ struct SectionData { }; } -template +template std::error_code readCoverageMappingData( SectionData &ProfileNames, StringRef Data, std::vector &Records, std::vector &Filenames) { + using namespace support; llvm::DenseSet UniqueFunctionMappingData; // Read the records in the coverage data section. - while (!Data.empty()) { - if (Data.size() < sizeof(CoverageMappingTURecord)) + for (const char *Buf = Data.data(), *End = Buf + Data.size(); Buf < End;) { + if (Buf + 4 * sizeof(uint32_t) > End) return instrprof_error::malformed; - auto TU = reinterpret_cast *>(Data.data()); - Data = Data.substr(sizeof(CoverageMappingTURecord)); - switch (TU->Version) { + uint32_t NRecords = endian::readNext(Buf); + uint32_t FilenamesSize = endian::readNext(Buf); + uint32_t CoverageSize = endian::readNext(Buf); + uint32_t Version = endian::readNext(Buf); + + switch (Version) { case CoverageMappingVersion1: break; default: return instrprof_error::unsupported_version; } - auto Version = CoverageMappingVersion(TU->Version); - // Get the function records. - auto FunctionRecords = - reinterpret_cast *>(Data.data()); - if (Data.size() < - sizeof(CoverageMappingFunctionRecord) * TU->FunctionRecordsSize) - return instrprof_error::malformed; - Data = Data.substr(sizeof(CoverageMappingFunctionRecord) * - TU->FunctionRecordsSize); + // Skip past the function records, saving the start and end for later. + const char *FunBuf = Buf; + Buf += NRecords * (sizeof(T) + 2 * sizeof(uint32_t) + sizeof(uint64_t)); + const char *FunEnd = Buf; // Get the filenames. - if (Data.size() < TU->FilenamesSize) + if (Buf + FilenamesSize > End) return instrprof_error::malformed; - auto RawFilenames = Data.substr(0, TU->FilenamesSize); - Data = Data.substr(TU->FilenamesSize); size_t FilenamesBegin = Filenames.size(); - RawCoverageFilenamesReader Reader(RawFilenames, Filenames); + RawCoverageFilenamesReader Reader(StringRef(Buf, FilenamesSize), Filenames); if (auto Err = Reader.read()) return Err; + Buf += FilenamesSize; - // Get the coverage mappings. - if (Data.size() < TU->CoverageMappingsSize) + // We'll read the coverage mapping records in the loop below. + const char *CovBuf = Buf; + Buf += CoverageSize; + const char *CovEnd = Buf; + if (Buf > End) return instrprof_error::malformed; - auto CoverageMappings = Data.substr(0, TU->CoverageMappingsSize); - Data = Data.substr(TU->CoverageMappingsSize); - for (unsigned I = 0; I < TU->FunctionRecordsSize; ++I) { - auto &MappingRecord = FunctionRecords[I]; + while (FunBuf < FunEnd) { + // Read the function information + T NamePtr = endian::readNext(FunBuf); + uint32_t NameSize = endian::readNext(FunBuf); + uint32_t DataSize = endian::readNext(FunBuf); + uint64_t FuncHash = endian::readNext(FunBuf); - // Get the coverage mapping. - if (CoverageMappings.size() < MappingRecord.CoverageMappingSize) + // Now use that to read the coverage data. + if (CovBuf + DataSize > CovEnd) return instrprof_error::malformed; - auto Mapping = - CoverageMappings.substr(0, MappingRecord.CoverageMappingSize); - CoverageMappings = - CoverageMappings.substr(MappingRecord.CoverageMappingSize); + auto Mapping = StringRef(CovBuf, DataSize); + CovBuf += DataSize; // Ignore this record if we already have a record that points to the same - // function name. - // This is useful to ignore the redundant records for the functions - // with ODR linkage. - if (!UniqueFunctionMappingData.insert(MappingRecord.FunctionNamePtr) - .second) + // function name. This is useful to ignore the redundant records for the + // functions with ODR linkage. + if (!UniqueFunctionMappingData.insert(NamePtr).second) continue; - StringRef FunctionName; - if (auto Err = - ProfileNames.get(MappingRecord.FunctionNamePtr, - MappingRecord.FunctionNameSize, FunctionName)) - return Err; + + // Finally, grab the name and create a record. + StringRef FuncName; + if (std::error_code EC = ProfileNames.get(NamePtr, NameSize, FuncName)) + return EC; Records.push_back(BinaryCoverageReader::ProfileMappingRecord( - Version, FunctionName, MappingRecord.FunctionHash, Mapping, + CoverageMappingVersion(Version), FuncName, FuncHash, Mapping, FilenamesBegin, Filenames.size() - FilenamesBegin)); } } @@ -414,8 +398,10 @@ static const char *TestingFormatMagic = "llvmcovmtestdata"; static std::error_code loadTestingFormat(StringRef Data, SectionData &ProfileNames, StringRef &CoverageMapping, - uint8_t &BytesInAddress) { + uint8_t &BytesInAddress, + support::endianness &Endian) { BytesInAddress = 8; + Endian = support::endianness::little; Data = Data.substr(StringRef(TestingFormatMagic).size()); if (Data.size() < 1) @@ -444,12 +430,35 @@ static std::error_code loadTestingFormat(StringRef Data, static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer, SectionData &ProfileNames, StringRef &CoverageMapping, - uint8_t &BytesInAddress) { - auto ObjectFileOrErr = object::ObjectFile::createObjectFile(ObjectBuffer); - if (std::error_code EC = ObjectFileOrErr.getError()) + uint8_t &BytesInAddress, + support::endianness &Endian, + Triple::ArchType Arch) { + auto BinOrErr = object::createBinary(ObjectBuffer); + if (std::error_code EC = BinOrErr.getError()) return EC; - auto OF = std::move(ObjectFileOrErr.get()); + auto Bin = std::move(BinOrErr.get()); + std::unique_ptr OF; + if (auto *Universal = dyn_cast(Bin.get())) { + // If we have a universal binary, try to look up the object for the + // appropriate architecture. + auto ObjectFileOrErr = Universal->getObjectForArch(Arch); + if (std::error_code EC = ObjectFileOrErr.getError()) + return EC; + OF = std::move(ObjectFileOrErr.get()); + } else if (isa(Bin.get())) { + // For any other object file, upcast and take ownership. + OF.reset(cast(Bin.release())); + // If we've asked for a particular arch, make sure they match. + if (Arch != Triple::ArchType::UnknownArch && OF->getArch() != Arch) + return object_error::arch_not_found; + } else + // We can only handle object files. + return instrprof_error::malformed; + + // The coverage uses native pointer sizes for the object it's written in. BytesInAddress = OF->getBytesInAddress(); + Endian = OF->isLittleEndian() ? support::endianness::little + : support::endianness::big; // Look for the sections that we are interested in. int FoundSectionCount = 0; @@ -479,28 +488,36 @@ static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer, } ErrorOr> -BinaryCoverageReader::create(std::unique_ptr &ObjectBuffer) { +BinaryCoverageReader::create(std::unique_ptr &ObjectBuffer, + Triple::ArchType Arch) { std::unique_ptr Reader(new BinaryCoverageReader()); SectionData Profile; StringRef Coverage; uint8_t BytesInAddress; + support::endianness Endian; std::error_code EC; if (ObjectBuffer->getBuffer().startswith(TestingFormatMagic)) // This is a special format used for testing. EC = loadTestingFormat(ObjectBuffer->getBuffer(), Profile, Coverage, - BytesInAddress); + BytesInAddress, Endian); else EC = loadBinaryFormat(ObjectBuffer->getMemBufferRef(), Profile, Coverage, - BytesInAddress); + BytesInAddress, Endian, Arch); if (EC) return EC; - if (BytesInAddress == 4) - EC = readCoverageMappingData( + if (BytesInAddress == 4 && Endian == support::endianness::little) + EC = readCoverageMappingData( + Profile, Coverage, Reader->MappingRecords, Reader->Filenames); + else if (BytesInAddress == 4 && Endian == support::endianness::big) + EC = readCoverageMappingData( + Profile, Coverage, Reader->MappingRecords, Reader->Filenames); + else if (BytesInAddress == 8 && Endian == support::endianness::little) + EC = readCoverageMappingData( Profile, Coverage, Reader->MappingRecords, Reader->Filenames); - else if (BytesInAddress == 8) - EC = readCoverageMappingData( + else if (BytesInAddress == 8 && Endian == support::endianness::big) + EC = readCoverageMappingData( Profile, Coverage, Reader->MappingRecords, Reader->Filenames); else return instrprof_error::malformed; diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp index 01e199d..3a5b266 100644 --- a/lib/ProfileData/InstrProfReader.cpp +++ b/lib/ProfileData/InstrProfReader.cpp @@ -14,6 +14,7 @@ #include "llvm/ProfileData/InstrProfReader.h" #include "InstrProfIndexed.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ProfileData/InstrProf.h" #include @@ -112,7 +113,7 @@ std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { // Read the function hash. if (Line.is_at_end()) return error(instrprof_error::truncated); - if ((Line++)->getAsInteger(10, Record.Hash)) + if ((Line++)->getAsInteger(0, Record.Hash)) return error(instrprof_error::malformed); // Read the number of counters. diff --git a/lib/Support/APFloat.cpp b/lib/Support/APFloat.cpp index 393ecf4..5a402bb 100644 --- a/lib/Support/APFloat.cpp +++ b/lib/Support/APFloat.cpp @@ -1248,10 +1248,10 @@ APFloat::roundAwayFromZero(roundingMode rounding_mode, return false; case rmTowardPositive: - return sign == false; + return !sign; case rmTowardNegative: - return sign == true; + return sign; } llvm_unreachable("Invalid rounding mode found"); } @@ -1430,7 +1430,7 @@ APFloat::addOrSubtractSignificand(const APFloat &rhs, bool subtract) /* Determine if the operation on the absolute values is effectively an addition or subtraction. */ - subtract ^= (sign ^ rhs.sign) ? true : false; + subtract ^= static_cast(sign ^ rhs.sign); /* Are we bigger exponent-wise than the RHS? */ bits = exponent - rhs.exponent; diff --git a/lib/Support/APInt.cpp b/lib/Support/APInt.cpp index 50a639c..2533fa0 100644 --- a/lib/Support/APInt.cpp +++ b/lib/Support/APInt.cpp @@ -672,6 +672,14 @@ hash_code llvm::hash_value(const APInt &Arg) { return hash_combine_range(Arg.pVal, Arg.pVal + Arg.getNumWords()); } +bool APInt::isSplat(unsigned SplatSizeInBits) const { + assert(getBitWidth() % SplatSizeInBits == 0 && + "SplatSizeInBits must divide width!"); + // We can check that all parts of an integer are equal by making use of a + // little trick: rotate and check if it's still the same value. + return *this == rotl(SplatSizeInBits); +} + /// HiBits - This function returns the high "numBits" bits of this APInt. APInt APInt::getHiBits(unsigned numBits) const { return APIntOps::lshr(*this, BitWidth - numBits); @@ -1310,13 +1318,8 @@ APInt APInt::sqrt() const { // libc sqrt function which will probably use a hardware sqrt computation. // This should be faster than the algorithm below. if (magnitude < 52) { -#if HAVE_ROUND return APInt(BitWidth, uint64_t(::round(::sqrt(double(isSingleWord()?VAL:pVal[0]))))); -#else - return APInt(BitWidth, - uint64_t(::sqrt(double(isSingleWord()?VAL:pVal[0])) + 0.5)); -#endif } // Okay, all the short cuts are exhausted. We must compute it. The following @@ -1508,21 +1511,18 @@ static void KnuthDiv(unsigned *u, unsigned *v, unsigned *q, unsigned* r, assert(u && "Must provide dividend"); assert(v && "Must provide divisor"); assert(q && "Must provide quotient"); - assert(u != v && u != q && v != q && "Must us different memory"); + assert(u != v && u != q && v != q && "Must use different memory"); assert(n>1 && "n must be > 1"); - // Knuth uses the value b as the base of the number system. In our case b - // is 2^31 so we just set it to -1u. - uint64_t b = uint64_t(1) << 32; + // b denotes the base of the number system. In our case b is 2^32. + LLVM_CONSTEXPR uint64_t b = uint64_t(1) << 32; -#if 0 DEBUG(dbgs() << "KnuthDiv: m=" << m << " n=" << n << '\n'); DEBUG(dbgs() << "KnuthDiv: original:"); DEBUG(for (int i = m+n; i >=0; i--) dbgs() << " " << u[i]); DEBUG(dbgs() << " by"); DEBUG(for (int i = n; i >0; i--) dbgs() << " " << v[i-1]); DEBUG(dbgs() << '\n'); -#endif // D1. [Normalize.] Set d = b / (v[n-1] + 1) and multiply all the digits of // u and v by d. Note that we have taken Knuth's advice here to use a power // of 2 value for d such that d * v[n-1] >= b/2 (b is the base). A power of @@ -1547,13 +1547,12 @@ static void KnuthDiv(unsigned *u, unsigned *v, unsigned *q, unsigned* r, } } u[m+n] = u_carry; -#if 0 + DEBUG(dbgs() << "KnuthDiv: normal:"); DEBUG(for (int i = m+n; i >=0; i--) dbgs() << " " << u[i]); DEBUG(dbgs() << " by"); DEBUG(for (int i = n; i >0; i--) dbgs() << " " << v[i-1]); DEBUG(dbgs() << '\n'); -#endif // D2. [Initialize j.] Set j to m. This is the loop counter over the places. int j = m; @@ -1583,46 +1582,35 @@ static void KnuthDiv(unsigned *u, unsigned *v, unsigned *q, unsigned* r, // (u[j+n]u[j+n-1]..u[j]) - qp * (v[n-1]...v[1]v[0]). This computation // consists of a simple multiplication by a one-place number, combined with // a subtraction. + // The digits (u[j+n]...u[j]) should be kept positive; if the result of + // this step is actually negative, (u[j+n]...u[j]) should be left as the + // true value plus b**(n+1), namely as the b's complement of + // the true value, and a "borrow" to the left should be remembered. bool isNeg = false; for (unsigned i = 0; i < n; ++i) { - uint64_t u_tmp = uint64_t(u[j+i]) | (uint64_t(u[j+i+1]) << 32); + uint64_t u_tmp = (uint64_t(u[j+i+1]) << 32) | uint64_t(u[j+i]); uint64_t subtrahend = uint64_t(qp) * uint64_t(v[i]); bool borrow = subtrahend > u_tmp; - DEBUG(dbgs() << "KnuthDiv: u_tmp == " << u_tmp - << ", subtrahend == " << subtrahend + DEBUG(dbgs() << "KnuthDiv: u_tmp = " << u_tmp + << ", subtrahend = " << subtrahend << ", borrow = " << borrow << '\n'); uint64_t result = u_tmp - subtrahend; unsigned k = j + i; - u[k++] = (unsigned)(result & (b-1)); // subtract low word - u[k++] = (unsigned)(result >> 32); // subtract high word - while (borrow && k <= m+n) { // deal with borrow to the left + u[k++] = (unsigned)result; // subtraction low word + u[k++] = (unsigned)(result >> 32); // subtraction high word + while (borrow && k <= m+n) { // deal with borrow to the left borrow = u[k] == 0; u[k]--; k++; } isNeg |= borrow; - DEBUG(dbgs() << "KnuthDiv: u[j+i] == " << u[j+i] << ", u[j+i+1] == " << - u[j+i+1] << '\n'); + DEBUG(dbgs() << "KnuthDiv: u[j+i] = " << u[j+i] + << ", u[j+i+1] = " << u[j+i+1] << '\n'); } DEBUG(dbgs() << "KnuthDiv: after subtraction:"); DEBUG(for (int i = m+n; i >=0; i--) dbgs() << " " << u[i]); DEBUG(dbgs() << '\n'); - // The digits (u[j+n]...u[j]) should be kept positive; if the result of - // this step is actually negative, (u[j+n]...u[j]) should be left as the - // true value plus b**(n+1), namely as the b's complement of - // the true value, and a "borrow" to the left should be remembered. - // - if (isNeg) { - bool carry = true; // true because b's complement is "complement + 1" - for (unsigned i = 0; i <= m+n; ++i) { - u[i] = ~u[i] + carry; // b's complement - carry = carry && u[i] == 0; - } - } - DEBUG(dbgs() << "KnuthDiv: after complement:"); - DEBUG(for (int i = m+n; i >=0; i--) dbgs() << " " << u[i]); - DEBUG(dbgs() << '\n'); // D5. [Test remainder.] Set q[j] = qp. If the result of step D4 was // negative, go to step D6; otherwise go on to step D7. @@ -1644,7 +1632,7 @@ static void KnuthDiv(unsigned *u, unsigned *v, unsigned *q, unsigned* r, u[j+n] += carry; } DEBUG(dbgs() << "KnuthDiv: after correction:"); - DEBUG(for (int i = m+n; i >=0; i--) dbgs() <<" " << u[i]); + DEBUG(for (int i = m+n; i >=0; i--) dbgs() << " " << u[i]); DEBUG(dbgs() << "\nKnuthDiv: digit result = " << q[j] << '\n'); // D7. [Loop on j.] Decrease j by one. Now if j >= 0, go back to D3. @@ -1677,9 +1665,7 @@ static void KnuthDiv(unsigned *u, unsigned *v, unsigned *q, unsigned* r, } DEBUG(dbgs() << '\n'); } -#if 0 DEBUG(dbgs() << '\n'); -#endif } void APInt::divide(const APInt LHS, unsigned lhsWords, @@ -1803,6 +1789,8 @@ void APInt::divide(const APInt LHS, unsigned lhsWords, // The quotient is in Q. Reconstitute the quotient into Quotient's low // order words. + // This case is currently dead as all users of divide() handle trivial cases + // earlier. if (lhsWords == 1) { uint64_t tmp = uint64_t(Q[0]) | (uint64_t(Q[1]) << (APINT_BITS_PER_WORD / 2)); @@ -2296,13 +2284,13 @@ void APInt::dump() const { this->toStringUnsigned(U); this->toStringSigned(S); dbgs() << "APInt(" << BitWidth << "b, " - << U.str() << "u " << S.str() << "s)"; + << U << "u " << S << "s)"; } void APInt::print(raw_ostream &OS, bool isSigned) const { SmallString<40> S; this->toString(S, 10, isSigned, /* formatAsCLiteral = */false); - OS << S.str(); + OS << S; } // This implements a variety of operations on a representation of diff --git a/lib/Support/Allocator.cpp b/lib/Support/Allocator.cpp index 7c306b2..f48edac 100644 --- a/lib/Support/Allocator.cpp +++ b/lib/Support/Allocator.cpp @@ -12,12 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/Allocator.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/DataTypes.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/Recycler.h" #include "llvm/Support/raw_ostream.h" -#include namespace llvm { diff --git a/lib/Support/Android.mk b/lib/Support/Android.mk index 34448a7..4d1f526 100644 --- a/lib/Support/Android.mk +++ b/lib/Support/Android.mk @@ -35,8 +35,6 @@ support_SRC_FILES := \ IntervalMap.cpp \ IntEqClasses.cpp \ IntrusiveRefCntPtr.cpp \ - IsInf.cpp \ - IsNAN.cpp \ LEB128.cpp \ LineIterator.cpp \ Locale.cpp \ diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt index a44c1a3..684afa9 100644 --- a/lib/Support/CMakeLists.txt +++ b/lib/Support/CMakeLists.txt @@ -58,8 +58,6 @@ add_llvm_library(LLVMSupport IntEqClasses.cpp IntervalMap.cpp IntrusiveRefCntPtr.cpp - IsInf.cpp - IsNAN.cpp LEB128.cpp LineIterator.cpp Locale.cpp diff --git a/lib/Support/CommandLine.cpp b/lib/Support/CommandLine.cpp index b49ec36..af6c605 100644 --- a/lib/Support/CommandLine.cpp +++ b/lib/Support/CommandLine.cpp @@ -19,6 +19,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm-c/Support.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" @@ -32,10 +33,8 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" -#include #include #include -#include using namespace llvm; using namespace cl; @@ -1463,10 +1462,9 @@ void basic_parser_impl::printOptionNoValue(const Option &O, // -help and -help-hidden option implementation // -static int OptNameCompare(const void *LHS, const void *RHS) { - typedef std::pair pair_ty; - - return strcmp(((const pair_ty *)LHS)->first, ((const pair_ty *)RHS)->first); +static int OptNameCompare(const std::pair *LHS, + const std::pair *RHS) { + return strcmp(LHS->first, RHS->first); } // Copy Options into a vector so we can sort them as we like. @@ -1494,7 +1492,7 @@ static void sortOpts(StringMap