From 2c3e0051c31c3f5b2328b447eadf1cf9c4427442 Mon Sep 17 00:00:00 2001 From: Pirama Arumuga Nainar Date: Wed, 6 May 2015 11:46:36 -0700 Subject: Update aosp/master LLVM for rebase to r235153 Change-Id: I9bf53792f9fc30570e81a8d80d296c681d005ea7 (cherry picked from commit 0c7f116bb6950ef819323d855415b2f2b0aad987) --- lib/Analysis/AliasAnalysis.cpp | 38 +- lib/Analysis/AliasAnalysisCounter.cpp | 2 +- lib/Analysis/AliasAnalysisEvaluator.cpp | 2 +- lib/Analysis/AliasSetTracker.cpp | 2 +- lib/Analysis/Analysis.cpp | 1 + lib/Analysis/Android.mk | 1 + lib/Analysis/BasicAliasAnalysis.cpp | 4 +- lib/Analysis/BlockFrequencyInfo.cpp | 2 +- lib/Analysis/BlockFrequencyInfoImpl.cpp | 54 +- lib/Analysis/BranchProbabilityInfo.cpp | 30 +- lib/Analysis/CFGPrinter.cpp | 4 +- lib/Analysis/CFLAliasAnalysis.cpp | 4 +- lib/Analysis/CMakeLists.txt | 1 + lib/Analysis/ConstantFolding.cpp | 25 +- lib/Analysis/DivergenceAnalysis.cpp | 337 +++++ lib/Analysis/IPA/CallGraphSCCPass.cpp | 11 +- lib/Analysis/IPA/GlobalsModRef.cpp | 2 +- lib/Analysis/IPA/InlineCost.cpp | 37 +- lib/Analysis/InstructionSimplify.cpp | 24 +- lib/Analysis/LoopAccessAnalysis.cpp | 42 +- lib/Analysis/MemDepPrinter.cpp | 2 +- lib/Analysis/MemoryDependenceAnalysis.cpp | 20 +- lib/Analysis/ModuleDebugInfoPrinter.cpp | 50 +- lib/Analysis/RegionPass.cpp | 2 +- lib/Analysis/RegionPrinter.cpp | 2 +- lib/Analysis/ScalarEvolution.cpp | 66 +- lib/Analysis/ScalarEvolutionExpander.cpp | 72 +- lib/Analysis/TargetLibraryInfo.cpp | 4 +- lib/Analysis/TargetTransformInfo.cpp | 4 + lib/Analysis/ValueTracking.cpp | 7 +- lib/AsmParser/LLLexer.cpp | 1 + lib/AsmParser/LLLexer.h | 1 - lib/AsmParser/LLParser.cpp | 105 +- lib/AsmParser/LLParser.h | 2 +- lib/AsmParser/LLToken.h | 1 + lib/Bitcode/Reader/BitcodeReader.cpp | 42 +- lib/Bitcode/Writer/BitcodeWriter.cpp | 78 +- lib/Bitcode/Writer/BitcodeWriterPass.cpp | 16 +- lib/Bitcode/Writer/ValueEnumerator.cpp | 20 +- lib/Bitcode/Writer/ValueEnumerator.h | 5 +- lib/CodeGen/AggressiveAntiDepBreaker.h | 2 +- lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 38 +- .../AsmPrinter/DbgValueHistoryCalculator.cpp | 22 +- lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h | 14 +- lib/CodeGen/AsmPrinter/DebugLocEntry.h | 29 +- lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp | 130 +- lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 299 ++-- lib/CodeGen/AsmPrinter/DwarfDebug.h | 55 +- lib/CodeGen/AsmPrinter/DwarfException.h | 4 +- lib/CodeGen/AsmPrinter/DwarfExpression.cpp | 28 +- lib/CodeGen/AsmPrinter/DwarfExpression.h | 3 +- lib/CodeGen/AsmPrinter/DwarfFile.cpp | 4 +- lib/CodeGen/AsmPrinter/DwarfUnit.cpp | 492 ++++--- lib/CodeGen/AsmPrinter/DwarfUnit.h | 4 +- lib/CodeGen/AsmPrinter/EHStreamer.cpp | 28 +- lib/CodeGen/AsmPrinter/EHStreamer.h | 6 +- lib/CodeGen/AsmPrinter/Win64Exception.cpp | 274 +++- lib/CodeGen/AsmPrinter/Win64Exception.h | 9 +- lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp | 29 +- lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h | 2 +- lib/CodeGen/CodeGenPrepare.cpp | 104 +- lib/CodeGen/CriticalAntiDepBreaker.h | 2 +- lib/CodeGen/EarlyIfConversion.cpp | 5 +- lib/CodeGen/GCMetadata.cpp | 4 - lib/CodeGen/GCRootLowering.cpp | 17 +- lib/CodeGen/GlobalMerge.cpp | 6 +- lib/CodeGen/InlineSpiller.cpp | 4 +- lib/CodeGen/LLVMTargetMachine.cpp | 19 +- lib/CodeGen/LexicalScopes.cpp | 105 +- lib/CodeGen/LiveDebugVariables.cpp | 69 +- lib/CodeGen/LiveDebugVariables.h | 2 +- lib/CodeGen/LiveInterval.cpp | 56 +- lib/CodeGen/MachineBasicBlock.cpp | 2 +- lib/CodeGen/MachineBlockPlacement.cpp | 6 +- lib/CodeGen/MachineFunction.cpp | 4 +- lib/CodeGen/MachineInstr.cpp | 27 +- lib/CodeGen/MachineLICM.cpp | 240 ++-- lib/CodeGen/MachineModuleInfo.cpp | 29 +- lib/CodeGen/MachineModuleInfoImpls.cpp | 9 +- lib/CodeGen/MachineScheduler.cpp | 4 +- lib/CodeGen/MachineTraceMetrics.cpp | 16 +- lib/CodeGen/PostRASchedulerList.cpp | 2 +- lib/CodeGen/PrologEpilogInserter.cpp | 20 + lib/CodeGen/RegAllocFast.cpp | 13 +- lib/CodeGen/RegAllocGreedy.cpp | 9 +- lib/CodeGen/RegisterCoalescer.cpp | 48 +- lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 339 +++-- lib/CodeGen/SelectionDAG/FastISel.cpp | 17 +- lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp | 209 ++- lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 2 + lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 55 +- lib/CodeGen/SelectionDAG/LegalizeTypes.h | 2 +- lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp | 4 +- lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 35 +- lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp | 5 +- lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h | 2 +- lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp | 2 +- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 57 +- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 127 +- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h | 8 +- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp | 30 +- lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 123 +- lib/CodeGen/SelectionDAG/StatepointLowering.cpp | 68 +- lib/CodeGen/ShadowStackGCLowering.cpp | 4 +- lib/CodeGen/SjLjEHPrepare.cpp | 43 +- lib/CodeGen/SpillPlacement.h | 2 +- lib/CodeGen/StackColoring.cpp | 2 +- lib/CodeGen/TargetLoweringBase.cpp | 19 +- lib/CodeGen/TargetLoweringObjectFileImpl.cpp | 21 +- lib/CodeGen/WinEHPrepare.cpp | 981 +++++++++----- lib/DebugInfo/DWARF/DWARFDebugFrame.cpp | 6 +- lib/DebugInfo/PDB/PDBSymbolFunc.cpp | 11 +- lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp | 12 +- lib/ExecutionEngine/EventListenerCommon.h | 68 - lib/ExecutionEngine/ExecutionEngine.cpp | 176 ++- lib/ExecutionEngine/ExecutionEngineBindings.cpp | 2 +- lib/ExecutionEngine/GDBRegistrationListener.cpp | 2 +- .../IntelJITEvents/IntelJITEventListener.cpp | 3 - lib/ExecutionEngine/Interpreter/Execution.cpp | 16 +- .../Interpreter/ExternalFunctions.cpp | 9 +- lib/ExecutionEngine/Interpreter/Interpreter.h | 2 +- lib/ExecutionEngine/MCJIT/MCJIT.cpp | 84 +- lib/ExecutionEngine/MCJIT/MCJIT.h | 84 +- .../OProfileJIT/OProfileJITEventListener.cpp | 2 - lib/ExecutionEngine/Orc/Android.mk | 1 + lib/ExecutionEngine/Orc/CMakeLists.txt | 1 + lib/ExecutionEngine/Orc/ExecutionUtils.cpp | 102 ++ lib/ExecutionEngine/Orc/IndirectionUtils.cpp | 63 +- lib/ExecutionEngine/Orc/OrcMCJITReplacement.h | 124 +- lib/ExecutionEngine/Orc/OrcTargetSupport.cpp | 100 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp | 95 +- .../RuntimeDyld/RuntimeDyldCOFF.cpp | 6 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h | 10 +- .../RuntimeDyld/RuntimeDyldChecker.cpp | 2 +- .../RuntimeDyld/RuntimeDyldCheckerImpl.h | 1 + lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp | 198 ++- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h | 35 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h | 18 +- .../RuntimeDyld/RuntimeDyldMachO.cpp | 52 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h | 14 +- .../RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h | 8 +- .../RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h | 5 +- .../RuntimeDyld/Targets/RuntimeDyldMachOARM.h | 4 +- .../RuntimeDyld/Targets/RuntimeDyldMachOI386.h | 5 +- .../RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h | 5 +- lib/Fuzzer/CMakeLists.txt | 8 +- lib/Fuzzer/FuzzerDFSan.cpp | 275 ++++ lib/Fuzzer/FuzzerDriver.cpp | 78 +- lib/Fuzzer/FuzzerFlags.def | 51 +- lib/Fuzzer/FuzzerIO.cpp | 6 + lib/Fuzzer/FuzzerInternal.h | 30 +- lib/Fuzzer/FuzzerLoop.cpp | 138 +- lib/Fuzzer/FuzzerUtil.cpp | 13 +- lib/Fuzzer/README.txt | 112 +- lib/Fuzzer/cxx_fuzzer_tokens.txt | 218 +++ lib/Fuzzer/dfsan_fuzzer_abi.list | 12 + lib/Fuzzer/test/CMakeLists.txt | 15 +- lib/Fuzzer/test/CxxTokensTest.cpp | 24 + lib/Fuzzer/test/dfsan/CMakeLists.txt | 17 + lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp | 30 + lib/Fuzzer/test/fuzzer.test | 6 + lib/IR/Android.mk | 1 - lib/IR/AsmWriter.cpp | 156 +-- lib/IR/AttributeImpl.h | 8 +- lib/IR/Attributes.cpp | 85 +- lib/IR/AutoUpgrade.cpp | 51 +- lib/IR/BasicBlock.cpp | 4 +- lib/IR/CMakeLists.txt | 1 - lib/IR/ConstantFold.cpp | 39 +- lib/IR/Constants.cpp | 27 +- lib/IR/Core.cpp | 16 +- lib/IR/DIBuilder.cpp | 702 +++++----- lib/IR/DebugInfo.cpp | 828 ++---------- lib/IR/DebugInfoMetadata.cpp | 148 +- lib/IR/DebugLoc.cpp | 126 +- lib/IR/DiagnosticInfo.cpp | 12 +- lib/IR/Function.cpp | 13 +- lib/IR/GCOV.cpp | 2 +- lib/IR/IRBuilder.cpp | 3 +- lib/IR/IRPrintingPasses.cpp | 19 +- lib/IR/InlineAsm.cpp | 4 +- lib/IR/Instruction.cpp | 4 +- lib/IR/Instructions.cpp | 38 +- lib/IR/LLVMContextImpl.h | 189 +-- lib/IR/LegacyPassManager.cpp | 2 +- lib/IR/Metadata.cpp | 8 +- lib/IR/UseListOrder.cpp | 43 - lib/IR/Value.cpp | 4 +- lib/IR/Verifier.cpp | 566 +++++++- lib/LTO/LTOCodeGenerator.cpp | 14 +- lib/LTO/LTOModule.cpp | 2 +- lib/Linker/LinkModules.cpp | 10 +- lib/MC/ELFObjectWriter.cpp | 349 ++--- lib/MC/MCAsmStreamer.cpp | 24 +- lib/MC/MCAssembler.cpp | 61 +- lib/MC/MCContext.cpp | 100 +- lib/MC/MCDisassembler/Disassembler.cpp | 50 +- lib/MC/MCDwarf.cpp | 2 +- lib/MC/MCELFObjectTargetWriter.cpp | 21 + lib/MC/MCELFStreamer.cpp | 92 +- lib/MC/MCExpr.cpp | 4 + lib/MC/MCMachOStreamer.cpp | 4 +- lib/MC/MCObjectStreamer.cpp | 23 +- lib/MC/MCObjectWriter.cpp | 20 +- lib/MC/MCParser/AsmParser.cpp | 4 +- lib/MC/MCParser/ELFAsmParser.cpp | 14 +- lib/MC/MCSectionELF.cpp | 6 +- lib/MC/MCSubtargetInfo.cpp | 7 +- lib/MC/MCSymbol.cpp | 8 +- lib/MC/MachObjectWriter.cpp | 24 +- lib/MC/SubtargetFeature.cpp | 7 +- lib/MC/WinCOFFObjectWriter.cpp | 29 +- lib/MC/WinCOFFStreamer.cpp | 2 +- lib/Object/COFFObjectFile.cpp | 2 +- lib/Object/MachOObjectFile.cpp | 4 +- lib/Option/ArgList.cpp | 2 +- lib/Support/APInt.cpp | 19 +- lib/Support/CommandLine.cpp | 4 +- lib/Support/DataStream.cpp | 4 +- lib/Support/Debug.cpp | 6 +- lib/Support/FoldingSet.cpp | 4 +- lib/Support/GraphWriter.cpp | 25 +- lib/Support/Host.cpp | 194 ++- lib/Support/Process.cpp | 18 - lib/Support/Regex.cpp | 3 +- lib/Support/Triple.cpp | 4 +- lib/Support/Unix/Signals.inc | 4 +- lib/Support/Windows/Path.inc | 1 + lib/Support/Windows/Signals.inc | 2 +- lib/Support/Windows/TimeValue.inc | 1 + lib/Support/raw_ostream.cpp | 89 +- lib/TableGen/Record.cpp | 2 +- lib/TableGen/TGLexer.h | 3 +- lib/Target/AArch64/AArch64.td | 16 +- lib/Target/AArch64/AArch64AsmPrinter.cpp | 27 +- lib/Target/AArch64/AArch64CollectLOH.cpp | 2 +- lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp | 9 +- lib/Target/AArch64/AArch64FastISel.cpp | 16 +- lib/Target/AArch64/AArch64FrameLowering.cpp | 182 ++- lib/Target/AArch64/AArch64FrameLowering.h | 2 +- lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 2 +- lib/Target/AArch64/AArch64ISelLowering.cpp | 234 +++- lib/Target/AArch64/AArch64ISelLowering.h | 5 + lib/Target/AArch64/AArch64InstrFormats.td | 215 ++- lib/Target/AArch64/AArch64InstrInfo.cpp | 6 +- lib/Target/AArch64/AArch64InstrInfo.td | 54 +- lib/Target/AArch64/AArch64RegisterInfo.cpp | 30 + lib/Target/AArch64/AArch64RegisterInfo.h | 3 + lib/Target/AArch64/AArch64SchedA57.td | 20 +- lib/Target/AArch64/AArch64Subtarget.cpp | 3 +- lib/Target/AArch64/AArch64Subtarget.h | 6 +- lib/Target/AArch64/AArch64TargetMachine.cpp | 9 +- lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp | 32 +- .../AArch64/Disassembler/AArch64Disassembler.cpp | 13 +- .../AArch64/InstPrinter/AArch64InstPrinter.cpp | 96 +- .../AArch64/InstPrinter/AArch64InstPrinter.h | 131 +- .../AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 4 +- .../MCTargetDesc/AArch64ELFObjectWriter.cpp | 8 +- .../AArch64/MCTargetDesc/AArch64ELFStreamer.cpp | 10 +- .../AArch64/MCTargetDesc/AArch64ELFStreamer.h | 4 +- .../AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp | 2 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp | 15 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.h | 9 +- .../MCTargetDesc/AArch64MachObjectWriter.cpp | 6 +- lib/Target/AArch64/Utils/AArch64BaseInfo.cpp | 1429 ++++++++++---------- lib/Target/AArch64/Utils/AArch64BaseInfo.h | 78 +- lib/Target/ARM/ARM.td | 36 +- lib/Target/ARM/ARMAsmPrinter.cpp | 32 +- lib/Target/ARM/ARMBaseRegisterInfo.cpp | 21 +- lib/Target/ARM/ARMConstantPoolValue.h | 2 +- lib/Target/ARM/ARMFrameLowering.cpp | 3 +- lib/Target/ARM/ARMISelLowering.cpp | 27 +- lib/Target/ARM/ARMInstrInfo.td | 76 +- lib/Target/ARM/ARMInstrThumb.td | 18 +- lib/Target/ARM/ARMInstrThumb2.td | 43 +- lib/Target/ARM/ARMSubtarget.cpp | 6 +- lib/Target/ARM/ARMSubtarget.h | 8 +- lib/Target/ARM/ARMTargetMachine.cpp | 9 +- lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 63 +- lib/Target/ARM/Disassembler/ARMDisassembler.cpp | 56 +- lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp | 729 +++++----- lib/Target/ARM/InstPrinter/ARMInstPrinter.h | 223 +-- lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp | 16 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h | 2 + lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h | 2 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h | 2 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h | 2 +- lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp | 12 +- lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp | 22 +- lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp | 2 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp | 13 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h | 27 +- .../ARM/MCTargetDesc/ARMMachObjectWriter.cpp | 12 +- .../ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp | 5 +- lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp | 7 +- lib/Target/BPF/BPFISelDAGToDAG.cpp | 2 +- lib/Target/BPF/BPFISelLowering.cpp | 15 +- lib/Target/BPF/BPFInstrInfo.td | 57 +- lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp | 2 +- lib/Target/BPF/InstPrinter/BPFInstPrinter.h | 3 +- lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp | 4 +- lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp | 2 +- lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp | 5 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp | 11 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h | 3 +- lib/Target/BPF/MCTargetDesc/LLVMBuild.txt | 2 +- lib/Target/CppBackend/CPPBackend.cpp | 24 +- lib/Target/CppBackend/CPPTargetMachine.h | 2 +- lib/Target/Hexagon/CMakeLists.txt | 1 + lib/Target/Hexagon/HexagonAsmPrinter.cpp | 15 - lib/Target/Hexagon/HexagonExpandCondsets.cpp | 1348 ++++++++++++++++++ lib/Target/Hexagon/HexagonISelLowering.cpp | 11 +- lib/Target/Hexagon/HexagonInstrInfo.cpp | 5 +- lib/Target/Hexagon/HexagonInstrInfo.h | 4 +- lib/Target/Hexagon/HexagonSubtarget.cpp | 2 +- lib/Target/Hexagon/HexagonTargetMachine.cpp | 24 +- .../Hexagon/MCTargetDesc/HexagonAsmBackend.cpp | 2 +- .../MCTargetDesc/HexagonELFObjectWriter.cpp | 8 +- .../Hexagon/MCTargetDesc/HexagonInstPrinter.cpp | 5 +- .../Hexagon/MCTargetDesc/HexagonInstPrinter.h | 3 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp | 11 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.h | 5 +- .../MSP430/InstPrinter/MSP430InstPrinter.cpp | 2 +- lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h | 3 +- .../MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp | 5 +- lib/Target/MSP430/MSP430ISelLowering.cpp | 1 + lib/Target/MSP430/MSP430ISelLowering.h | 6 - lib/Target/MSP430/MSP430MCInstLower.cpp | 4 +- lib/Target/Mips/AsmParser/MipsAsmParser.cpp | 76 +- lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp | 2 +- lib/Target/Mips/InstPrinter/MipsInstPrinter.h | 3 +- lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp | 3 +- lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h | 2 +- .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 185 ++- lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp | 13 +- lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h | 9 +- lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h | 2 +- lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h | 5 +- lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h | 3 +- .../Mips/MCTargetDesc/MipsNaClELFStreamer.cpp | 8 +- .../Mips/MCTargetDesc/MipsTargetStreamer.cpp | 17 +- lib/Target/Mips/Mips16FrameLowering.cpp | 19 - lib/Target/Mips/Mips16FrameLowering.h | 4 - lib/Target/Mips/Mips16InstrInfo.cpp | 3 + lib/Target/Mips/Mips16InstrInfo.h | 2 +- lib/Target/Mips/MipsAsmPrinter.cpp | 31 +- lib/Target/Mips/MipsFastISel.cpp | 2 +- lib/Target/Mips/MipsFrameLowering.cpp | 17 + lib/Target/Mips/MipsFrameLowering.h | 5 + lib/Target/Mips/MipsISelLowering.cpp | 1 + lib/Target/Mips/MipsInstrInfo.h | 4 + lib/Target/Mips/MipsInstrInfo.td | 10 +- lib/Target/Mips/MipsMachineFunction.cpp | 22 +- lib/Target/Mips/MipsMachineFunction.h | 5 +- lib/Target/Mips/MipsOptionRecord.h | 2 +- lib/Target/Mips/MipsSEFrameLowering.cpp | 20 - lib/Target/Mips/MipsSEFrameLowering.h | 4 - lib/Target/Mips/MipsSEISelDAGToDAG.cpp | 15 +- lib/Target/Mips/MipsSEInstrInfo.cpp | 3 + lib/Target/Mips/MipsSEInstrInfo.h | 2 +- lib/Target/Mips/MipsTargetStreamer.h | 3 + lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp | 10 +- lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h | 5 +- .../NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp | 7 +- lib/Target/NVPTX/NVPTX.td | 22 +- lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 77 +- .../NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp | 5 +- lib/Target/NVPTX/NVPTXGenericToNVVM.cpp | 1 + lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 +- lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp | 14 +- lib/Target/NVPTX/NVPTXTargetMachine.h | 2 +- lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp | 70 + lib/Target/NVPTX/NVPTXTargetTransformInfo.h | 2 + lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp | 52 + .../PowerPC/Disassembler/PPCDisassembler.cpp | 2 +- lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp | 2 +- lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h | 3 +- lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp | 5 +- .../PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp | 2 +- .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp | 2 +- .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp | 8 +- lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h | 43 +- .../PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp | 4 +- lib/Target/PowerPC/PPC.td | 77 +- lib/Target/PowerPC/PPCAsmPrinter.cpp | 19 - lib/Target/PowerPC/PPCFastISel.cpp | 7 + lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 31 - lib/Target/PowerPC/PPCISelLowering.cpp | 150 +- lib/Target/PowerPC/PPCISelLowering.h | 17 +- lib/Target/PowerPC/PPCInstr64Bit.td | 36 +- lib/Target/PowerPC/PPCInstrFormats.td | 6 + lib/Target/PowerPC/PPCInstrInfo.td | 64 +- lib/Target/PowerPC/PPCInstrVSX.td | 25 + lib/Target/PowerPC/PPCLoopDataPrefetch.cpp | 9 +- lib/Target/PowerPC/PPCLoopPreIncPrep.cpp | 39 +- lib/Target/PowerPC/PPCMCInstLower.cpp | 2 +- lib/Target/PowerPC/PPCSubtarget.cpp | 9 +- lib/Target/PowerPC/PPCSubtarget.h | 6 + lib/Target/PowerPC/PPCTargetStreamer.h | 2 +- lib/Target/PowerPC/README.txt | 19 + lib/Target/PowerPC/README_ALTIVEC.txt | 2 +- lib/Target/R600/AMDGPU.td | 24 +- lib/Target/R600/AMDGPUAsmPrinter.cpp | 22 + lib/Target/R600/AMDGPUAsmPrinter.h | 4 + lib/Target/R600/AMDGPUISelDAGToDAG.cpp | 2 +- lib/Target/R600/AMDGPUISelLowering.cpp | 12 +- lib/Target/R600/AMDGPUInstructions.td | 4 +- lib/Target/R600/AMDGPUMCInstLower.cpp | 3 +- lib/Target/R600/AMDGPUPromoteAlloca.cpp | 4 +- lib/Target/R600/AMDGPUSubtarget.cpp | 1 + lib/Target/R600/AMDGPUSubtarget.h | 3 + lib/Target/R600/AMDILCFGStructurizer.cpp | 2 +- lib/Target/R600/AsmParser/AMDGPUAsmParser.cpp | 1094 ++++++++++++++- lib/Target/R600/InstPrinter/AMDGPUInstPrinter.cpp | 19 +- lib/Target/R600/InstPrinter/AMDGPUInstPrinter.h | 5 +- lib/Target/R600/MCTargetDesc/AMDGPUAsmBackend.cpp | 4 +- .../R600/MCTargetDesc/AMDGPUELFObjectWriter.cpp | 2 +- .../R600/MCTargetDesc/AMDGPUMCTargetDesc.cpp | 5 +- lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.h | 3 +- lib/Target/R600/MCTargetDesc/SIMCCodeEmitter.cpp | 2 +- lib/Target/R600/R600ISelLowering.cpp | 4 +- lib/Target/R600/R600Instructions.td | 5 + lib/Target/R600/R600TextureIntrinsicsReplacer.cpp | 4 +- lib/Target/R600/SIAnnotateControlFlow.cpp | 19 +- lib/Target/R600/SIISelLowering.cpp | 37 +- lib/Target/R600/SIISelLowering.h | 4 + lib/Target/R600/SIInstrFormats.td | 31 +- lib/Target/R600/SIInstrInfo.td | 274 +++- lib/Target/R600/SIInstructions.td | 43 +- lib/Target/R600/SIRegisterInfo.td | 47 +- lib/Target/R600/SITypeRewriter.cpp | 2 +- lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp | 46 +- lib/Target/Sparc/InstPrinter/SparcInstPrinter.h | 40 +- lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp | 2 +- .../Sparc/MCTargetDesc/SparcELFObjectWriter.cpp | 8 +- .../Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp | 2 +- .../Sparc/MCTargetDesc/SparcMCTargetDesc.cpp | 13 +- lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h | 4 +- lib/Target/Sparc/Sparc.td | 7 + lib/Target/Sparc/SparcAsmPrinter.cpp | 18 - lib/Target/Sparc/SparcISelLowering.cpp | 3 +- lib/Target/Sparc/SparcSelectionDAGInfo.h | 2 +- lib/Target/SystemZ/CMakeLists.txt | 1 + .../SystemZ/Disassembler/SystemZDisassembler.cpp | 2 +- .../SystemZ/InstPrinter/SystemZInstPrinter.cpp | 3 +- .../SystemZ/InstPrinter/SystemZInstPrinter.h | 3 +- lib/Target/SystemZ/LLVMBuild.txt | 2 +- .../SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp | 2 +- .../SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp | 2 +- .../SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp | 4 +- .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp | 5 +- .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.h | 3 +- lib/Target/SystemZ/SystemZ.h | 12 + lib/Target/SystemZ/SystemZAsmPrinter.cpp | 23 - lib/Target/SystemZ/SystemZAsmPrinter.h | 1 - lib/Target/SystemZ/SystemZISelDAGToDAG.cpp | 11 +- lib/Target/SystemZ/SystemZISelLowering.cpp | 250 +++- lib/Target/SystemZ/SystemZISelLowering.h | 20 + lib/Target/SystemZ/SystemZInstrFormats.td | 11 + lib/Target/SystemZ/SystemZInstrInfo.cpp | 34 +- lib/Target/SystemZ/SystemZInstrInfo.td | 65 + lib/Target/SystemZ/SystemZOperators.td | 13 + lib/Target/SystemZ/SystemZProcessors.td | 30 +- lib/Target/SystemZ/SystemZSubtarget.cpp | 9 +- lib/Target/SystemZ/SystemZSubtarget.h | 18 + lib/Target/SystemZ/SystemZTargetMachine.cpp | 7 + lib/Target/SystemZ/SystemZTargetMachine.h | 1 + lib/Target/SystemZ/SystemZTargetTransformInfo.cpp | 240 ++++ lib/Target/SystemZ/SystemZTargetTransformInfo.h | 70 + lib/Target/TargetLoweringObjectFile.cpp | 2 +- lib/Target/TargetMachine.cpp | 2 +- lib/Target/TargetMachineC.cpp | 20 +- lib/Target/X86/AsmParser/X86AsmParser.cpp | 2 +- lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp | 49 +- lib/Target/X86/InstPrinter/X86ATTInstPrinter.h | 11 +- lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp | 3 +- lib/Target/X86/InstPrinter/X86IntelInstPrinter.h | 3 +- lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 12 +- lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp | 8 +- lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp | 2 +- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp | 9 +- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h | 34 +- .../X86/MCTargetDesc/X86MachObjectWriter.cpp | 5 +- .../X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp | 4 +- lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp | 8 +- lib/Target/X86/X86.td | 20 +- lib/Target/X86/X86AsmPrinter.cpp | 23 +- lib/Target/X86/X86FastISel.cpp | 2 + lib/Target/X86/X86ISelDAGToDAG.cpp | 7 +- lib/Target/X86/X86ISelLowering.cpp | 138 +- lib/Target/X86/X86ISelLowering.h | 3 + lib/Target/X86/X86InstrAVX512.td | 155 +-- lib/Target/X86/X86InstrArithmetic.td | 8 +- lib/Target/X86/X86InstrCompiler.td | 11 + lib/Target/X86/X86InstrFragmentsSIMD.td | 28 +- lib/Target/X86/X86InstrInfo.cpp | 32 + lib/Target/X86/X86InstrSSE.td | 63 +- lib/Target/X86/X86IntrinsicsInfo.h | 14 + lib/Target/X86/X86SelectionDAGInfo.cpp | 5 +- lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp | 2 +- lib/Target/XCore/InstPrinter/XCoreInstPrinter.h | 3 +- .../XCore/MCTargetDesc/XCoreMCTargetDesc.cpp | 5 +- lib/Target/XCore/XCoreISelLowering.cpp | 8 +- lib/Target/XCore/XCoreLowerThreadLocal.cpp | 8 +- lib/Target/XCore/XCoreTargetStreamer.h | 2 +- lib/Transforms/IPO/ArgumentPromotion.cpp | 22 +- lib/Transforms/IPO/DeadArgumentElimination.cpp | 10 +- lib/Transforms/IPO/GlobalOpt.cpp | 15 +- lib/Transforms/IPO/LowerBitSets.cpp | 16 +- lib/Transforms/IPO/PassManagerBuilder.cpp | 7 + lib/Transforms/IPO/StripSymbols.cpp | 26 +- lib/Transforms/InstCombine/InstCombineCalls.cpp | 155 +-- lib/Transforms/InstCombine/InstCombineCasts.cpp | 36 - lib/Transforms/InstCombine/InstCombineCompares.cpp | 148 +- lib/Transforms/InstCombine/InstCombineInternal.h | 58 +- .../InstCombine/InstCombineLoadStoreAlloca.cpp | 8 +- .../InstCombine/InstCombineVectorOps.cpp | 5 +- .../InstCombine/InstructionCombining.cpp | 49 +- .../Instrumentation/AddressSanitizer.cpp | 92 +- .../Instrumentation/DataFlowSanitizer.cpp | 34 +- lib/Transforms/Instrumentation/GCOVProfiling.cpp | 71 +- lib/Transforms/Instrumentation/MemorySanitizer.cpp | 10 +- .../Instrumentation/SanitizerCoverage.cpp | 38 +- lib/Transforms/Instrumentation/ThreadSanitizer.cpp | 72 +- lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h | 2 - lib/Transforms/ObjCARC/DependencyAnalysis.cpp | 4 +- lib/Transforms/Scalar/AlignmentFromAssumptions.cpp | 4 +- lib/Transforms/Scalar/Android.mk | 2 + lib/Transforms/Scalar/CMakeLists.txt | 2 + lib/Transforms/Scalar/DeadStoreElimination.cpp | 8 +- lib/Transforms/Scalar/Float2Int.cpp | 540 ++++++++ lib/Transforms/Scalar/GVN.cpp | 6 +- lib/Transforms/Scalar/IndVarSimplify.cpp | 64 +- lib/Transforms/Scalar/LoadCombine.cpp | 2 +- lib/Transforms/Scalar/LoopStrengthReduce.cpp | 4 +- lib/Transforms/Scalar/LoopUnrollPass.cpp | 5 +- lib/Transforms/Scalar/MemCpyOptimizer.cpp | 2 +- lib/Transforms/Scalar/NaryReassociate.cpp | 252 ++++ lib/Transforms/Scalar/PlaceSafepoints.cpp | 5 +- lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 1237 ++++++++++------- lib/Transforms/Scalar/SCCP.cpp | 3 +- lib/Transforms/Scalar/SROA.cpp | 30 +- lib/Transforms/Scalar/SampleProfile.cpp | 11 +- lib/Transforms/Scalar/Scalar.cpp | 2 + lib/Transforms/Scalar/ScalarReplAggregates.cpp | 9 +- lib/Transforms/Scalar/Scalarizer.cpp | 2 +- .../Scalar/SeparateConstOffsetFromGEP.cpp | 6 +- .../Scalar/StraightLineStrengthReduce.cpp | 338 +++-- lib/Transforms/Scalar/StructurizeCFG.cpp | 2 +- lib/Transforms/Utils/AddDiscriminators.cpp | 43 +- lib/Transforms/Utils/CloneFunction.cpp | 25 +- lib/Transforms/Utils/GlobalStatus.cpp | 2 +- lib/Transforms/Utils/InlineFunction.cpp | 28 +- lib/Transforms/Utils/Local.cpp | 45 +- lib/Transforms/Utils/LoopUnroll.cpp | 15 +- lib/Transforms/Utils/LoopUnrollRuntime.cpp | 17 +- lib/Transforms/Utils/ModuleUtils.cpp | 11 + lib/Transforms/Utils/PromoteMemoryToRegister.cpp | 9 +- lib/Transforms/Utils/SimplifyCFG.cpp | 6 +- lib/Transforms/Utils/SimplifyLibCalls.cpp | 72 +- lib/Transforms/Vectorize/LoopVectorize.cpp | 59 +- lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 +- 563 files changed, 17617 insertions(+), 8940 deletions(-) create mode 100644 lib/Analysis/DivergenceAnalysis.cpp delete mode 100644 lib/ExecutionEngine/EventListenerCommon.h create mode 100644 lib/ExecutionEngine/Orc/ExecutionUtils.cpp create mode 100644 lib/Fuzzer/FuzzerDFSan.cpp create mode 100644 lib/Fuzzer/cxx_fuzzer_tokens.txt create mode 100644 lib/Fuzzer/dfsan_fuzzer_abi.list create mode 100644 lib/Fuzzer/test/CxxTokensTest.cpp create mode 100644 lib/Fuzzer/test/dfsan/CMakeLists.txt create mode 100644 lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp delete mode 100644 lib/IR/UseListOrder.cpp create mode 100644 lib/Target/Hexagon/HexagonExpandCondsets.cpp create mode 100644 lib/Target/SystemZ/SystemZTargetTransformInfo.cpp create mode 100644 lib/Target/SystemZ/SystemZTargetTransformInfo.h create mode 100644 lib/Transforms/Scalar/Float2Int.cpp create mode 100644 lib/Transforms/Scalar/NaryReassociate.cpp (limited to 'lib') diff --git a/lib/Analysis/AliasAnalysis.cpp b/lib/Analysis/AliasAnalysis.cpp index 0b0fd50..43db176 100644 --- a/lib/Analysis/AliasAnalysis.cpp +++ b/lib/Analysis/AliasAnalysis.cpp @@ -82,6 +82,23 @@ void AliasAnalysis::addEscapingUse(Use &U) { AA->addEscapingUse(U); } +AliasAnalysis::ModRefResult +AliasAnalysis::getModRefInfo(Instruction *I, ImmutableCallSite Call) { + // We may have two calls + if (auto CS = ImmutableCallSite(I)) { + // Check if the two calls modify the same memory + return getModRefInfo(Call, CS); + } else { + // Otherwise, check if the call modifies or references the + // location this memory access defines. The best we can say + // is that if the call references what this instruction + // defines, it must be clobbered by this location. + const AliasAnalysis::Location DefLoc = AA->getLocation(I); + if (getModRefInfo(Call, DefLoc) != AliasAnalysis::NoModRef) + return AliasAnalysis::ModRef; + } + return AliasAnalysis::NoModRef; +} AliasAnalysis::ModRefResult AliasAnalysis::getModRefInfo(ImmutableCallSite CS, @@ -330,7 +347,7 @@ AliasAnalysis::getModRefInfo(const LoadInst *L, const Location &Loc) { // If the load address doesn't alias the given address, it doesn't read // or write the specified memory. - if (!alias(getLocation(L), Loc)) + if (Loc.Ptr && !alias(getLocation(L), Loc)) return NoModRef; // Otherwise, a load just reads. @@ -343,15 +360,18 @@ AliasAnalysis::getModRefInfo(const StoreInst *S, const Location &Loc) { if (!S->isUnordered()) return ModRef; - // If the store address cannot alias the pointer in question, then the - // specified memory cannot be modified by the store. - if (!alias(getLocation(S), Loc)) - return NoModRef; + if (Loc.Ptr) { + // If the store address cannot alias the pointer in question, then the + // specified memory cannot be modified by the store. + if (!alias(getLocation(S), Loc)) + return NoModRef; - // If the pointer is a pointer to constant memory, then it could not have been - // modified by this store. - if (pointsToConstantMemory(Loc)) - return NoModRef; + // If the pointer is a pointer to constant memory, then it could not have + // been modified by this store. + if (pointsToConstantMemory(Loc)) + return NoModRef; + + } // Otherwise, a store just writes. return Mod; diff --git a/lib/Analysis/AliasAnalysisCounter.cpp b/lib/Analysis/AliasAnalysisCounter.cpp index 5865259..a1bfba1 100644 --- a/lib/Analysis/AliasAnalysisCounter.cpp +++ b/lib/Analysis/AliasAnalysisCounter.cpp @@ -44,7 +44,7 @@ namespace { errs() << " " << Val << " " << Desc << " responses (" << Val*100/Sum << "%)\n"; } - ~AliasAnalysisCounter() { + ~AliasAnalysisCounter() override { unsigned AASum = No+May+Partial+Must; unsigned MRSum = NoMR+JustRef+JustMod+MR; if (AASum + MRSum) { // Print a report if any counted queries occurred... diff --git a/lib/Analysis/AliasAnalysisEvaluator.cpp b/lib/Analysis/AliasAnalysisEvaluator.cpp index fe4bd4c..273eacc 100644 --- a/lib/Analysis/AliasAnalysisEvaluator.cpp +++ b/lib/Analysis/AliasAnalysisEvaluator.cpp @@ -158,7 +158,7 @@ bool AAEval::runOnFunction(Function &F) { if (EvalAAMD && isa(&*I)) Stores.insert(&*I); Instruction &Inst = *I; - if (CallSite CS = cast(&Inst)) { + if (auto CS = CallSite(&Inst)) { Value *Callee = CS.getCalledValue(); // Skip actual functions for direct function calls. if (!isa(Callee) && isInterestingPointer(Callee)) diff --git a/lib/Analysis/AliasSetTracker.cpp b/lib/Analysis/AliasSetTracker.cpp index 45442b0..4c79b9f 100644 --- a/lib/Analysis/AliasSetTracker.cpp +++ b/lib/Analysis/AliasSetTracker.cpp @@ -187,7 +187,7 @@ bool AliasSet::aliasesUnknownInst(Instruction *Inst, AliasAnalysis &AA) const { return false; for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i) { - CallSite C1 = getUnknownInst(i), C2 = Inst; + CallSite C1(getUnknownInst(i)), C2(Inst); if (!C1 || !C2 || AA.getModRefInfo(C1, C2) != AliasAnalysis::NoModRef || AA.getModRefInfo(C2, C1) != AliasAnalysis::NoModRef) diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp index 4549c1e..842ff0a 100644 --- a/lib/Analysis/Analysis.cpp +++ b/lib/Analysis/Analysis.cpp @@ -37,6 +37,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) { initializeCFLAliasAnalysisPass(Registry); initializeDependenceAnalysisPass(Registry); initializeDelinearizationPass(Registry); + initializeDivergenceAnalysisPass(Registry); initializeDominanceFrontierPass(Registry); initializeDomViewerPass(Registry); initializeDomPrinterPass(Registry); diff --git a/lib/Analysis/Android.mk b/lib/Analysis/Android.mk index 277956c..94a1d2e 100644 --- a/lib/Analysis/Android.mk +++ b/lib/Analysis/Android.mk @@ -22,6 +22,7 @@ analysis_SRC_FILES := \ CostModel.cpp \ Delinearization.cpp \ DependenceAnalysis.cpp \ + DivergenceAnalysis.cpp \ DomPrinter.cpp \ DominanceFrontier.cpp \ IVUsers.cpp \ diff --git a/lib/Analysis/BasicAliasAnalysis.cpp b/lib/Analysis/BasicAliasAnalysis.cpp index be2282f..2767e41 100644 --- a/lib/Analysis/BasicAliasAnalysis.cpp +++ b/lib/Analysis/BasicAliasAnalysis.cpp @@ -932,14 +932,14 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size, // Also, check that they all index through arrays. for (unsigned i = 1, e = GEP1->getNumIndices() - 1; i != e; ++i) { if (!isa(GetElementPtrInst::getIndexedType( - GEP1->getPointerOperandType(), IntermediateIndices))) + GEP1->getSourceElementType(), IntermediateIndices))) return AliasAnalysis::MayAlias; IntermediateIndices.push_back(GEP1->getOperand(i + 1)); } StructType *LastIndexedStruct = dyn_cast(GetElementPtrInst::getIndexedType( - GEP1->getPointerOperandType(), IntermediateIndices)); + GEP1->getSourceElementType(), IntermediateIndices)); if (!LastIndexedStruct) return AliasAnalysis::MayAlias; diff --git a/lib/Analysis/BlockFrequencyInfo.cpp b/lib/Analysis/BlockFrequencyInfo.cpp index 37f2fae..3d819eb 100644 --- a/lib/Analysis/BlockFrequencyInfo.cpp +++ b/lib/Analysis/BlockFrequencyInfo.cpp @@ -85,7 +85,7 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { std::string Result; raw_string_ostream OS(Result); - OS << Node->getName().str() << ":"; + OS << Node->getName() << ":"; switch (ViewBlockFreqPropagationDAG) { case GVDT_Fraction: Graph->printBlockFreq(OS, Node); diff --git a/lib/Analysis/BlockFrequencyInfoImpl.cpp b/lib/Analysis/BlockFrequencyInfoImpl.cpp index 278073c..456cee1 100644 --- a/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/lib/Analysis/BlockFrequencyInfoImpl.cpp @@ -331,32 +331,35 @@ bool BlockFrequencyInfoImplBase::addLoopSuccessorsToDist( return true; } -/// \brief Get the maximum allowed loop scale. -/// -/// Gives the maximum number of estimated iterations allowed for a loop. Very -/// large numbers cause problems downstream (even within 64-bits). -static Scaled64 getMaxLoopScale() { return Scaled64(1, 12); } - /// \brief Compute the loop scale for a loop. void BlockFrequencyInfoImplBase::computeLoopScale(LoopData &Loop) { // Compute loop scale. DEBUG(dbgs() << "compute-loop-scale: " << getLoopName(Loop) << "\n"); + // Infinite loops need special handling. If we give the back edge an infinite + // mass, they may saturate all the other scales in the function down to 1, + // making all the other region temperatures look exactly the same. Choose an + // arbitrary scale to avoid these issues. + // + // FIXME: An alternate way would be to select a symbolic scale which is later + // replaced to be the maximum of all computed scales plus 1. This would + // appropriately describe the loop as having a large scale, without skewing + // the final frequency computation. + const Scaled64 InifiniteLoopScale(1, 12); + // LoopScale == 1 / ExitMass // ExitMass == HeadMass - BackedgeMass BlockMass ExitMass = BlockMass::getFull() - Loop.BackedgeMass; - // Block scale stores the inverse of the scale. - Loop.Scale = ExitMass.toScaled().inverse(); + // Block scale stores the inverse of the scale. If this is an infinite loop, + // its exit mass will be zero. In this case, use an arbitrary scale for the + // loop scale. + Loop.Scale = + ExitMass.isEmpty() ? InifiniteLoopScale : ExitMass.toScaled().inverse(); DEBUG(dbgs() << " - exit-mass = " << ExitMass << " (" << BlockMass::getFull() << " - " << Loop.BackedgeMass << ")\n" << " - scale = " << Loop.Scale << "\n"); - - if (Loop.Scale > getMaxLoopScale()) { - Loop.Scale = getMaxLoopScale(); - DEBUG(dbgs() << " - reduced-to-max-scale: " << getMaxLoopScale() << "\n"); - } } /// \brief Package up a loop. @@ -424,15 +427,24 @@ static void convertFloatingToInteger(BlockFrequencyInfoImplBase &BFI, const Scaled64 &Min, const Scaled64 &Max) { // Scale the Factor to a size that creates integers. Ideally, integers would // be scaled so that Max == UINT64_MAX so that they can be best - // differentiated. However, the register allocator currently deals poorly - // with large numbers. Instead, push Min up a little from 1 to give some - // room to differentiate small, unequal numbers. - // - // TODO: fix issues downstream so that ScalingFactor can be - // Scaled64(1,64)/Max. - Scaled64 ScalingFactor = Min.inverse(); - if ((Max / Min).lg() < 60) + // differentiated. However, in the presence of large frequency values, small + // frequencies are scaled down to 1, making it impossible to differentiate + // small, unequal numbers. When the spread between Min and Max frequencies + // fits well within MaxBits, we make the scale be at least 8. + const unsigned MaxBits = 64; + const unsigned SpreadBits = (Max / Min).lg(); + Scaled64 ScalingFactor; + if (SpreadBits <= MaxBits - 3) { + // If the values are small enough, make the scaling factor at least 8 to + // allow distinguishing small values. + ScalingFactor = Min.inverse(); ScalingFactor <<= 3; + } else { + // If the values need more than MaxBits to be represented, saturate small + // frequency values down to 1 by using a scaling factor that benefits large + // frequency values. + ScalingFactor = Scaled64(1, MaxBits) / Max; + } // Translate the floats to integers. DEBUG(dbgs() << "float-to-int: min = " << Min << ", max = " << Max diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp index 14800f4..8799a71 100644 --- a/lib/Analysis/BranchProbabilityInfo.cpp +++ b/lib/Analysis/BranchProbabilityInfo.cpp @@ -379,6 +379,14 @@ bool BranchProbabilityInfo::calcZeroHeuristics(BasicBlock *BB) { if (!CV) return false; + // If the LHS is the result of AND'ing a value with a single bit bitmask, + // we don't have information about probabilities. + if (Instruction *LHS = dyn_cast(CI->getOperand(0))) + if (LHS->getOpcode() == Instruction::And) + if (ConstantInt *AndRHS = dyn_cast(LHS->getOperand(1))) + if (AndRHS->getUniqueInteger().isPowerOf2()) + return false; + bool isProb; if (CV->isZero()) { switch (CI->getPredicate()) { @@ -499,25 +507,23 @@ bool BranchProbabilityInfo::runOnFunction(Function &F) { // Walk the basic blocks in post-order so that we can build up state about // the successors of a block iteratively. - for (po_iterator I = po_begin(&F.getEntryBlock()), - E = po_end(&F.getEntryBlock()); - I != E; ++I) { - DEBUG(dbgs() << "Computing probabilities for " << I->getName() << "\n"); - if (calcUnreachableHeuristics(*I)) + for (auto BB : post_order(&F.getEntryBlock())) { + DEBUG(dbgs() << "Computing probabilities for " << BB->getName() << "\n"); + if (calcUnreachableHeuristics(BB)) continue; - if (calcMetadataWeights(*I)) + if (calcMetadataWeights(BB)) continue; - if (calcColdCallHeuristics(*I)) + if (calcColdCallHeuristics(BB)) continue; - if (calcLoopBranchHeuristics(*I)) + if (calcLoopBranchHeuristics(BB)) continue; - if (calcPointerHeuristics(*I)) + if (calcPointerHeuristics(BB)) continue; - if (calcZeroHeuristics(*I)) + if (calcZeroHeuristics(BB)) continue; - if (calcFloatingPointHeuristics(*I)) + if (calcFloatingPointHeuristics(BB)) continue; - calcInvokeHeuristics(*I); + calcInvokeHeuristics(BB); } PostDominatedByUnreachable.clear(); diff --git a/lib/Analysis/CFGPrinter.cpp b/lib/Analysis/CFGPrinter.cpp index 89787f82..c86f1f5 100644 --- a/lib/Analysis/CFGPrinter.cpp +++ b/lib/Analysis/CFGPrinter.cpp @@ -77,7 +77,7 @@ namespace { } bool runOnFunction(Function &F) override { - std::string Filename = "cfg." + F.getName().str() + ".dot"; + std::string Filename = ("cfg." + F.getName() + ".dot").str(); errs() << "Writing '" << Filename << "'..."; std::error_code EC; @@ -111,7 +111,7 @@ namespace { } bool runOnFunction(Function &F) override { - std::string Filename = "cfg." + F.getName().str() + ".dot"; + std::string Filename = ("cfg." + F.getName() + ".dot").str(); errs() << "Writing '" << Filename << "'..."; std::error_code EC; diff --git a/lib/Analysis/CFLAliasAnalysis.cpp b/lib/Analysis/CFLAliasAnalysis.cpp index 53d748d..3147992 100644 --- a/lib/Analysis/CFLAliasAnalysis.cpp +++ b/lib/Analysis/CFLAliasAnalysis.cpp @@ -161,7 +161,7 @@ struct FunctionHandle : public CallbackVH { assert(CFLAA != nullptr); } - virtual ~FunctionHandle() {} + ~FunctionHandle() override {} void deleted() override { removeSelfFromCache(); } void allUsesReplacedWith(Value *) override { removeSelfFromCache(); } @@ -189,7 +189,7 @@ public: initializeCFLAliasAnalysisPass(*PassRegistry::getPassRegistry()); } - virtual ~CFLAliasAnalysis() {} + ~CFLAliasAnalysis() override {} void getAnalysisUsage(AnalysisUsage &AU) const override { AliasAnalysis::getAnalysisUsage(AU); diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt index ae40321..1335a6d 100644 --- a/lib/Analysis/CMakeLists.txt +++ b/lib/Analysis/CMakeLists.txt @@ -20,6 +20,7 @@ add_llvm_library(LLVMAnalysis ConstantFolding.cpp Delinearization.cpp DependenceAnalysis.cpp + DivergenceAnalysis.cpp DomPrinter.cpp DominanceFrontier.cpp IVUsers.cpp diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp index 995465d..a85e813 100644 --- a/lib/Analysis/ConstantFolding.cpp +++ b/lib/Analysis/ConstantFolding.cpp @@ -671,8 +671,8 @@ static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, /// If array indices are not pointer-sized integers, explicitly cast them so /// that they aren't implicitly casted by the getelementptr. -static Constant *CastGEPIndices(ArrayRef Ops, Type *ResultTy, - const DataLayout &DL, +static Constant *CastGEPIndices(Type *SrcTy, ArrayRef Ops, + Type *ResultTy, const DataLayout &DL, const TargetLibraryInfo *TLI) { Type *IntPtrTy = DL.getIntPtrType(ResultTy); @@ -681,8 +681,9 @@ static Constant *CastGEPIndices(ArrayRef Ops, Type *ResultTy, for (unsigned i = 1, e = Ops.size(); i != e; ++i) { if ((i == 1 || !isa(GetElementPtrInst::getIndexedType( - Ops[0]->getType(), - Ops.slice(1, i - 1)))) && + cast(Ops[0]->getType()->getScalarType()) + ->getElementType(), + Ops.slice(1, i - 1)))) && Ops[i]->getType() != IntPtrTy) { Any = true; NewIdxs.push_back(ConstantExpr::getCast(CastInst::getCastOpcode(Ops[i], @@ -697,7 +698,7 @@ static Constant *CastGEPIndices(ArrayRef Ops, Type *ResultTy, if (!Any) return nullptr; - Constant *C = ConstantExpr::getGetElementPtr(Ops[0], NewIdxs); + Constant *C = ConstantExpr::getGetElementPtr(SrcTy, Ops[0], NewIdxs); if (ConstantExpr *CE = dyn_cast(C)) { if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI)) C = Folded; @@ -723,7 +724,7 @@ static Constant* StripPtrCastKeepAS(Constant* Ptr) { } /// If we can symbolically evaluate the GEP constant expression, do so. -static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, +static Constant *SymbolicallyEvaluateGEP(Type *SrcTy, ArrayRef Ops, Type *ResultTy, const DataLayout &DL, const TargetLibraryInfo *TLI) { Constant *Ptr = Ops[0]; @@ -865,7 +866,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef Ops, return nullptr; // Create a GEP. - Constant *C = ConstantExpr::getGetElementPtr(Ptr, NewIdxs); + Constant *C = ConstantExpr::getGetElementPtr(SrcTy, Ptr, NewIdxs); assert(C->getType()->getPointerElementType() == Ty && "Computed GetElementPtr has unexpected type!"); @@ -1085,13 +1086,15 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]); case Instruction::ShuffleVector: return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]); - case Instruction::GetElementPtr: - if (Constant *C = CastGEPIndices(Ops, DestTy, DL, TLI)) + case Instruction::GetElementPtr: { + Type *SrcTy = nullptr; + if (Constant *C = CastGEPIndices(SrcTy, Ops, DestTy, DL, TLI)) return C; - if (Constant *C = SymbolicallyEvaluateGEP(Ops, DestTy, DL, TLI)) + if (Constant *C = SymbolicallyEvaluateGEP(SrcTy, Ops, DestTy, DL, TLI)) return C; - return ConstantExpr::getGetElementPtr(Ops[0], Ops.slice(1)); + return ConstantExpr::getGetElementPtr(SrcTy, Ops[0], Ops.slice(1)); + } } } diff --git a/lib/Analysis/DivergenceAnalysis.cpp b/lib/Analysis/DivergenceAnalysis.cpp new file mode 100644 index 0000000..e5ee295 --- /dev/null +++ b/lib/Analysis/DivergenceAnalysis.cpp @@ -0,0 +1,337 @@ +//===- DivergenceAnalysis.cpp ------ Divergence Analysis ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines divergence analysis which determines whether a branch in a +// GPU program is divergent. It can help branch optimizations such as jump +// threading and loop unswitching to make better decisions. +// +// GPU programs typically use the SIMD execution model, where multiple threads +// in the same execution group have to execute in lock-step. Therefore, if the +// code contains divergent branches (i.e., threads in a group do not agree on +// which path of the branch to take), the group of threads has to execute all +// the paths from that branch with different subsets of threads enabled until +// they converge at the immediately post-dominating BB of the paths. +// +// Due to this execution model, some optimizations such as jump +// threading and loop unswitching can be unfortunately harmful when performed on +// divergent branches. Therefore, an analysis that computes which branches in a +// GPU program are divergent can help the compiler to selectively run these +// optimizations. +// +// This file defines divergence analysis which computes a conservative but +// non-trivial approximation of all divergent branches in a GPU program. It +// partially implements the approach described in +// +// Divergence Analysis +// Sampaio, Souza, Collange, Pereira +// TOPLAS '13 +// +// The divergence analysis identifies the sources of divergence (e.g., special +// variables that hold the thread ID), and recursively marks variables that are +// data or sync dependent on a source of divergence as divergent. +// +// While data dependency is a well-known concept, the notion of sync dependency +// is worth more explanation. Sync dependence characterizes the control flow +// aspect of the propagation of branch divergence. For example, +// +// %cond = icmp slt i32 %tid, 10 +// br i1 %cond, label %then, label %else +// then: +// br label %merge +// else: +// br label %merge +// merge: +// %a = phi i32 [ 0, %then ], [ 1, %else ] +// +// Suppose %tid holds the thread ID. Although %a is not data dependent on %tid +// because %tid is not on its use-def chains, %a is sync dependent on %tid +// because the branch "br i1 %cond" depends on %tid and affects which value %a +// is assigned to. +// +// The current implementation has the following limitations: +// 1. intra-procedural. It conservatively considers the arguments of a +// non-kernel-entry function and the return value of a function call as +// divergent. +// 2. memory as black box. It conservatively considers values loaded from +// generic or local address as divergent. This can be improved by leveraging +// pointer analysis. +//===----------------------------------------------------------------------===// + +#include +#include "llvm/IR/Dominators.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/Analysis/Passes.h" +#include "llvm/Analysis/PostDominators.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Value.h" +#include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Scalar.h" +using namespace llvm; + +#define DEBUG_TYPE "divergence" + +namespace { +class DivergenceAnalysis : public FunctionPass { +public: + static char ID; + + DivergenceAnalysis() : FunctionPass(ID) { + initializeDivergenceAnalysisPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + AU.setPreservesAll(); + } + + bool runOnFunction(Function &F) override; + + // Print all divergent branches in the function. + void print(raw_ostream &OS, const Module *) const override; + + // Returns true if V is divergent. + bool isDivergent(const Value *V) const { return DivergentValues.count(V); } + // Returns true if V is uniform/non-divergent. + bool isUniform(const Value *V) const { return !isDivergent(V); } + +private: + // Stores all divergent values. + DenseSet DivergentValues; +}; +} // End of anonymous namespace + +// Register this pass. +char DivergenceAnalysis::ID = 0; +INITIALIZE_PASS_BEGIN(DivergenceAnalysis, "divergence", "Divergence Analysis", + false, true) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(PostDominatorTree) +INITIALIZE_PASS_END(DivergenceAnalysis, "divergence", "Divergence Analysis", + false, true) + +namespace { + +class DivergencePropagator { +public: + DivergencePropagator(Function &F, TargetTransformInfo &TTI, + DominatorTree &DT, PostDominatorTree &PDT, + DenseSet &DV) + : F(F), TTI(TTI), DT(DT), PDT(PDT), DV(DV) {} + void populateWithSourcesOfDivergence(); + void propagate(); + +private: + // A helper function that explores data dependents of V. + void exploreDataDependency(Value *V); + // A helper function that explores sync dependents of TI. + void exploreSyncDependency(TerminatorInst *TI); + // Computes the influence region from Start to End. This region includes all + // basic blocks on any path from Start to End. + void computeInfluenceRegion(BasicBlock *Start, BasicBlock *End, + DenseSet &InfluenceRegion); + // Finds all users of I that are outside the influence region, and add these + // users to Worklist. + void findUsersOutsideInfluenceRegion( + Instruction &I, const DenseSet &InfluenceRegion); + + Function &F; + TargetTransformInfo &TTI; + DominatorTree &DT; + PostDominatorTree &PDT; + std::vector Worklist; // Stack for DFS. + DenseSet &DV; // Stores all divergent values. +}; + +void DivergencePropagator::populateWithSourcesOfDivergence() { + Worklist.clear(); + DV.clear(); + for (auto &I : inst_range(F)) { + if (TTI.isSourceOfDivergence(&I)) { + Worklist.push_back(&I); + DV.insert(&I); + } + } + for (auto &Arg : F.args()) { + if (TTI.isSourceOfDivergence(&Arg)) { + Worklist.push_back(&Arg); + DV.insert(&Arg); + } + } +} + +void DivergencePropagator::exploreSyncDependency(TerminatorInst *TI) { + // Propagation rule 1: if branch TI is divergent, all PHINodes in TI's + // immediate post dominator are divergent. This rule handles if-then-else + // patterns. For example, + // + // if (tid < 5) + // a1 = 1; + // else + // a2 = 2; + // a = phi(a1, a2); // sync dependent on (tid < 5) + BasicBlock *ThisBB = TI->getParent(); + BasicBlock *IPostDom = PDT.getNode(ThisBB)->getIDom()->getBlock(); + if (IPostDom == nullptr) + return; + + for (auto I = IPostDom->begin(); isa(I); ++I) { + // A PHINode is uniform if it returns the same value no matter which path is + // taken. + if (!cast(I)->hasConstantValue() && DV.insert(I).second) + Worklist.push_back(I); + } + + // Propagation rule 2: if a value defined in a loop is used outside, the user + // is sync dependent on the condition of the loop exits that dominate the + // user. For example, + // + // int i = 0; + // do { + // i++; + // if (foo(i)) ... // uniform + // } while (i < tid); + // if (bar(i)) ... // divergent + // + // A program may contain unstructured loops. Therefore, we cannot leverage + // LoopInfo, which only recognizes natural loops. + // + // The algorithm used here handles both natural and unstructured loops. Given + // a branch TI, we first compute its influence region, the union of all simple + // paths from TI to its immediate post dominator (IPostDom). Then, we search + // for all the values defined in the influence region but used outside. All + // these users are sync dependent on TI. + DenseSet InfluenceRegion; + computeInfluenceRegion(ThisBB, IPostDom, InfluenceRegion); + // An insight that can speed up the search process is that all the in-region + // values that are used outside must dominate TI. Therefore, instead of + // searching every basic blocks in the influence region, we search all the + // dominators of TI until it is outside the influence region. + BasicBlock *InfluencedBB = ThisBB; + while (InfluenceRegion.count(InfluencedBB)) { + for (auto &I : *InfluencedBB) + findUsersOutsideInfluenceRegion(I, InfluenceRegion); + DomTreeNode *IDomNode = DT.getNode(InfluencedBB)->getIDom(); + if (IDomNode == nullptr) + break; + InfluencedBB = IDomNode->getBlock(); + } +} + +void DivergencePropagator::findUsersOutsideInfluenceRegion( + Instruction &I, const DenseSet &InfluenceRegion) { + for (User *U : I.users()) { + Instruction *UserInst = cast(U); + if (!InfluenceRegion.count(UserInst->getParent())) { + if (DV.insert(UserInst).second) + Worklist.push_back(UserInst); + } + } +} + +void DivergencePropagator::computeInfluenceRegion( + BasicBlock *Start, BasicBlock *End, + DenseSet &InfluenceRegion) { + assert(PDT.properlyDominates(End, Start) && + "End does not properly dominate Start"); + std::vector InfluenceStack; + InfluenceStack.push_back(Start); + InfluenceRegion.insert(Start); + while (!InfluenceStack.empty()) { + BasicBlock *BB = InfluenceStack.back(); + InfluenceStack.pop_back(); + for (BasicBlock *Succ : successors(BB)) { + if (End != Succ && InfluenceRegion.insert(Succ).second) + InfluenceStack.push_back(Succ); + } + } +} + +void DivergencePropagator::exploreDataDependency(Value *V) { + // Follow def-use chains of V. + for (User *U : V->users()) { + Instruction *UserInst = cast(U); + if (DV.insert(UserInst).second) + Worklist.push_back(UserInst); + } +} + +void DivergencePropagator::propagate() { + // Traverse the dependency graph using DFS. + while (!Worklist.empty()) { + Value *V = Worklist.back(); + Worklist.pop_back(); + if (TerminatorInst *TI = dyn_cast(V)) { + // Terminators with less than two successors won't introduce sync + // dependency. Ignore them. + if (TI->getNumSuccessors() > 1) + exploreSyncDependency(TI); + } + exploreDataDependency(V); + } +} + +} /// end namespace anonymous + +FunctionPass *llvm::createDivergenceAnalysisPass() { + return new DivergenceAnalysis(); +} + +bool DivergenceAnalysis::runOnFunction(Function &F) { + auto *TTIWP = getAnalysisIfAvailable(); + if (TTIWP == nullptr) + return false; + + TargetTransformInfo &TTI = TTIWP->getTTI(F); + // Fast path: if the target does not have branch divergence, we do not mark + // any branch as divergent. + if (!TTI.hasBranchDivergence()) + return false; + + DivergentValues.clear(); + DivergencePropagator DP(F, TTI, + getAnalysis().getDomTree(), + getAnalysis(), DivergentValues); + DP.populateWithSourcesOfDivergence(); + DP.propagate(); + return false; +} + +void DivergenceAnalysis::print(raw_ostream &OS, const Module *) const { + if (DivergentValues.empty()) + return; + const Value *FirstDivergentValue = *DivergentValues.begin(); + const Function *F; + if (const Argument *Arg = dyn_cast(FirstDivergentValue)) { + F = Arg->getParent(); + } else if (const Instruction *I = + dyn_cast(FirstDivergentValue)) { + F = I->getParent()->getParent(); + } else { + llvm_unreachable("Only arguments and instructions can be divergent"); + } + + // Dumps all divergent values in F, arguments and then instructions. + for (auto &Arg : F->args()) { + if (DivergentValues.count(&Arg)) + OS << "DIVERGENT: " << Arg << "\n"; + } + // Iterate instructions using inst_range to ensure a deterministic order. + for (auto &I : inst_range(F)) { + if (DivergentValues.count(&I)) + OS << "DIVERGENT:" << I << "\n"; + } +} diff --git a/lib/Analysis/IPA/CallGraphSCCPass.cpp b/lib/Analysis/IPA/CallGraphSCCPass.cpp index 9d607cc..65ba1c7 100644 --- a/lib/Analysis/IPA/CallGraphSCCPass.cpp +++ b/lib/Analysis/IPA/CallGraphSCCPass.cpp @@ -212,10 +212,13 @@ bool CGPassManager::RefreshCallGraph(CallGraphSCC &CurSCC, // list of the same call. CallSites.count(I->first) || - // If the call edge is not from a call or invoke, then the function - // pass RAUW'd a call with another value. This can happen when - // constant folding happens of well known functions etc. - !CallSite(I->first)) { + // If the call edge is not from a call or invoke, or it is a + // instrinsic call, then the function pass RAUW'd a call with + // another value. This can happen when constant folding happens + // of well known functions etc. + !CallSite(I->first) || + (CallSite(I->first).getCalledFunction() && + CallSite(I->first).getCalledFunction()->isIntrinsic())) { assert(!CheckingMode && "CallGraphSCCPass did not update the CallGraph correctly!"); diff --git a/lib/Analysis/IPA/GlobalsModRef.cpp b/lib/Analysis/IPA/GlobalsModRef.cpp index 2208f32..018ae99 100644 --- a/lib/Analysis/IPA/GlobalsModRef.cpp +++ b/lib/Analysis/IPA/GlobalsModRef.cpp @@ -269,7 +269,7 @@ bool GlobalsModRef::AnalyzeUsesOfPointer(Value *V, } else if (Operator::getOpcode(I) == Instruction::BitCast) { if (AnalyzeUsesOfPointer(I, Readers, Writers, OkayStoreDest)) return true; - } else if (CallSite CS = I) { + } else if (auto CS = CallSite(I)) { // Make sure that this is just the function being called, not that it is // passing into the function. if (!CS.isCallee(&U)) { diff --git a/lib/Analysis/IPA/InlineCost.cpp b/lib/Analysis/IPA/InlineCost.cpp index eeb3b87..cacf70d 100644 --- a/lib/Analysis/IPA/InlineCost.cpp +++ b/lib/Analysis/IPA/InlineCost.cpp @@ -64,6 +64,7 @@ class CallAnalyzer : public InstVisitor { bool ContainsNoDuplicateCall; bool HasReturn; bool HasIndirectBr; + bool HasFrameEscape; /// Number of bytes allocated statically by the callee. uint64_t AllocatedSize; @@ -148,12 +149,12 @@ public: IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), - AllocatedSize(0), NumInstructions(0), NumVectorInstructions(0), - FiftyPercentVectorBonus(0), TenPercentVectorBonus(0), VectorBonus(0), - NumConstantArgs(0), NumConstantOffsetPtrArgs(0), NumAllocaArgs(0), - NumConstantPtrCmps(0), NumConstantPtrDiffs(0), - NumInstructionsSimplified(0), SROACostSavings(0), - SROACostSavingsLost(0) {} + HasFrameEscape(false), AllocatedSize(0), NumInstructions(0), + NumVectorInstructions(0), FiftyPercentVectorBonus(0), + TenPercentVectorBonus(0), VectorBonus(0), NumConstantArgs(0), + NumConstantOffsetPtrArgs(0), NumAllocaArgs(0), NumConstantPtrCmps(0), + NumConstantPtrDiffs(0), NumInstructionsSimplified(0), + SROACostSavings(0), SROACostSavingsLost(0) {} bool analyzeCall(CallSite CS); @@ -743,6 +744,9 @@ bool CallAnalyzer::visitCallSite(CallSite CS) { case Intrinsic::memmove: // SROA can usually chew through these intrinsics, but they aren't free. return false; + case Intrinsic::frameescape: + HasFrameEscape = true; + return false; } } @@ -941,7 +945,7 @@ bool CallAnalyzer::analyzeBlock(BasicBlock *BB, // If the visit this instruction detected an uninlinable pattern, abort. if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca || - HasIndirectBr) + HasIndirectBr || HasFrameEscape) return false; // If the caller is a recursive function then we don't want to inline @@ -1171,7 +1175,7 @@ bool CallAnalyzer::analyzeCall(CallSite CS) { // returns false, and we can bail on out. if (!analyzeBlock(BB, EphValues)) { if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca || - HasIndirectBr) + HasIndirectBr || HasFrameEscape) return false; // If the caller is a recursive function then we don't want to inline @@ -1286,16 +1290,18 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, int Threshold) { /// \brief Test that two functions either have or have not the given attribute /// at the same time. -static bool attributeMatches(Function *F1, Function *F2, - Attribute::AttrKind Attr) { - return F1->hasFnAttribute(Attr) == F2->hasFnAttribute(Attr); +template +static bool attributeMatches(Function *F1, Function *F2, AttrKind Attr) { + return F1->getFnAttribute(Attr) == F2->getFnAttribute(Attr); } /// \brief Test that there are no attribute conflicts between Caller and Callee /// that prevent inlining. static bool functionsHaveCompatibleAttributes(Function *Caller, Function *Callee) { - return attributeMatches(Caller, Callee, Attribute::SanitizeAddress) && + return attributeMatches(Caller, Callee, "target-cpu") && + attributeMatches(Caller, Callee, "target-features") && + attributeMatches(Caller, Callee, Attribute::SanitizeAddress) && attributeMatches(Caller, Callee, Attribute::SanitizeMemory) && attributeMatches(Caller, Callee, Attribute::SanitizeThread); } @@ -1370,6 +1376,13 @@ bool InlineCostAnalysis::isInlineViable(Function &F) { if (!ReturnsTwice && CS.isCall() && cast(CS.getInstruction())->canReturnTwice()) return false; + + // Disallow inlining functions that call @llvm.frameescape. Doing this + // correctly would require major changes to the inliner. + if (CS.getCalledFunction() && + CS.getCalledFunction()->getIntrinsicID() == + llvm::Intrinsic::frameescape) + return false; } } diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp index 99c477d..d45f7bd 100644 --- a/lib/Analysis/InstructionSimplify.cpp +++ b/lib/Analysis/InstructionSimplify.cpp @@ -2978,10 +2978,12 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, // what constant folding can make out of it. Constant *Null = Constant::getNullValue(GLHS->getPointerOperandType()); SmallVector IndicesLHS(GLHS->idx_begin(), GLHS->idx_end()); - Constant *NewLHS = ConstantExpr::getGetElementPtr(Null, IndicesLHS); + Constant *NewLHS = ConstantExpr::getGetElementPtr( + GLHS->getSourceElementType(), Null, IndicesLHS); SmallVector IndicesRHS(GRHS->idx_begin(), GRHS->idx_end()); - Constant *NewRHS = ConstantExpr::getGetElementPtr(Null, IndicesRHS); + Constant *NewRHS = ConstantExpr::getGetElementPtr( + GLHS->getSourceElementType(), Null, IndicesRHS); return ConstantExpr::getICmp(Pred, NewLHS, NewRHS); } } @@ -3241,17 +3243,18 @@ Value *llvm::SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, /// SimplifyGEPInst - Given operands for an GetElementPtrInst, see if we can /// fold the result. If not, this returns null. -static Value *SimplifyGEPInst(ArrayRef Ops, const Query &Q, unsigned) { +static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef Ops, + const Query &Q, unsigned) { // The type of the GEP pointer operand. - PointerType *PtrTy = cast(Ops[0]->getType()->getScalarType()); - unsigned AS = PtrTy->getAddressSpace(); + unsigned AS = + cast(Ops[0]->getType()->getScalarType())->getAddressSpace(); // getelementptr P -> P. if (Ops.size() == 1) return Ops[0]; // Compute the (pointer) type returned by the GEP instruction. - Type *LastType = GetElementPtrInst::getIndexedType(PtrTy, Ops.slice(1)); + Type *LastType = GetElementPtrInst::getIndexedType(SrcTy, Ops.slice(1)); Type *GEPTy = PointerType::get(LastType, AS); if (VectorType *VT = dyn_cast(Ops[0]->getType())) GEPTy = VectorType::get(GEPTy, VT->getNumElements()); @@ -3264,7 +3267,7 @@ static Value *SimplifyGEPInst(ArrayRef Ops, const Query &Q, unsigned) { if (match(Ops[1], m_Zero())) return Ops[0]; - Type *Ty = PtrTy->getElementType(); + Type *Ty = SrcTy; if (Ty->isSized()) { Value *P; uint64_t C; @@ -3318,14 +3321,17 @@ static Value *SimplifyGEPInst(ArrayRef Ops, const Query &Q, unsigned) { if (!isa(Ops[i])) return nullptr; - return ConstantExpr::getGetElementPtr(cast(Ops[0]), Ops.slice(1)); + return ConstantExpr::getGetElementPtr(SrcTy, cast(Ops[0]), + Ops.slice(1)); } Value *llvm::SimplifyGEPInst(ArrayRef Ops, const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) { - return ::SimplifyGEPInst(Ops, Query(DL, TLI, DT, AC, CxtI), RecursionLimit); + return ::SimplifyGEPInst( + cast(Ops[0]->getType()->getScalarType())->getElementType(), + Ops, Query(DL, TLI, DT, AC, CxtI), RecursionLimit); } /// SimplifyInsertValueInst - Given operands for an InsertValueInst, see if we diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp index 1818e93..724c21f 100644 --- a/lib/Analysis/LoopAccessAnalysis.cpp +++ b/lib/Analysis/LoopAccessAnalysis.cpp @@ -177,6 +177,17 @@ void LoopAccessInfo::RuntimePointerCheck::print( } } +bool LoopAccessInfo::RuntimePointerCheck::needsAnyChecking( + const SmallVectorImpl *PtrPartition) const { + unsigned NumPointers = Pointers.size(); + + for (unsigned I = 0; I < NumPointers; ++I) + for (unsigned J = I + 1; J < NumPointers; ++J) + if (needsChecking(I, J, PtrPartition)) + return true; + return false; +} + namespace { /// \brief Analyses memory accesses in a loop. /// @@ -1033,16 +1044,8 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) { for (I = Stores.begin(), IE = Stores.end(); I != IE; ++I) { StoreInst *ST = cast(*I); Value* Ptr = ST->getPointerOperand(); - - if (isUniform(Ptr)) { - emitAnalysis( - LoopAccessReport(ST) - << "write to a loop invariant address could not be vectorized"); - DEBUG(dbgs() << "LAA: We don't allow storing to uniform addresses\n"); - CanVecMem = false; - return; - } - + // Check for store to loop invariant address. + StoreToLoopInvariantAddress |= isUniform(Ptr); // If we did *not* see this pointer before, insert it to the read-write // list. At this phase it is only a 'write' list. if (Seen.insert(Ptr).second) { @@ -1211,9 +1214,8 @@ static Instruction *getFirstInst(Instruction *FirstInst, Value *V, std::pair LoopAccessInfo::addRuntimeCheck( Instruction *Loc, const SmallVectorImpl *PtrPartition) const { - Instruction *tnullptr = nullptr; if (!PtrRtCheck.Need) - return std::pair(tnullptr, tnullptr); + return std::make_pair(nullptr, nullptr); unsigned NumPointers = PtrRtCheck.Pointers.size(); SmallVector , 2> Starts; @@ -1284,6 +1286,9 @@ std::pair LoopAccessInfo::addRuntimeCheck( } } + if (!MemoryRuntimeCheck) + return std::make_pair(nullptr, nullptr); + // We have to do this trickery because the IRBuilder might fold the check to a // constant expression in which case there is no Instruction anchored in a // the block. @@ -1301,19 +1306,24 @@ LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE, const ValueToValueMap &Strides) : DepChecker(SE, L), NumComparisons(0), TheLoop(L), SE(SE), DL(DL), TLI(TLI), AA(AA), DT(DT), NumLoads(0), NumStores(0), - MaxSafeDepDistBytes(-1U), CanVecMem(false) { + MaxSafeDepDistBytes(-1U), CanVecMem(false), + StoreToLoopInvariantAddress(false) { if (canAnalyzeLoop()) analyzeLoop(Strides); } void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const { if (CanVecMem) { - if (PtrRtCheck.empty()) - OS.indent(Depth) << "Memory dependences are safe\n"; - else + if (PtrRtCheck.Need) OS.indent(Depth) << "Memory dependences are safe with run-time checks\n"; + else + OS.indent(Depth) << "Memory dependences are safe\n"; } + OS.indent(Depth) << "Store to invariant address was " + << (StoreToLoopInvariantAddress ? "" : "not ") + << "found in loop.\n"; + if (Report) OS.indent(Depth) << "Report: " << Report->str() << "\n"; diff --git a/lib/Analysis/MemDepPrinter.cpp b/lib/Analysis/MemDepPrinter.cpp index e1b7b4b..da3b829 100644 --- a/lib/Analysis/MemDepPrinter.cpp +++ b/lib/Analysis/MemDepPrinter.cpp @@ -106,7 +106,7 @@ bool MemDepPrinter::runOnFunction(Function &F) { if (!Res.isNonLocal()) { Deps[Inst].insert(std::make_pair(getInstTypePair(Res), static_cast(nullptr))); - } else if (CallSite CS = cast(Inst)) { + } else if (auto CS = CallSite(Inst)) { const MemoryDependenceAnalysis::NonLocalDepInfo &NLDI = MDA.getNonLocalCallDependency(CS); diff --git a/lib/Analysis/MemoryDependenceAnalysis.cpp b/lib/Analysis/MemoryDependenceAnalysis.cpp index 716e3e6..84769cb 100644 --- a/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/lib/Analysis/MemoryDependenceAnalysis.cpp @@ -223,7 +223,7 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall, continue; } - if (CallSite InstCS = cast(Inst)) { + if (auto InstCS = CallSite(Inst)) { // Debug intrinsics don't cause dependences. if (isa(Inst)) continue; // If these two calls do not interfere, look past it. @@ -874,23 +874,7 @@ MemoryDependenceAnalysis::getNonLocalCallDependency(CallSite QueryCS) { void MemoryDependenceAnalysis:: getNonLocalPointerDependency(Instruction *QueryInst, SmallVectorImpl &Result) { - - auto getLocation = [](AliasAnalysis *AA, Instruction *Inst) { - if (auto *I = dyn_cast(Inst)) - return AA->getLocation(I); - else if (auto *I = dyn_cast(Inst)) - return AA->getLocation(I); - else if (auto *I = dyn_cast(Inst)) - return AA->getLocation(I); - else if (auto *I = dyn_cast(Inst)) - return AA->getLocation(I); - else if (auto *I = dyn_cast(Inst)) - return AA->getLocation(I); - else - llvm_unreachable("unsupported memory instruction"); - }; - - const AliasAnalysis::Location Loc = getLocation(AA, QueryInst); + const AliasAnalysis::Location Loc = AA->getLocation(QueryInst); bool isLoad = isa(QueryInst); BasicBlock *FromBB = QueryInst->getParent(); assert(FromBB); diff --git a/lib/Analysis/ModuleDebugInfoPrinter.cpp b/lib/Analysis/ModuleDebugInfoPrinter.cpp index cbc4700..f2a11cb 100644 --- a/lib/Analysis/ModuleDebugInfoPrinter.cpp +++ b/lib/Analysis/ModuleDebugInfoPrinter.cpp @@ -72,55 +72,53 @@ void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const { // Printing the nodes directly isn't particularly helpful (since they // reference other nodes that won't be printed, particularly for the // filenames), so just print a few useful things. - for (DICompileUnit CU : Finder.compile_units()) { + for (MDCompileUnit *CU : Finder.compile_units()) { O << "Compile unit: "; - if (const char *Lang = LanguageString(CU.getLanguage())) + if (const char *Lang = dwarf::LanguageString(CU->getSourceLanguage())) O << Lang; else - O << "unknown-language(" << CU.getLanguage() << ")"; - printFile(O, CU.getFilename(), CU.getDirectory()); + O << "unknown-language(" << CU->getSourceLanguage() << ")"; + printFile(O, CU->getFilename(), CU->getDirectory()); O << '\n'; } - for (DISubprogram S : Finder.subprograms()) { - O << "Subprogram: " << S.getName(); - printFile(O, S.getFilename(), S.getDirectory(), S.getLineNumber()); - if (!S.getLinkageName().empty()) - O << " ('" << S.getLinkageName() << "')"; + for (MDSubprogram *S : Finder.subprograms()) { + O << "Subprogram: " << S->getName(); + printFile(O, S->getFilename(), S->getDirectory(), S->getLine()); + if (!S->getLinkageName().empty()) + O << " ('" << S->getLinkageName() << "')"; O << '\n'; } for (DIGlobalVariable GV : Finder.global_variables()) { - O << "Global variable: " << GV.getName(); - printFile(O, GV.getFilename(), GV.getDirectory(), GV.getLineNumber()); - if (!GV.getLinkageName().empty()) - O << " ('" << GV.getLinkageName() << "')"; + O << "Global variable: " << GV->getName(); + printFile(O, GV->getFilename(), GV->getDirectory(), GV->getLine()); + if (!GV->getLinkageName().empty()) + O << " ('" << GV->getLinkageName() << "')"; O << '\n'; } - for (DIType T : Finder.types()) { + for (const MDType *T : Finder.types()) { O << "Type:"; - if (!T.getName().empty()) - O << ' ' << T.getName(); - printFile(O, T.getFilename(), T.getDirectory(), T.getLineNumber()); - if (T.isBasicType()) { - DIBasicType BT(T.get()); + if (!T->getName().empty()) + O << ' ' << T->getName(); + printFile(O, T->getFilename(), T->getDirectory(), T->getLine()); + if (auto *BT = dyn_cast(T)) { O << " "; if (const char *Encoding = - dwarf::AttributeEncodingString(BT.getEncoding())) + dwarf::AttributeEncodingString(BT->getEncoding())) O << Encoding; else - O << "unknown-encoding(" << BT.getEncoding() << ')'; + O << "unknown-encoding(" << BT->getEncoding() << ')'; } else { O << ' '; - if (const char *Tag = dwarf::TagString(T.getTag())) + if (const char *Tag = dwarf::TagString(T->getTag())) O << Tag; else - O << "unknown-tag(" << T.getTag() << ")"; + O << "unknown-tag(" << T->getTag() << ")"; } - if (T.isCompositeType()) { - DICompositeType CT(T.get()); - if (auto *S = CT.getIdentifier()) + if (auto *CT = dyn_cast(T)) { + if (auto *S = CT->getRawIdentifier()) O << " (identifier: '" << S->getString() << "')"; } O << '\n'; diff --git a/lib/Analysis/RegionPass.cpp b/lib/Analysis/RegionPass.cpp index cd1e944..5e1cdd4 100644 --- a/lib/Analysis/RegionPass.cpp +++ b/lib/Analysis/RegionPass.cpp @@ -199,7 +199,7 @@ public: bool runOnRegion(Region *R, RGPassManager &RGM) override { Out << Banner; - for (const auto &BB : R->blocks()) { + for (const auto *BB : R->blocks()) { if (BB) BB->print(Out); else diff --git a/lib/Analysis/RegionPrinter.cpp b/lib/Analysis/RegionPrinter.cpp index ad83113..d7f5109 100644 --- a/lib/Analysis/RegionPrinter.cpp +++ b/lib/Analysis/RegionPrinter.cpp @@ -123,7 +123,7 @@ struct DOTGraphTraits : public DOTGraphTraits { const RegionInfo &RI = *static_cast(R.getRegionInfo()); - for (const auto &BB : R.blocks()) + for (auto *BB : R.blocks()) if (RI.getRegionFor(BB) == &R) O.indent(2 * (depth + 1)) << "Node" << static_cast(RI.getTopLevelRegion()->getBBNode(BB)) diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index 4e713fb..37377f0 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -5690,7 +5690,7 @@ static Constant *BuildConstantFromSCEV(const SCEV *V) { if (PTy->getElementType()->isStructTy()) C2 = ConstantExpr::getIntegerCast( C2, Type::getInt32Ty(C->getContext()), true); - C = ConstantExpr::getGetElementPtr(C, C2); + C = ConstantExpr::getGetElementPtr(PTy->getElementType(), C, C2); } else C = ConstantExpr::getAdd(C, C2); } @@ -6698,6 +6698,65 @@ ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L, return true; } + struct ClearWalkingBEDominatingCondsOnExit { + ScalarEvolution &SE; + + explicit ClearWalkingBEDominatingCondsOnExit(ScalarEvolution &SE) + : SE(SE){}; + + ~ClearWalkingBEDominatingCondsOnExit() { + SE.WalkingBEDominatingConds = false; + } + }; + + // We don't want more than one activation of the following loop on the stack + // -- that can lead to O(n!) time complexity. + if (WalkingBEDominatingConds) + return false; + + WalkingBEDominatingConds = true; + ClearWalkingBEDominatingCondsOnExit ClearOnExit(*this); + + // If the loop is not reachable from the entry block, we risk running into an + // infinite loop as we walk up into the dom tree. These loops do not matter + // anyway, so we just return a conservative answer when we see them. + if (!DT->isReachableFromEntry(L->getHeader())) + return false; + + for (DomTreeNode *DTN = (*DT)[Latch], *HeaderDTN = (*DT)[L->getHeader()]; + DTN != HeaderDTN; + DTN = DTN->getIDom()) { + + assert(DTN && "should reach the loop header before reaching the root!"); + + BasicBlock *BB = DTN->getBlock(); + BasicBlock *PBB = BB->getSinglePredecessor(); + if (!PBB) + continue; + + BranchInst *ContinuePredicate = dyn_cast(PBB->getTerminator()); + if (!ContinuePredicate || !ContinuePredicate->isConditional()) + continue; + + Value *Condition = ContinuePredicate->getCondition(); + + // If we have an edge `E` within the loop body that dominates the only + // latch, the condition guarding `E` also guards the backedge. This + // reasoning works only for loops with a single latch. + + BasicBlockEdge DominatingEdge(PBB, BB); + if (DominatingEdge.isSingleEdge()) { + // We're constructively (and conservatively) enumerating edges within the + // loop body that dominate the latch. The dominator tree better agree + // with us on this: + assert(DT->dominates(DominatingEdge, Latch) && "should be!"); + + if (isImpliedCond(Pred, LHS, RHS, Condition, + BB != ContinuePredicate->getSuccessor(0))) + return true; + } + } + return false; } @@ -7968,8 +8027,8 @@ ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se) //===----------------------------------------------------------------------===// ScalarEvolution::ScalarEvolution() - : FunctionPass(ID), ValuesAtScopes(64), LoopDispositions(64), - BlockDispositions(64), FirstUnknown(nullptr) { + : FunctionPass(ID), WalkingBEDominatingConds(false), ValuesAtScopes(64), + LoopDispositions(64), BlockDispositions(64), FirstUnknown(nullptr) { initializeScalarEvolutionPass(*PassRegistry::getPassRegistry()); } @@ -8000,6 +8059,7 @@ void ScalarEvolution::releaseMemory() { } assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); + assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!"); BackedgeTakenCounts.clear(); ConstantEvolutionLoopExitValue.clear(); diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp index a73ec9e..0bd427b 100644 --- a/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/lib/Analysis/ScalarEvolutionExpander.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -488,7 +489,8 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, // Fold a GEP with constant operands. if (Constant *CLHS = dyn_cast(V)) if (Constant *CRHS = dyn_cast(Idx)) - return ConstantExpr::getGetElementPtr(CLHS, CRHS); + return ConstantExpr::getGetElementPtr(Type::getInt8Ty(Ty->getContext()), + CLHS, CRHS); // Do a quick scan to see if we have this GEP nearby. If so, reuse it. unsigned ScanLimit = 6; @@ -523,7 +525,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin, } // Emit a GEP. - Value *GEP = Builder.CreateGEP(V, Idx, "uglygep"); + Value *GEP = Builder.CreateGEP(Builder.getInt8Ty(), V, Idx, "uglygep"); rememberInstruction(GEP); return GEP; @@ -1803,6 +1805,72 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT, return NumElim; } +bool SCEVExpander::isHighCostExpansionHelper( + const SCEV *S, Loop *L, SmallPtrSetImpl &Processed) { + if (!Processed.insert(S).second) + return false; + + if (auto *UDivExpr = dyn_cast(S)) { + // If the divisor is a power of two and the SCEV type fits in a native + // integer, consider the divison cheap irrespective of whether it occurs in + // the user code since it can be lowered into a right shift. + if (auto *SC = dyn_cast(UDivExpr->getRHS())) + if (SC->getValue()->getValue().isPowerOf2()) { + const DataLayout &DL = + L->getHeader()->getParent()->getParent()->getDataLayout(); + unsigned Width = cast(UDivExpr->getType())->getBitWidth(); + return DL.isIllegalInteger(Width); + } + + // UDivExpr is very likely a UDiv that ScalarEvolution's HowFarToZero or + // HowManyLessThans produced to compute a precise expression, rather than a + // UDiv from the user's code. If we can't find a UDiv in the code with some + // simple searching, assume the former consider UDivExpr expensive to + // compute. + BasicBlock *ExitingBB = L->getExitingBlock(); + if (!ExitingBB) + return true; + + BranchInst *ExitingBI = dyn_cast(ExitingBB->getTerminator()); + if (!ExitingBI || !ExitingBI->isConditional()) + return true; + + ICmpInst *OrigCond = dyn_cast(ExitingBI->getCondition()); + if (!OrigCond) + return true; + + const SCEV *RHS = SE.getSCEV(OrigCond->getOperand(1)); + RHS = SE.getMinusSCEV(RHS, SE.getConstant(RHS->getType(), 1)); + if (RHS != S) { + const SCEV *LHS = SE.getSCEV(OrigCond->getOperand(0)); + LHS = SE.getMinusSCEV(LHS, SE.getConstant(LHS->getType(), 1)); + if (LHS != S) + return true; + } + } + + // Recurse past add expressions, which commonly occur in the + // BackedgeTakenCount. They may already exist in program code, and if not, + // they are not too expensive rematerialize. + if (const SCEVAddExpr *Add = dyn_cast(S)) { + for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); + I != E; ++I) { + if (isHighCostExpansionHelper(*I, L, Processed)) + return true; + } + return false; + } + + // HowManyLessThans uses a Max expression whenever the loop is not guarded by + // the exit condition. + if (isa(S) || isa(S)) + return true; + + // If we haven't recognized an expensive SCEV pattern, assume it's an + // expression produced by program code. + return false; +} + namespace { // Search for a SCEV subexpression that is not safe to expand. Any expression // that may expand to a !isSafeToSpeculativelyExecute value is unsafe, namely diff --git a/lib/Analysis/TargetLibraryInfo.cpp b/lib/Analysis/TargetLibraryInfo.cpp index 7e574d5..8b378a3 100644 --- a/lib/Analysis/TargetLibraryInfo.cpp +++ b/lib/Analysis/TargetLibraryInfo.cpp @@ -409,9 +409,7 @@ static StringRef sanitizeFunctionName(StringRef funcName) { // Check for \01 prefix that is used to mangle __asm declarations and // strip it if present. - if (funcName.front() == '\01') - funcName = funcName.substr(1); - return funcName; + return GlobalValue::getRealLinkageName(funcName); } bool TargetLibraryInfoImpl::getLibFunc(StringRef funcName, diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp index f51c7f54..a1519de 100644 --- a/lib/Analysis/TargetTransformInfo.cpp +++ b/lib/Analysis/TargetTransformInfo.cpp @@ -76,6 +76,10 @@ bool TargetTransformInfo::hasBranchDivergence() const { return TTIImpl->hasBranchDivergence(); } +bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { + return TTIImpl->isSourceOfDivergence(V); +} + bool TargetTransformInfo::isLoweredToCall(const Function *F) const { return TTIImpl->isLoweredToCall(F); } diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp index f329e3a..3651301 100644 --- a/lib/Analysis/ValueTracking.cpp +++ b/lib/Analysis/ValueTracking.cpp @@ -694,10 +694,9 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero, // We're running this loop for once for each value queried resulting in a // runtime of ~O(#assumes * #values). - assert(isa(I) && - dyn_cast(I)->getIntrinsicID() == Intrinsic::assume && + assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && "must be an assume intrinsic"); - + Value *Arg = I->getArgOperand(0); if (Arg == V && isValidAssumeForContext(I, Q)) { @@ -2935,7 +2934,7 @@ bool llvm::isKnownNonNull(const Value *V, const TargetLibraryInfo *TLI) { if (const LoadInst *LI = dyn_cast(V)) return LI->getMetadata(LLVMContext::MD_nonnull); - if (ImmutableCallSite CS = V) + if (auto CS = ImmutableCallSite(V)) if (CS.isReturnNonNull()) return true; diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp index 3bf090a..a72f713 100644 --- a/lib/AsmParser/LLLexer.cpp +++ b/lib/AsmParser/LLLexer.cpp @@ -598,6 +598,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(inalloca); KEYWORD(cold); KEYWORD(dereferenceable); + KEYWORD(dereferenceable_or_null); KEYWORD(inlinehint); KEYWORD(inreg); KEYWORD(jumptable); diff --git a/lib/AsmParser/LLLexer.h b/lib/AsmParser/LLLexer.h index 3343168..90bf17d 100644 --- a/lib/AsmParser/LLLexer.h +++ b/lib/AsmParser/LLLexer.h @@ -45,7 +45,6 @@ namespace llvm { public: explicit LLLexer(StringRef StartBuf, SourceMgr &SM, SMDiagnostic &, LLVMContext &C); - ~LLLexer() {} lltok::Kind Lex() { return CurKind = LexToken(); diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index 103c8c4..546363b 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -976,6 +976,7 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B, break; case lltok::kw_byval: case lltok::kw_dereferenceable: + case lltok::kw_dereferenceable_or_null: case lltok::kw_inalloca: case lltok::kw_nest: case lltok::kw_noalias: @@ -1220,11 +1221,18 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) { case lltok::kw_byval: B.addAttribute(Attribute::ByVal); break; case lltok::kw_dereferenceable: { uint64_t Bytes; - if (ParseOptionalDereferenceableBytes(Bytes)) + if (ParseOptionalDerefAttrBytes(lltok::kw_dereferenceable, Bytes)) return true; B.addDereferenceableAttr(Bytes); continue; } + case lltok::kw_dereferenceable_or_null: { + uint64_t Bytes; + if (ParseOptionalDerefAttrBytes(lltok::kw_dereferenceable_or_null, Bytes)) + return true; + B.addDereferenceableOrNullAttr(Bytes); + continue; + } case lltok::kw_inalloca: B.addAttribute(Attribute::InAlloca); break; case lltok::kw_inreg: B.addAttribute(Attribute::InReg); break; case lltok::kw_nest: B.addAttribute(Attribute::Nest); break; @@ -1284,11 +1292,18 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) { return HaveError; case lltok::kw_dereferenceable: { uint64_t Bytes; - if (ParseOptionalDereferenceableBytes(Bytes)) + if (ParseOptionalDerefAttrBytes(lltok::kw_dereferenceable, Bytes)) return true; B.addDereferenceableAttr(Bytes); continue; } + case lltok::kw_dereferenceable_or_null: { + uint64_t Bytes; + if (ParseOptionalDerefAttrBytes(lltok::kw_dereferenceable_or_null, Bytes)) + return true; + B.addDereferenceableOrNullAttr(Bytes); + continue; + } case lltok::kw_inreg: B.addAttribute(Attribute::InReg); break; case lltok::kw_noalias: B.addAttribute(Attribute::NoAlias); break; case lltok::kw_nonnull: B.addAttribute(Attribute::NonNull); break; @@ -1516,12 +1531,19 @@ bool LLParser::ParseOptionalAlignment(unsigned &Alignment) { return false; } -/// ParseOptionalDereferenceableBytes +/// ParseOptionalDerefAttrBytes /// ::= /* empty */ -/// ::= 'dereferenceable' '(' 4 ')' -bool LLParser::ParseOptionalDereferenceableBytes(uint64_t &Bytes) { +/// ::= AttrKind '(' 4 ')' +/// +/// where AttrKind is either 'dereferenceable' or 'dereferenceable_or_null'. +bool LLParser::ParseOptionalDerefAttrBytes(lltok::Kind AttrKind, + uint64_t &Bytes) { + assert((AttrKind == lltok::kw_dereferenceable || + AttrKind == lltok::kw_dereferenceable_or_null) && + "contract!"); + Bytes = 0; - if (!EatIfPresent(lltok::kw_dereferenceable)) + if (!EatIfPresent(AttrKind)) return false; LocTy ParenLoc = Lex.getLoc(); if (!EatIfPresent(lltok::lparen)) @@ -2831,10 +2853,10 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { !BasePointerType->getElementType()->isSized(&Visited)) return Error(ID.Loc, "base element of getelementptr must be sized"); - if (!GetElementPtrInst::getIndexedType(Elts[0]->getType(), Indices)) + if (!GetElementPtrInst::getIndexedType(Ty, Indices)) return Error(ID.Loc, "invalid getelementptr indices"); - ID.ConstantVal = ConstantExpr::getGetElementPtr(Elts[0], Indices, - InBounds); + ID.ConstantVal = + ConstantExpr::getGetElementPtr(Ty, Elts[0], Indices, InBounds); } else if (Opc == Instruction::Select) { if (Elts.size() != 3) return Error(ID.Loc, "expected three operands to select"); @@ -3030,13 +3052,17 @@ struct MDBoolField : public MDFieldImpl { MDBoolField(bool Default = false) : ImplTy(Default) {} }; struct MDField : public MDFieldImpl { - MDField() : ImplTy(nullptr) {} + bool AllowNull; + + MDField(bool AllowNull = true) : ImplTy(nullptr), AllowNull(AllowNull) {} }; struct MDConstant : public MDFieldImpl { MDConstant() : ImplTy(nullptr) {} }; -struct MDStringField : public MDFieldImpl { - MDStringField() : ImplTy(std::string()) {} +struct MDStringField : public MDFieldImpl { + bool AllowEmpty; + MDStringField(bool AllowEmpty = true) + : ImplTy(nullptr), AllowEmpty(AllowEmpty) {} }; struct MDFieldList : public MDFieldImpl> { MDFieldList() : ImplTy(SmallVector()) {} @@ -3161,7 +3187,7 @@ bool LLParser::ParseMDField(LocTy Loc, StringRef Name, DIFlagField &Result) { if (Lex.getKind() != lltok::DIFlag) return TokError("expected debug info flag"); - Val = DIDescriptor::getFlag(Lex.getStrVal()); + Val = DebugNode::getFlag(Lex.getStrVal()); if (!Val) return TokError(Twine("invalid debug info flag flag '") + Lex.getStrVal() + "'"); @@ -3221,6 +3247,8 @@ bool LLParser::ParseMDField(LocTy Loc, StringRef Name, MDBoolField &Result) { template <> bool LLParser::ParseMDField(LocTy Loc, StringRef Name, MDField &Result) { if (Lex.getKind() == lltok::kw_null) { + if (!Result.AllowNull) + return TokError("'" + Name + "' cannot be null"); Lex.Lex(); Result.assign(nullptr); return false; @@ -3246,11 +3274,15 @@ bool LLParser::ParseMDField(LocTy Loc, StringRef Name, MDConstant &Result) { template <> bool LLParser::ParseMDField(LocTy Loc, StringRef Name, MDStringField &Result) { + LocTy ValueLoc = Lex.getLoc(); std::string S; if (ParseStringConstant(S)) return true; - Result.assign(std::move(S)); + if (!Result.AllowEmpty && S.empty()) + return Error(ValueLoc, "'" + Name + "' cannot be empty"); + + Result.assign(S.empty() ? nullptr : MDString::get(Context, S)); return false; } @@ -3343,7 +3375,7 @@ bool LLParser::ParseMDLocation(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ OPTIONAL(line, LineField, ); \ OPTIONAL(column, ColumnField, ); \ - REQUIRED(scope, MDField, ); \ + REQUIRED(scope, MDField, (/* AllowNull */ false)); \ OPTIONAL(inlinedAt, MDField, ); PARSE_MD_FIELDS(); #undef VISIT_MD_FIELDS @@ -3499,7 +3531,7 @@ bool LLParser::ParseMDFile(MDNode *&Result, bool IsDistinct) { bool LLParser::ParseMDCompileUnit(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ REQUIRED(language, DwarfLangField, ); \ - REQUIRED(file, MDField, ); \ + REQUIRED(file, MDField, (/* AllowNull */ false)); \ OPTIONAL(producer, MDStringField, ); \ OPTIONAL(isOptimized, MDBoolField, ); \ OPTIONAL(flags, MDStringField, ); \ @@ -3567,7 +3599,7 @@ bool LLParser::ParseMDSubprogram(MDNode *&Result, bool IsDistinct) { /// ::= !MDLexicalBlock(scope: !0, file: !2, line: 7, column: 9) bool LLParser::ParseMDLexicalBlock(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ - REQUIRED(scope, MDField, ); \ + REQUIRED(scope, MDField, (/* AllowNull */ false)); \ OPTIONAL(file, MDField, ); \ OPTIONAL(line, LineField, ); \ OPTIONAL(column, ColumnField, ); @@ -3583,7 +3615,7 @@ bool LLParser::ParseMDLexicalBlock(MDNode *&Result, bool IsDistinct) { /// ::= !MDLexicalBlockFile(scope: !0, file: !2, discriminator: 9) bool LLParser::ParseMDLexicalBlockFile(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ - REQUIRED(scope, MDField, ); \ + REQUIRED(scope, MDField, (/* AllowNull */ false)); \ OPTIONAL(file, MDField, ); \ REQUIRED(discriminator, MDUnsignedField, (0, UINT32_MAX)); PARSE_MD_FIELDS(); @@ -3648,8 +3680,8 @@ bool LLParser::ParseMDTemplateValueParameter(MDNode *&Result, bool IsDistinct) { /// declaration: !3) bool LLParser::ParseMDGlobalVariable(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ + REQUIRED(name, MDStringField, (/* AllowEmpty */ false)); \ OPTIONAL(scope, MDField, ); \ - OPTIONAL(name, MDStringField, ); \ OPTIONAL(linkageName, MDStringField, ); \ OPTIONAL(file, MDField, ); \ OPTIONAL(line, LineField, ); \ @@ -3670,25 +3702,23 @@ bool LLParser::ParseMDGlobalVariable(MDNode *&Result, bool IsDistinct) { /// ParseMDLocalVariable: /// ::= !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !0, name: "foo", -/// file: !1, line: 7, type: !2, arg: 2, flags: 7, -/// inlinedAt: !3) +/// file: !1, line: 7, type: !2, arg: 2, flags: 7) bool LLParser::ParseMDLocalVariable(MDNode *&Result, bool IsDistinct) { #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ REQUIRED(tag, DwarfTagField, ); \ - OPTIONAL(scope, MDField, ); \ + REQUIRED(scope, MDField, (/* AllowNull */ false)); \ OPTIONAL(name, MDStringField, ); \ OPTIONAL(file, MDField, ); \ OPTIONAL(line, LineField, ); \ OPTIONAL(type, MDField, ); \ OPTIONAL(arg, MDUnsignedField, (0, UINT8_MAX)); \ - OPTIONAL(flags, DIFlagField, ); \ - OPTIONAL(inlinedAt, MDField, ); + OPTIONAL(flags, DIFlagField, ); PARSE_MD_FIELDS(); #undef VISIT_MD_FIELDS - Result = GET_OR_DISTINCT( - MDLocalVariable, (Context, tag.Val, scope.Val, name.Val, file.Val, - line.Val, type.Val, arg.Val, flags.Val, inlinedAt.Val)); + Result = GET_OR_DISTINCT(MDLocalVariable, + (Context, tag.Val, scope.Val, name.Val, file.Val, + line.Val, type.Val, arg.Val, flags.Val)); return false; } @@ -5130,10 +5160,8 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS, // If RetType is a non-function pointer type, then this is the short syntax // for the call, which means that RetType is just the return type. Infer the // rest of the function argument types from the arguments that are present. - PointerType *PFTy = nullptr; - FunctionType *Ty = nullptr; - if (!(PFTy = dyn_cast(RetType)) || - !(Ty = dyn_cast(PFTy->getElementType()))) { + FunctionType *Ty = dyn_cast(RetType); + if (!Ty) { // Pull out the types of all of the arguments... std::vector ParamTypes; for (unsigned i = 0, e = ArgList.size(); i != e; ++i) @@ -5143,12 +5171,12 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS, return Error(RetTypeLoc, "Invalid result type for LLVM function"); Ty = FunctionType::get(RetType, ParamTypes, false); - PFTy = PointerType::getUnqual(Ty); } // Look up the callee. Value *Callee; - if (ConvertValIDToValue(PFTy, CalleeID, Callee, &PFS)) return true; + if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS)) + return true; // Set up the Attribute for the function. SmallVector Attrs; @@ -5269,7 +5297,7 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { Lex.Lex(); } - Type *Ty = nullptr; + Type *Ty; LocTy ExplicitTypeLoc = Lex.getLoc(); if (ParseType(Ty) || ParseToken(lltok::comma, "expected comma after load's type") || @@ -5278,8 +5306,7 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { ParseOptionalCommaAlign(Alignment, AteExtraComma)) return true; - if (!Val->getType()->isPointerTy() || - !cast(Val->getType())->getElementType()->isFirstClassType()) + if (!Val->getType()->isPointerTy() || !Ty->isFirstClassType()) return Error(Loc, "load operand must be a pointer to a first class type"); if (isAtomic && !Alignment) return Error(Loc, "atomic load must have explicit non-zero alignment"); @@ -5290,7 +5317,7 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { return Error(ExplicitTypeLoc, "explicit pointee type doesn't match operand's pointee type"); - Inst = new LoadInst(Val, "", isVolatile, Alignment, Ordering, Scope); + Inst = new LoadInst(Ty, Val, "", isVolatile, Alignment, Ordering, Scope); return AteExtraComma ? InstExtraComma : InstNormal; } @@ -5519,7 +5546,9 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) { !BasePointerType->getElementType()->isSized(&Visited)) return Error(Loc, "base element of getelementptr must be sized"); - if (!GetElementPtrInst::getIndexedType(BaseType, Indices)) + if (!GetElementPtrInst::getIndexedType( + cast(BaseType->getScalarType())->getElementType(), + Indices)) return Error(Loc, "invalid getelementptr indices"); Inst = GetElementPtrInst::Create(Ty, Ptr, Indices); if (InBounds) diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h index 5e92e57..117cdcb 100644 --- a/lib/AsmParser/LLParser.h +++ b/lib/AsmParser/LLParser.h @@ -223,7 +223,7 @@ namespace llvm { bool ParseOptionalDLLStorageClass(unsigned &DLLStorageClass); bool ParseOptionalCallingConv(unsigned &CC); bool ParseOptionalAlignment(unsigned &Alignment); - bool ParseOptionalDereferenceableBytes(uint64_t &Bytes); + bool ParseOptionalDerefAttrBytes(lltok::Kind AttrKind, uint64_t &Bytes); bool ParseScopeAndOrdering(bool isAtomic, SynchronizationScope &Scope, AtomicOrdering &Ordering); bool ParseOrdering(AtomicOrdering &Ordering); diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h index a7aa17c..2bdc53b 100644 --- a/lib/AsmParser/LLToken.h +++ b/lib/AsmParser/LLToken.h @@ -106,6 +106,7 @@ namespace lltok { kw_inalloca, kw_cold, kw_dereferenceable, + kw_dereferenceable_or_null, kw_inlinehint, kw_inreg, kw_jumptable, diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 84753ff..5366f5f 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -16,6 +16,7 @@ #include "llvm/Bitcode/LLVMBitCodes.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticPrinter.h" @@ -218,6 +219,8 @@ class BitcodeReader : public GVMaterializer { /// True if any Metadata block has been materialized. bool IsMetadataMaterialized; + bool StripDebugInfo = false; + public: std::error_code Error(BitcodeError E, const Twine &Message); std::error_code Error(BitcodeError E); @@ -227,7 +230,7 @@ public: DiagnosticHandlerFunction DiagnosticHandler); explicit BitcodeReader(DataStreamer *streamer, LLVMContext &C, DiagnosticHandlerFunction DiagnosticHandler); - ~BitcodeReader() { FreeState(); } + ~BitcodeReader() override { FreeState(); } std::error_code materializeForwardReferencedFunctions(); @@ -255,6 +258,8 @@ public: /// Materialize any deferred Metadata block. std::error_code materializeMetadata() override; + void setStripDebugInfo() override; + private: std::vector IdentifiedStructTypes; StructType *createIdentifiedStructType(LLVMContext &Context, StringRef Name); @@ -1093,6 +1098,8 @@ static Attribute::AttrKind GetAttrFromCode(uint64_t Code) { return Attribute::NonNull; case bitc::ATTR_KIND_DEREFERENCEABLE: return Attribute::Dereferenceable; + case bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL: + return Attribute::DereferenceableOrNull; case bitc::ATTR_KIND_NO_RED_ZONE: return Attribute::NoRedZone; case bitc::ATTR_KIND_NO_RETURN: @@ -1209,6 +1216,8 @@ std::error_code BitcodeReader::ParseAttributeGroupBlock() { B.addStackAlignmentAttr(Record[++i]); else if (Kind == Attribute::Dereferenceable) B.addDereferenceableAttr(Record[++i]); + else if (Kind == Attribute::DereferenceableOrNull) + B.addDereferenceableOrNullAttr(Record[++i]); } else { // String attribute assert((Record[i] == 3 || Record[i] == 4) && "Invalid attribute group entry"); @@ -1906,7 +1915,8 @@ std::error_code BitcodeReader::ParseMetadata() { break; } case bitc::METADATA_LOCAL_VAR: { - if (Record.size() != 10) + // 10th field is for the obseleted 'inlinedAt:' field. + if (Record.size() != 9 && Record.size() != 10) return Error("Invalid record"); MDValueList.AssignValue( @@ -1914,7 +1924,7 @@ std::error_code BitcodeReader::ParseMetadata() { (Context, Record[1], getMDOrNull(Record[2]), getMDString(Record[3]), getMDOrNull(Record[4]), Record[5], getMDOrNull(Record[6]), Record[7], - Record[8], getMDOrNull(Record[9]))), + Record[8])), NextMDValueNo++); break; } @@ -2308,14 +2318,17 @@ std::error_code BitcodeReader::ParseConstants() { Elts.push_back(ValueList.getConstantFwdRef(Record[OpNum++], ElTy)); } - ArrayRef Indices(Elts.begin() + 1, Elts.end()); - V = ConstantExpr::getGetElementPtr(Elts[0], Indices, - BitCode == - bitc::CST_CODE_CE_INBOUNDS_GEP); if (PointeeType && - PointeeType != cast(V)->getSourceElementType()) + PointeeType != + cast(Elts[0]->getType()->getScalarType()) + ->getElementType()) return Error("Explicit gep operator type does not match pointee type " "of pointer operand"); + + ArrayRef Indices(Elts.begin() + 1, Elts.end()); + V = ConstantExpr::getGetElementPtr(PointeeType, Elts[0], Indices, + BitCode == + bitc::CST_CODE_CE_INBOUNDS_GEP); break; } case bitc::CST_CODE_CE_SELECT: { // CE_SELECT: [opval#, opval#, opval#] @@ -2609,6 +2622,8 @@ std::error_code BitcodeReader::materializeMetadata() { return std::error_code(); } +void BitcodeReader::setStripDebugInfo() { StripDebugInfo = true; } + /// RememberAndSkipFunctionBody - When we see the block for a function body, /// remember where it is and then skip it. This lets us lazily deserialize the /// functions. @@ -3053,8 +3068,12 @@ std::error_code BitcodeReader::ParseBitcodeInto(Module *M, // We expect a number of well-defined blocks, though we don't necessarily // need to understand them all. while (1) { - if (Stream.AtEndOfStream()) - return std::error_code(); + if (Stream.AtEndOfStream()) { + if (TheModule) + return std::error_code(); + // We didn't really read a proper Module. + return Error("Malformed IR file"); + } BitstreamEntry Entry = Stream.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs); @@ -4305,6 +4324,9 @@ std::error_code BitcodeReader::materialize(GlobalValue *GV) { return EC; F->setIsMaterializable(false); + if (StripDebugInfo) + stripDebugInfo(*F); + // Upgrade any old intrinsic calls in the function. for (UpgradedIntrinsicMap::iterator I = UpgradedIntrinsics.begin(), E = UpgradedIntrinsics.end(); I != E; ++I) { diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index 0123fb2..aa4a6a4 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -200,6 +200,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NON_NULL; case Attribute::Dereferenceable: return bitc::ATTR_KIND_DEREFERENCEABLE; + case Attribute::DereferenceableOrNull: + return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL; case Attribute::NoRedZone: return bitc::ATTR_KIND_NO_RED_ZONE; case Attribute::NoReturn: @@ -821,7 +823,7 @@ static void WriteMDSubrange(const MDSubrange *N, const ValueEnumerator &, unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getCount()); - Record.push_back(rotateSign(N->getLo())); + Record.push_back(rotateSign(N->getLowerBound())); Stream.EmitRecord(bitc::METADATA_SUBRANGE, Record, Abbrev); Record.clear(); @@ -892,10 +894,10 @@ static void WriteMDCompositeType(const MDCompositeType *N, Record.push_back(N->getAlignInBits()); Record.push_back(N->getOffsetInBits()); Record.push_back(N->getFlags()); - Record.push_back(VE.getMetadataOrNullID(N->getElements())); + Record.push_back(VE.getMetadataOrNullID(N->getElements().get())); Record.push_back(N->getRuntimeLang()); Record.push_back(VE.getMetadataOrNullID(N->getVTableHolder())); - Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams())); + Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier())); Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev); @@ -909,7 +911,7 @@ static void WriteMDSubroutineType(const MDSubroutineType *N, unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getFlags()); - Record.push_back(VE.getMetadataOrNullID(N->getTypeArray())); + Record.push_back(VE.getMetadataOrNullID(N->getTypeArray().get())); Stream.EmitRecord(bitc::METADATA_SUBROUTINE_TYPE, Record, Abbrev); Record.clear(); @@ -940,11 +942,11 @@ static void WriteMDCompileUnit(const MDCompileUnit *N, Record.push_back(N->getRuntimeVersion()); Record.push_back(VE.getMetadataOrNullID(N->getRawSplitDebugFilename())); Record.push_back(N->getEmissionKind()); - Record.push_back(VE.getMetadataOrNullID(N->getEnumTypes())); - Record.push_back(VE.getMetadataOrNullID(N->getRetainedTypes())); - Record.push_back(VE.getMetadataOrNullID(N->getSubprograms())); - Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables())); - Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities())); + Record.push_back(VE.getMetadataOrNullID(N->getEnumTypes().get())); + Record.push_back(VE.getMetadataOrNullID(N->getRetainedTypes().get())); + Record.push_back(VE.getMetadataOrNullID(N->getSubprograms().get())); + Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables().get())); + Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get())); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -970,10 +972,10 @@ static void WriteMDSubprogram(const MDSubprogram *N, Record.push_back(N->getVirtualIndex()); Record.push_back(N->getFlags()); Record.push_back(N->isOptimized()); - Record.push_back(VE.getMetadataOrNullID(N->getFunction())); - Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams())); + Record.push_back(VE.getMetadataOrNullID(N->getRawFunction())); + Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getDeclaration())); - Record.push_back(VE.getMetadataOrNullID(N->getVariables())); + Record.push_back(VE.getMetadataOrNullID(N->getVariables().get())); Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev); Record.clear(); @@ -1064,7 +1066,7 @@ static void WriteMDGlobalVariable(const MDGlobalVariable *N, Record.push_back(VE.getMetadataOrNullID(N->getType())); Record.push_back(N->isLocalToUnit()); Record.push_back(N->isDefinition()); - Record.push_back(VE.getMetadataOrNullID(N->getVariable())); + Record.push_back(VE.getMetadataOrNullID(N->getRawVariable())); Record.push_back(VE.getMetadataOrNullID(N->getStaticDataMemberDeclaration())); Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR, Record, Abbrev); @@ -1085,7 +1087,6 @@ static void WriteMDLocalVariable(const MDLocalVariable *N, Record.push_back(VE.getMetadataOrNullID(N->getType())); Record.push_back(N->getArg()); Record.push_back(N->getFlags()); - Record.push_back(VE.getMetadataOrNullID(N->getInlinedAt())); Stream.EmitRecord(bitc::METADATA_LOCAL_VAR, Record, Abbrev); Record.clear(); @@ -2047,6 +2048,9 @@ static void WriteUseList(ValueEnumerator &VE, UseListOrder &&Order, static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, BitstreamWriter &Stream) { + assert(VE.shouldPreserveUseListOrder() && + "Expected to be preserving use-list order"); + auto hasMore = [&]() { return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F; }; @@ -2089,7 +2093,7 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE, bool NeedsMetadataAttachment = false; - DebugLoc LastDL; + MDLocation *LastDL = nullptr; // Finally, emit all the instructions, in order. for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) @@ -2104,26 +2108,22 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE, NeedsMetadataAttachment |= I->hasMetadataOtherThanDebugLoc(); // If the instruction has a debug location, emit it. - DebugLoc DL = I->getDebugLoc(); - if (DL.isUnknown()) { - // nothing todo. - } else if (DL == LastDL) { + MDLocation *DL = I->getDebugLoc(); + if (!DL) + continue; + + if (DL == LastDL) { // Just repeat the same debug loc as last time. Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals); - } else { - MDNode *Scope, *IA; - DL.getScopeAndInlinedAt(Scope, IA, I->getContext()); - assert(Scope && "Expected valid scope"); - - Vals.push_back(DL.getLine()); - Vals.push_back(DL.getCol()); - Vals.push_back(VE.getMetadataOrNullID(Scope)); - Vals.push_back(VE.getMetadataOrNullID(IA)); - Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals); - Vals.clear(); - - LastDL = DL; + continue; } + + Vals.push_back(DL->getLine()); + Vals.push_back(DL->getColumn()); + Vals.push_back(VE.getMetadataOrNullID(DL->getScope())); + Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt())); + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals); + Vals.clear(); } // Emit names for all the instructions etc. @@ -2131,7 +2131,7 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE, if (NeedsMetadataAttachment) WriteMetadataAttachment(F, VE, Stream); - if (shouldPreserveBitcodeUseListOrder()) + if (VE.shouldPreserveUseListOrder()) WriteUseListBlock(&F, VE, Stream); VE.purgeFunction(); Stream.ExitBlock(); @@ -2313,7 +2313,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { } /// WriteModule - Emit the specified module to the bitstream. -static void WriteModule(const Module *M, BitstreamWriter &Stream) { +static void WriteModule(const Module *M, BitstreamWriter &Stream, + bool ShouldPreserveUseListOrder) { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); SmallVector Vals; @@ -2322,7 +2323,7 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream) { Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); // Analyze the module, enumerating globals, functions, etc. - ValueEnumerator VE(*M); + ValueEnumerator VE(*M, ShouldPreserveUseListOrder); // Emit blockinfo, which defines the standard abbreviations etc. WriteBlockInfo(VE, Stream); @@ -2355,7 +2356,7 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream) { WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream); // Emit module-level use-lists. - if (shouldPreserveBitcodeUseListOrder()) + if (VE.shouldPreserveUseListOrder()) WriteUseListBlock(nullptr, VE, Stream); // Emit function bodies. @@ -2441,7 +2442,8 @@ static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl &Buffer, /// WriteBitcodeToFile - Write the specified module to the specified output /// stream. -void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out) { +void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, + bool ShouldPreserveUseListOrder) { SmallVector Buffer; Buffer.reserve(256*1024); @@ -2464,7 +2466,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out) { Stream.Emit(0xD, 4); // Emit the module. - WriteModule(M, Stream); + WriteModule(M, Stream, ShouldPreserveUseListOrder); } if (TT.isOSDarwin()) diff --git a/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 25456a4..3165743 100644 --- a/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -19,22 +19,25 @@ using namespace llvm; PreservedAnalyses BitcodeWriterPass::run(Module &M) { - WriteBitcodeToFile(&M, OS); + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder); return PreservedAnalyses::all(); } namespace { class WriteBitcodePass : public ModulePass { raw_ostream &OS; // raw_ostream to print on + bool ShouldPreserveUseListOrder; + public: static char ID; // Pass identification, replacement for typeid - explicit WriteBitcodePass(raw_ostream &o) - : ModulePass(ID), OS(o) {} + explicit WriteBitcodePass(raw_ostream &o, bool ShouldPreserveUseListOrder) + : ModulePass(ID), OS(o), + ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {} const char *getPassName() const override { return "Bitcode Writer"; } bool runOnModule(Module &M) override { - WriteBitcodeToFile(&M, OS); + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder); return false; } }; @@ -42,6 +45,7 @@ namespace { char WriteBitcodePass::ID = 0; -ModulePass *llvm::createBitcodeWriterPass(raw_ostream &Str) { - return new WriteBitcodePass(Str); +ModulePass *llvm::createBitcodeWriterPass(raw_ostream &Str, + bool ShouldPreserveUseListOrder) { + return new WriteBitcodePass(Str, ShouldPreserveUseListOrder); } diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp index 549e94f..7f576d7 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -283,9 +283,11 @@ static bool isIntOrIntVectorValue(const std::pair &V) { return V.first->getType()->isIntOrIntVectorTy(); } -ValueEnumerator::ValueEnumerator(const Module &M) - : HasMDString(false), HasMDLocation(false), HasGenericDebugNode(false) { - if (shouldPreserveBitcodeUseListOrder()) +ValueEnumerator::ValueEnumerator(const Module &M, + bool ShouldPreserveUseListOrder) + : HasMDString(false), HasMDLocation(false), HasGenericDebugNode(false), + ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) { + if (ShouldPreserveUseListOrder) UseListOrders = predictUseListOrder(M); // Enumerate the global variables. @@ -373,12 +375,10 @@ ValueEnumerator::ValueEnumerator(const Module &M) for (unsigned i = 0, e = MDs.size(); i != e; ++i) EnumerateMetadata(MDs[i].second); - if (!I.getDebugLoc().isUnknown()) { - MDNode *Scope, *IA; - I.getDebugLoc().getScopeAndInlinedAt(Scope, IA, I.getContext()); - if (Scope) EnumerateMetadata(Scope); - if (IA) EnumerateMetadata(IA); - } + // Don't enumerate the location directly -- it has a special record + // type -- but enumerate its operands. + if (MDLocation *L = I.getDebugLoc()) + EnumerateMDNodeOperands(L); } } @@ -463,7 +463,7 @@ void ValueEnumerator::print(raw_ostream &OS, const MetadataMapType &Map, void ValueEnumerator::OptimizeConstants(unsigned CstStart, unsigned CstEnd) { if (CstStart == CstEnd || CstStart+1 == CstEnd) return; - if (shouldPreserveBitcodeUseListOrder()) + if (ShouldPreserveUseListOrder) // Optimizing constants makes the use-list order difficult to predict. // Disable it for now when trying to preserve the order. return; diff --git a/lib/Bitcode/Writer/ValueEnumerator.h b/lib/Bitcode/Writer/ValueEnumerator.h index b94c370..ba245a3 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.h +++ b/lib/Bitcode/Writer/ValueEnumerator.h @@ -67,6 +67,7 @@ private: bool HasMDString; bool HasMDLocation; bool HasGenericDebugNode; + bool ShouldPreserveUseListOrder; typedef DenseMap AttributeGroupMapType; AttributeGroupMapType AttributeGroupMap; @@ -102,7 +103,7 @@ private: ValueEnumerator(const ValueEnumerator &) = delete; void operator=(const ValueEnumerator &) = delete; public: - ValueEnumerator(const Module &M); + ValueEnumerator(const Module &M, bool ShouldPreserveUseListOrder); void dump() const; void print(raw_ostream &OS, const ValueMapType &Map, const char *Name) const; @@ -123,6 +124,8 @@ public: bool hasMDLocation() const { return HasMDLocation; } bool hasGenericDebugNode() const { return HasGenericDebugNode; } + bool shouldPreserveUseListOrder() const { return ShouldPreserveUseListOrder; } + unsigned getTypeID(Type *T) const { TypeMapType::const_iterator I = TypeMap.find(T); assert(I != TypeMap.end() && "Type not in ValueEnumerator!"); diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.h b/lib/CodeGen/AggressiveAntiDepBreaker.h index 12cf95b..f9544dd 100644 --- a/lib/CodeGen/AggressiveAntiDepBreaker.h +++ b/lib/CodeGen/AggressiveAntiDepBreaker.h @@ -127,7 +127,7 @@ class RegisterClassInfo; AggressiveAntiDepBreaker(MachineFunction& MFi, const RegisterClassInfo &RCI, TargetSubtargetInfo::RegClassVector& CriticalPathRCs); - ~AggressiveAntiDepBreaker(); + ~AggressiveAntiDepBreaker() override; /// Initialize anti-dep breaking for a new basic block. void StartBlock(MachineBasicBlock *BB) override; diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 07d6731..43d7a38 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -28,7 +28,7 @@ #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Mangler.h" @@ -671,17 +671,17 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) { OS << "DEBUG_VALUE: "; DIVariable V = MI->getDebugVariable(); - if (V.getContext().isSubprogram()) { - StringRef Name = DISubprogram(V.getContext()).getDisplayName(); + if (auto *SP = dyn_cast(V->getScope())) { + StringRef Name = SP->getDisplayName(); if (!Name.empty()) OS << Name << ":"; } - OS << V.getName(); + OS << V->getName(); DIExpression Expr = MI->getDebugExpression(); - if (Expr.isBitPiece()) - OS << " [bit_piece offset=" << Expr.getBitPieceOffset() - << " size=" << Expr.getBitPieceSize() << "]"; + if (Expr->isBitPiece()) + OS << " [bit_piece offset=" << Expr->getBitPieceOffset() + << " size=" << Expr->getBitPieceSize() << "]"; OS << " <- "; // The second operand is only an offset if it's an immediate. @@ -1034,11 +1034,31 @@ bool AsmPrinter::doFinalization(Module &M) { EmitVisibility(Name, V, false); } + const TargetLoweringObjectFile &TLOF = getObjFileLowering(); + // Emit module flags. SmallVector ModuleFlags; M.getModuleFlagsMetadata(ModuleFlags); if (!ModuleFlags.empty()) - getObjFileLowering().emitModuleFlags(OutStreamer, ModuleFlags, *Mang, TM); + TLOF.emitModuleFlags(OutStreamer, ModuleFlags, *Mang, TM); + + Triple TT(TM.getTargetTriple()); + if (TT.isOSBinFormatELF()) { + MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo(); + + // Output stubs for external and common global variables. + MachineModuleInfoELF::SymbolListTy Stubs = MMIELF.GetGVStubList(); + if (!Stubs.empty()) { + OutStreamer.SwitchSection(TLOF.getDataRelSection()); + const DataLayout *DL = TM.getDataLayout(); + + for (const auto &Stub : Stubs) { + OutStreamer.EmitLabel(Stub.first); + OutStreamer.EmitSymbolValue(Stub.second.getPointer(), + DL->getPointerSize()); + } + } + } // Make sure we wrote out everything we need. OutStreamer.Flush(); @@ -2302,7 +2322,7 @@ MCSymbol *AsmPrinter::getSymbolWithGlobalValueBase(const GlobalValue *GV, MCSymbol *AsmPrinter::GetExternalSymbolSymbol(StringRef Sym) const { SmallString<60> NameStr; Mang->getNameWithPrefix(NameStr, Sym); - return OutContext.GetOrCreateSymbol(NameStr.str()); + return OutContext.GetOrCreateSymbol(NameStr); } diff --git a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp index bbdf237..1e3c5d7 100644 --- a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp +++ b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp @@ -33,7 +33,7 @@ static unsigned isDescribedByReg(const MachineInstr &MI) { return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : 0; } -void DbgValueHistoryMap::startInstrRange(const MDNode *Var, +void DbgValueHistoryMap::startInstrRange(InlinedVariable Var, const MachineInstr &MI) { // Instruction range should start with a DBG_VALUE instruction for the // variable. @@ -48,7 +48,7 @@ void DbgValueHistoryMap::startInstrRange(const MDNode *Var, Ranges.push_back(std::make_pair(&MI, nullptr)); } -void DbgValueHistoryMap::endInstrRange(const MDNode *Var, +void DbgValueHistoryMap::endInstrRange(InlinedVariable Var, const MachineInstr &MI) { auto &Ranges = VarInstrRanges[Var]; // Verify that the current instruction range is not yet closed. @@ -59,7 +59,7 @@ void DbgValueHistoryMap::endInstrRange(const MDNode *Var, Ranges.back().second = &MI; } -unsigned DbgValueHistoryMap::getRegisterForVar(const MDNode *Var) const { +unsigned DbgValueHistoryMap::getRegisterForVar(InlinedVariable Var) const { const auto &I = VarInstrRanges.find(Var); if (I == VarInstrRanges.end()) return 0; @@ -71,12 +71,13 @@ unsigned DbgValueHistoryMap::getRegisterForVar(const MDNode *Var) const { namespace { // Maps physreg numbers to the variables they describe. -typedef std::map> RegDescribedVarsMap; +typedef DbgValueHistoryMap::InlinedVariable InlinedVariable; +typedef std::map> RegDescribedVarsMap; } // \brief Claim that @Var is not described by @RegNo anymore. -static void dropRegDescribedVar(RegDescribedVarsMap &RegVars, - unsigned RegNo, const MDNode *Var) { +static void dropRegDescribedVar(RegDescribedVarsMap &RegVars, unsigned RegNo, + InlinedVariable Var) { const auto &I = RegVars.find(RegNo); assert(RegNo != 0U && I != RegVars.end()); auto &VarSet = I->second; @@ -89,8 +90,8 @@ static void dropRegDescribedVar(RegDescribedVarsMap &RegVars, } // \brief Claim that @Var is now described by @RegNo. -static void addRegDescribedVar(RegDescribedVarsMap &RegVars, - unsigned RegNo, const MDNode *Var) { +static void addRegDescribedVar(RegDescribedVarsMap &RegVars, unsigned RegNo, + InlinedVariable Var) { assert(RegNo != 0U); auto &VarSet = RegVars[RegNo]; assert(std::find(VarSet.begin(), VarSet.end(), Var) == VarSet.end()); @@ -203,7 +204,10 @@ void llvm::calculateDbgValueHistory(const MachineFunction *MF, // Use the base variable (without any DW_OP_piece expressions) // as index into History. The full variables including the // piece expressions are attached to the MI. - DIVariable Var = MI.getDebugVariable(); + MDLocalVariable *RawVar = MI.getDebugVariable(); + assert(RawVar->isValidLocationForIntrinsic(MI.getDebugLoc()) && + "Expected inlined-at fields to agree"); + InlinedVariable Var(RawVar, MI.getDebugLoc()->getInlinedAt()); if (unsigned PrevReg = Result.getRegisterForVar(Var)) dropRegDescribedVar(RegVars, PrevReg, Var); diff --git a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h index 4b62007..c25aaff 100644 --- a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h +++ b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h @@ -17,7 +17,8 @@ namespace llvm { class MachineFunction; class MachineInstr; -class MDNode; +class MDLocalVariable; +class MDLocation; class TargetRegisterInfo; // For each user variable, keep a list of instruction ranges where this variable @@ -31,16 +32,19 @@ class DbgValueHistoryMap { public: typedef std::pair InstrRange; typedef SmallVector InstrRanges; - typedef MapVector InstrRangesMap; + typedef std::pair + InlinedVariable; + typedef MapVector InstrRangesMap; + private: InstrRangesMap VarInstrRanges; public: - void startInstrRange(const MDNode *Var, const MachineInstr &MI); - void endInstrRange(const MDNode *Var, const MachineInstr &MI); + void startInstrRange(InlinedVariable Var, const MachineInstr &MI); + void endInstrRange(InlinedVariable Var, const MachineInstr &MI); // Returns register currently describing @Var. If @Var is currently // unaccessible or is not described by a register, returns 0. - unsigned getRegisterForVar(const MDNode *Var) const; + unsigned getRegisterForVar(InlinedVariable Var) const; bool empty() const { return VarInstrRanges.empty(); } void clear() { VarInstrRanges.clear(); } diff --git a/lib/CodeGen/AsmPrinter/DebugLocEntry.h b/lib/CodeGen/AsmPrinter/DebugLocEntry.h index 6914bbe..4f6714e 100644 --- a/lib/CodeGen/AsmPrinter/DebugLocEntry.h +++ b/lib/CodeGen/AsmPrinter/DebugLocEntry.h @@ -42,8 +42,8 @@ public: } Value(const MDNode *Var, const MDNode *Expr, MachineLocation Loc) : Variable(Var), Expression(Expr), EntryKind(E_Location), Loc(Loc) { - assert(DIVariable(Var).Verify()); - assert(DIExpression(Expr)->isValid()); + assert(isa(Var)); + assert(cast(Expr)->isValid()); } /// The variable to which this location entry corresponds. @@ -74,10 +74,11 @@ public: const ConstantFP *getConstantFP() const { return Constant.CFP; } const ConstantInt *getConstantInt() const { return Constant.CIP; } MachineLocation getLoc() const { return Loc; } - const MDNode *getVariableNode() const { return Variable; } - DIVariable getVariable() const { return DIVariable(Variable); } - bool isBitPiece() const { return getExpression().isBitPiece(); } - DIExpression getExpression() const { return DIExpression(Expression); } + DIVariable getVariable() const { return cast(Variable); } + bool isBitPiece() const { return getExpression()->isBitPiece(); } + DIExpression getExpression() const { + return cast_or_null(Expression); + } friend bool operator==(const Value &, const Value &); friend bool operator<(const Value &, const Value &); }; @@ -101,12 +102,12 @@ public: /// Return true if the merge was successful. bool MergeValues(const DebugLocEntry &Next) { if (Begin == Next.Begin) { - DIExpression Expr(Values[0].Expression); - DIVariable Var(Values[0].Variable); - DIExpression NextExpr(Next.Values[0].Expression); - DIVariable NextVar(Next.Values[0].Variable); - if (Var == NextVar && Expr.isBitPiece() && - NextExpr.isBitPiece()) { + DIExpression Expr = cast_or_null(Values[0].Expression); + DIVariable Var = cast_or_null(Values[0].Variable); + DIExpression NextExpr = + cast_or_null(Next.Values[0].Expression); + DIVariable NextVar = cast_or_null(Next.Values[0].Variable); + if (Var == NextVar && Expr->isBitPiece() && NextExpr->isBitPiece()) { addValues(Next.Values); End = Next.End; return true; @@ -189,8 +190,8 @@ inline bool operator==(const DebugLocEntry::Value &A, /// \brief Compare two pieces based on their offset. inline bool operator<(const DebugLocEntry::Value &A, const DebugLocEntry::Value &B) { - return A.getExpression().getBitPieceOffset() < - B.getExpression().getBitPieceOffset(); + return A.getExpression()->getBitPieceOffset() < + B.getExpression()->getBitPieceOffset(); } } diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index eee5fc5..75d3b68 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -101,51 +101,52 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) { if (DIE *Die = getDIE(GV)) return Die; - assert(GV.isGlobalVariable()); + assert(GV); - DIScope GVContext = GV.getContext(); - DIType GTy = DD->resolve(GV.getType()); + DIScope GVContext = GV->getScope(); + DIType GTy = DD->resolve(GV->getType()); // Construct the context before querying for the existence of the DIE in // case such construction creates the DIE. DIE *ContextDIE = getOrCreateContextDIE(GVContext); // Add to map. - DIE *VariableDIE = &createAndAddDIE(GV.getTag(), *ContextDIE, GV); + DIE *VariableDIE = &createAndAddDIE(GV->getTag(), *ContextDIE, GV); DIScope DeclContext; - if (DIDerivedType SDMDecl = GV.getStaticDataMemberDeclaration()) { - DeclContext = resolve(SDMDecl.getContext()); - assert(SDMDecl.isStaticMember() && "Expected static member decl"); - assert(GV.isDefinition()); + if (auto *SDMDecl = GV->getStaticDataMemberDeclaration()) { + DeclContext = resolve(SDMDecl->getScope()); + assert(SDMDecl->isStaticMember() && "Expected static member decl"); + assert(GV->isDefinition()); // We need the declaration DIE that is in the static member's class. DIE *VariableSpecDIE = getOrCreateStaticMemberDIE(SDMDecl); addDIEEntry(*VariableDIE, dwarf::DW_AT_specification, *VariableSpecDIE); } else { - DeclContext = GV.getContext(); + DeclContext = GV->getScope(); // Add name and type. - addString(*VariableDIE, dwarf::DW_AT_name, GV.getDisplayName()); + addString(*VariableDIE, dwarf::DW_AT_name, GV->getDisplayName()); addType(*VariableDIE, GTy); // Add scoping info. - if (!GV.isLocalToUnit()) + if (!GV->isLocalToUnit()) addFlag(*VariableDIE, dwarf::DW_AT_external); // Add line number info. addSourceLine(*VariableDIE, GV); } - if (!GV.isDefinition()) + if (!GV->isDefinition()) addFlag(*VariableDIE, dwarf::DW_AT_declaration); + else + addGlobalName(GV->getName(), *VariableDIE, DeclContext); // Add location. bool addToAccelTable = false; - bool isGlobalVariable = GV.getGlobal() != nullptr; - if (isGlobalVariable) { + if (auto *Global = dyn_cast_or_null(GV->getVariable())) { addToAccelTable = true; DIELoc *Loc = new (DIEValueAllocator) DIELoc(); - const MCSymbol *Sym = Asm->getSymbol(GV.getGlobal()); - if (GV.getGlobal()->isThreadLocal()) { + const MCSymbol *Sym = Asm->getSymbol(Global); + if (Global->isThreadLocal()) { // FIXME: Make this work with -gsplit-dwarf. unsigned PointerSize = Asm->getDataLayout().getPointerSize(); assert((PointerSize == 4 || PointerSize == 8) && @@ -174,11 +175,11 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) { } addBlock(*VariableDIE, dwarf::DW_AT_location, Loc); - addLinkageName(*VariableDIE, GV.getLinkageName()); + addLinkageName(*VariableDIE, GV->getLinkageName()); } else if (const ConstantInt *CI = - dyn_cast_or_null(GV.getConstant())) { + dyn_cast_or_null(GV->getVariable())) { addConstantValue(*VariableDIE, CI, GTy); - } else if (const ConstantExpr *CE = getMergedGlobalExpr(GV.getConstant())) { + } else if (const ConstantExpr *CE = getMergedGlobalExpr(GV->getVariable())) { addToAccelTable = true; // GV is a merged global. DIELoc *Loc = new (DIEValueAllocator) DIELoc(); @@ -195,15 +196,14 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) { } if (addToAccelTable) { - DD->addAccelName(GV.getName(), *VariableDIE); + DD->addAccelName(GV->getName(), *VariableDIE); // If the linkage name is different than the name, go ahead and output // that as well into the name table. - if (GV.getLinkageName() != "" && GV.getName() != GV.getLinkageName()) - DD->addAccelName(GV.getLinkageName(), *VariableDIE); + if (GV->getLinkageName() != "" && GV->getName() != GV->getLinkageName()) + DD->addAccelName(GV->getLinkageName(), *VariableDIE); } - addGlobalName(GV.getName(), *VariableDIE, DeclContext); return VariableDIE; } @@ -307,7 +307,7 @@ void DwarfCompileUnit::constructScopeDIE( DIScope DS(Scope->getScopeNode()); - assert((Scope->getInlinedAt() || !DS.isSubprogram()) && + assert((Scope->getInlinedAt() || !isa(DS)) && "Only handle inlined subprograms here, use " "constructSubprogramScopeDIE for non-inlined " "subprograms"); @@ -318,7 +318,7 @@ void DwarfCompileUnit::constructScopeDIE( // avoid creating un-used children then removing them later when we find out // the scope DIE is null. std::unique_ptr ScopeDIE; - if (Scope->getParent() && DS.isSubprogram()) { + if (Scope->getParent() && isa(DS)) { ScopeDIE = constructInlinedScopeDIE(Scope); if (!ScopeDIE) return; @@ -340,7 +340,7 @@ void DwarfCompileUnit::constructScopeDIE( // There is no need to emit empty lexical block DIE. for (const auto &E : DD->findImportedEntitiesForScope(DS)) Children.push_back( - constructImportedEntityDIE(DIImportedEntity(E.second))); + constructImportedEntityDIE(cast(E.second))); } // If there are only other scopes as children, put them directly in the @@ -431,10 +431,10 @@ DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope) { attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges()); // Add the call site information to the DIE. - DILocation DL(Scope->getInlinedAt()); + const MDLocation *IA = Scope->getInlinedAt(); addUInt(*ScopeDIE, dwarf::DW_AT_call_file, None, - getOrCreateSourceID(DL.getFilename(), DL.getDirectory())); - addUInt(*ScopeDIE, dwarf::DW_AT_call_line, None, DL.getLineNumber()); + getOrCreateSourceID(IA->getFilename(), IA->getDirectory())); + addUInt(*ScopeDIE, dwarf::DW_AT_call_line, None, IA->getLine()); // Add name to the name table, we do this here because we're guaranteed // to have concrete versions of our DW_TAG_inlined_subprogram nodes. @@ -523,7 +523,7 @@ DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV, assert(Expr != DV.getExpression().end() && "Wrong number of expressions"); DwarfExpr.AddMachineRegIndirect(FrameReg, Offset); - DwarfExpr.AddExpression(Expr->begin(), Expr->end()); + DwarfExpr.AddExpression((*Expr)->expr_op_begin(), (*Expr)->expr_op_end()); ++Expr; } addBlock(*VariableDie, dwarf::DW_AT_location, Loc); @@ -562,16 +562,14 @@ void DwarfCompileUnit::constructSubprogramScopeDIE(LexicalScope *Scope) { assert(Scope && Scope->getScopeNode()); assert(!Scope->getInlinedAt()); assert(!Scope->isAbstractScope()); - DISubprogram Sub(Scope->getScopeNode()); - - assert(Sub.isSubprogram()); + DISubprogram Sub = cast(Scope->getScopeNode()); DD->getProcessedSPNodes().insert(Sub); DIE &ScopeDIE = updateSubprogramScopeDIE(Sub); // If this is a variadic function, add an unspecified parameter. - DITypeArray FnArgs = Sub.getType().getTypeArray(); + DITypeArray FnArgs = Sub->getType()->getTypeArray(); // Collect lexical scope children first. // ObjectPointer might be a local (non-argument) local variable if it's a @@ -582,8 +580,7 @@ void DwarfCompileUnit::constructSubprogramScopeDIE(LexicalScope *Scope) { // If we have a single element of null, it is a function that returns void. // If we have more than one elements and the last one is null, it is a // variadic function. - if (FnArgs.getNumElements() > 1 && - !FnArgs.getElement(FnArgs.getNumElements() - 1) && + if (FnArgs.size() > 1 && !FnArgs[FnArgs.size() - 1] && !includeMinimalInlineScopes()) ScopeDIE.addChild(make_unique(dwarf::DW_TAG_unspecified_parameters)); } @@ -607,7 +604,7 @@ DwarfCompileUnit::constructAbstractSubprogramScopeDIE(LexicalScope *Scope) { if (AbsDef) return; - DISubprogram SP(Scope->getScopeNode()); + DISubprogram SP = cast(Scope->getScopeNode()); DIE *ContextDIE; @@ -617,11 +614,11 @@ DwarfCompileUnit::constructAbstractSubprogramScopeDIE(LexicalScope *Scope) { // the important distinction that the DIDescriptor is not associated with the // DIE (since the DIDescriptor will be associated with the concrete DIE, if // any). It could be refactored to some common utility function. - else if (DISubprogram SPDecl = SP.getFunctionDeclaration()) { + else if (auto *SPDecl = SP->getDeclaration()) { ContextDIE = &getUnitDie(); getOrCreateSubprogramDIE(SPDecl); } else - ContextDIE = getOrCreateContextDIE(resolve(SP.getContext())); + ContextDIE = getOrCreateContextDIE(resolve(SP->getScope())); // Passing null as the associated DIDescriptor because the abstract definition // shouldn't be found by lookup. @@ -637,28 +634,25 @@ DwarfCompileUnit::constructAbstractSubprogramScopeDIE(LexicalScope *Scope) { std::unique_ptr DwarfCompileUnit::constructImportedEntityDIE(const DIImportedEntity &Module) { - assert(Module.Verify() && - "Use one of the MDNode * overloads to handle invalid metadata"); - std::unique_ptr IMDie = make_unique((dwarf::Tag)Module.getTag()); + std::unique_ptr IMDie = make_unique((dwarf::Tag)Module->getTag()); insertDIE(Module, IMDie.get()); DIE *EntityDie; - DIDescriptor Entity = resolve(Module.getEntity()); - if (Entity.isNameSpace()) - EntityDie = getOrCreateNameSpace(DINameSpace(Entity)); - else if (Entity.isSubprogram()) - EntityDie = getOrCreateSubprogramDIE(DISubprogram(Entity)); - else if (Entity.isType()) - EntityDie = getOrCreateTypeDIE(DIType(Entity)); - else if (Entity.isGlobalVariable()) - EntityDie = getOrCreateGlobalVariableDIE(DIGlobalVariable(Entity)); + auto *Entity = resolve(Module->getEntity()); + if (auto *NS = dyn_cast(Entity)) + EntityDie = getOrCreateNameSpace(NS); + else if (auto *SP = dyn_cast(Entity)) + EntityDie = getOrCreateSubprogramDIE(SP); + else if (auto *T = dyn_cast(Entity)) + EntityDie = getOrCreateTypeDIE(T); + else if (auto *GV = dyn_cast(Entity)) + EntityDie = getOrCreateGlobalVariableDIE(GV); else EntityDie = getDIE(Entity); assert(EntityDie); - addSourceLine(*IMDie, Module.getLineNumber(), - Module.getContext().getFilename(), - Module.getContext().getDirectory()); + addSourceLine(*IMDie, Module->getLine(), Module->getScope()->getFilename(), + Module->getScope()->getDirectory()); addDIEEntry(*IMDie, dwarf::DW_AT_import, *EntityDie); - StringRef Name = Module.getName(); + StringRef Name = Module->getName(); if (!Name.empty()) addString(*IMDie, dwarf::DW_AT_name, Name); @@ -683,21 +677,19 @@ void DwarfCompileUnit::finishSubprogramDefinition(DISubprogram SP) { } } void DwarfCompileUnit::collectDeadVariables(DISubprogram SP) { - assert(SP.isSubprogram() && "CU's subprogram list contains a non-subprogram"); - assert(SP.isDefinition() && + assert(SP && "CU's subprogram list contains a non-subprogram"); + assert(SP->isDefinition() && "CU's subprogram list contains a subprogram declaration"); - DIArray Variables = SP.getVariables(); - if (Variables.getNumElements() == 0) + auto Variables = SP->getVariables(); + if (Variables.size() == 0) return; DIE *SPDIE = DU->getAbstractSPDies().lookup(SP); if (!SPDIE) SPDIE = getDIE(SP); assert(SPDIE); - for (unsigned vi = 0, ve = Variables.getNumElements(); vi != ve; ++vi) { - DIVariable DV(Variables.getElement(vi)); - assert(DV.isVariable()); - DbgVariable NewVar(DV, DIExpression(), DD); + for (DIVariable DV : Variables) { + DbgVariable NewVar(DV, nullptr, DIExpression(), DD); auto VariableDie = constructVariableDIE(NewVar); applyVariableAttributes(NewVar, *VariableDie); SPDIE->addChild(std::move(VariableDie)); @@ -728,7 +720,7 @@ void DwarfCompileUnit::addGlobalType(DIType Ty, const DIE &Die, DIScope Context) { if (includeMinimalInlineScopes()) return; - std::string FullName = getParentContextString(Context) + Ty.getName().str(); + std::string FullName = getParentContextString(Context) + Ty->getName().str(); GlobalTypes[FullName] = &Die; } @@ -778,7 +770,7 @@ void DwarfCompileUnit::addComplexAddress(const DbgVariable &DV, DIE &Die, ValidReg = DwarfExpr.AddMachineRegIndirect(Location.getReg(), Location.getOffset()); if (ValidReg) - DwarfExpr.AddExpression(Expr.begin(), Expr.end()); + DwarfExpr.AddExpression(Expr->expr_op_begin(), Expr->expr_op_end()); } else ValidReg = DwarfExpr.AddMachineRegExpression(Expr, Location.getReg()); @@ -816,10 +808,10 @@ void DwarfCompileUnit::addExpr(DIELoc &Die, dwarf::Form Form, void DwarfCompileUnit::applySubprogramAttributesToDefinition(DISubprogram SP, DIE &SPDie) { - DISubprogram SPDecl = SP.getFunctionDeclaration(); - DIScope Context = resolve(SPDecl ? SPDecl.getContext() : SP.getContext()); + auto *SPDecl = SP->getDeclaration(); + DIScope Context = resolve(SPDecl ? SPDecl->getScope() : SP->getScope()); applySubprogramAttributes(SP, SPDie, includeMinimalInlineScopes()); - addGlobalName(SP.getName(), SPDie, Context); + addGlobalName(SP->getName(), SPDie, Context); } bool DwarfCompileUnit::isDwoUnit() const { @@ -827,7 +819,7 @@ bool DwarfCompileUnit::isDwoUnit() const { } bool DwarfCompileUnit::includeMinimalInlineScopes() const { - return getCUNode().getEmissionKind() == DIBuilder::LineTablesOnly || + return getCUNode()->getEmissionKind() == DIBuilder::LineTablesOnly || (DD->useSplitDwarf() && !Skeleton); } } // end llvm namespace diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index e9ebd97..fb8fc6e 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -129,20 +129,22 @@ bool DebugLocDwarfExpression::isFrameRegister(unsigned MachineReg) { /// resolve - Look in the DwarfDebug map for the MDNode that /// corresponds to the reference. -template T DbgVariable::resolve(DIRef Ref) const { +template T *DbgVariable::resolve(TypedDebugNodeRef Ref) const { return DD->resolve(Ref); } bool DbgVariable::isBlockByrefVariable() const { - assert(Var.isVariable() && "Invalid complex DbgVariable!"); - return Var.isBlockByrefVariable(DD->getTypeIdentifierMap()); + assert(Var && "Invalid complex DbgVariable!"); + return Var->getType() + .resolve(DD->getTypeIdentifierMap()) + ->isBlockByrefStruct(); } DIType DbgVariable::getType() const { - DIType Ty = Var.getType().resolve(DD->getTypeIdentifierMap()); + MDType *Ty = Var->getType().resolve(DD->getTypeIdentifierMap()); // FIXME: isBlockByrefVariable should be reformulated in terms of complex // addresses instead. - if (Var.isBlockByrefVariable(DD->getTypeIdentifierMap())) { + if (Ty->isBlockByrefStruct()) { /* Byref variables, in Blocks, are declared by the programmer as "SomeType VarName;", but the compiler creates a __Block_byref_x_VarName struct, and gives the variable VarName @@ -167,17 +169,17 @@ DIType DbgVariable::getType() const { have a DW_AT_location that tells the debugger how to unwind through the pointers and __Block_byref_x_VarName struct to find the actual value of the variable. The function addBlockByrefType does this. */ - DIType subType = Ty; - uint16_t tag = Ty.getTag(); + MDType *subType = Ty; + uint16_t tag = Ty->getTag(); if (tag == dwarf::DW_TAG_pointer_type) - subType = resolve(DIDerivedType(Ty).getTypeDerivedFrom()); + subType = resolve(DITypeRef(cast(Ty)->getBaseType())); - DIArray Elements = DICompositeType(subType).getElements(); - for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { - DIDerivedType DT(Elements.getElement(i)); - if (getName() == DT.getName()) - return (resolve(DT.getTypeDerivedFrom())); + auto Elements = cast(subType)->getElements(); + for (unsigned i = 0, N = Elements.size(); i < N; ++i) { + auto *DT = cast(Elements[i]); + if (getName() == DT->getName()) + return resolve(DITypeRef(DT->getBaseType())); } } return Ty; @@ -275,25 +277,25 @@ static StringRef getObjCMethodName(StringRef In) { // that do not have a DW_AT_name or DW_AT_linkage_name field - this // is only slightly different than the lookup of non-standard ObjC names. void DwarfDebug::addSubprogramNames(DISubprogram SP, DIE &Die) { - if (!SP.isDefinition()) + if (!SP->isDefinition()) return; - addAccelName(SP.getName(), Die); + addAccelName(SP->getName(), Die); // If the linkage name is different than the name, go ahead and output // that as well into the name table. - if (SP.getLinkageName() != "" && SP.getName() != SP.getLinkageName()) - addAccelName(SP.getLinkageName(), Die); + if (SP->getLinkageName() != "" && SP->getName() != SP->getLinkageName()) + addAccelName(SP->getLinkageName(), Die); // If this is an Objective-C selector name add it to the ObjC accelerator // too. - if (isObjCClass(SP.getName())) { + if (isObjCClass(SP->getName())) { StringRef Class, Category; - getObjCClassCategory(SP.getName(), Class, Category); + getObjCClassCategory(SP->getName(), Class, Category); addAccelObjC(Class, Die); if (Category != "") addAccelObjC(Category, Die); // Also add the base method name to the name table. - addAccelName(getObjCMethodName(SP.getName()), Die); + addAccelName(getObjCMethodName(SP->getName()), Die); } } @@ -302,11 +304,10 @@ void DwarfDebug::addSubprogramNames(DISubprogram SP, DIE &Die) { bool DwarfDebug::isSubprogramContext(const MDNode *Context) { if (!Context) return false; - DIDescriptor D(Context); - if (D.isSubprogram()) + if (isa(Context)) return true; - if (D.isType()) - return isSubprogramContext(resolve(DIType(Context).getContext())); + if (auto *T = dyn_cast(Context)) + return isSubprogramContext(resolve(T->getScope())); return false; } @@ -362,8 +363,8 @@ void DwarfDebug::addGnuPubAttributes(DwarfUnit &U, DIE &D) const { // Create new DwarfCompileUnit for the given metadata node with tag // DW_TAG_compile_unit. DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) { - StringRef FN = DIUnit.getFilename(); - CompilationDir = DIUnit.getDirectory(); + StringRef FN = DIUnit->getFilename(); + CompilationDir = DIUnit->getDirectory(); auto OwnedUnit = make_unique( InfoHolder.getUnits().size(), DIUnit, Asm, this, &InfoHolder); @@ -381,9 +382,9 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) { Asm->OutStreamer.getContext().setMCLineTableCompilationDir( NewCU.getUniqueID(), CompilationDir); - NewCU.addString(Die, dwarf::DW_AT_producer, DIUnit.getProducer()); + NewCU.addString(Die, dwarf::DW_AT_producer, DIUnit->getProducer()); NewCU.addUInt(Die, dwarf::DW_AT_language, dwarf::DW_FORM_data2, - DIUnit.getLanguage()); + DIUnit->getSourceLanguage()); NewCU.addString(Die, dwarf::DW_AT_name, FN); if (!useSplitDwarf()) { @@ -397,14 +398,14 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) { addGnuPubAttributes(NewCU, Die); } - if (DIUnit.isOptimized()) + if (DIUnit->isOptimized()) NewCU.addFlag(Die, dwarf::DW_AT_APPLE_optimized); - StringRef Flags = DIUnit.getFlags(); + StringRef Flags = DIUnit->getFlags(); if (!Flags.empty()) NewCU.addString(Die, dwarf::DW_AT_APPLE_flags, Flags); - if (unsigned RVer = DIUnit.getRunTimeVersion()) + if (unsigned RVer = DIUnit->getRuntimeVersion()) NewCU.addUInt(Die, dwarf::DW_AT_APPLE_major_runtime_vers, dwarf::DW_FORM_data1, RVer); @@ -420,9 +421,8 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) { void DwarfDebug::constructAndAddImportedEntityDIE(DwarfCompileUnit &TheCU, const MDNode *N) { - DIImportedEntity Module(N); - assert(Module.Verify()); - if (DIE *D = TheCU.getOrCreateContextDIE(Module.getContext())) + DIImportedEntity Module = cast(N); + if (DIE *D = TheCU.getOrCreateContextDIE(Module->getScope())) D->addChild(TheCU.constructImportedEntityDIE(Module)); } @@ -445,44 +445,35 @@ void DwarfDebug::beginModule() { SingleCU = CU_Nodes->getNumOperands() == 1; for (MDNode *N : CU_Nodes->operands()) { - DICompileUnit CUNode(N); + DICompileUnit CUNode = cast(N); DwarfCompileUnit &CU = constructDwarfCompileUnit(CUNode); - DIArray ImportedEntities = CUNode.getImportedEntities(); - for (unsigned i = 0, e = ImportedEntities.getNumElements(); i != e; ++i) - ScopesWithImportedEntities.push_back(std::make_pair( - DIImportedEntity(ImportedEntities.getElement(i)).getContext(), - ImportedEntities.getElement(i))); + for (auto *IE : CUNode->getImportedEntities()) + ScopesWithImportedEntities.push_back(std::make_pair(IE->getScope(), IE)); // Stable sort to preserve the order of appearance of imported entities. // This is to avoid out-of-order processing of interdependent declarations // within the same scope, e.g. { namespace A = base; namespace B = A; } std::stable_sort(ScopesWithImportedEntities.begin(), ScopesWithImportedEntities.end(), less_first()); - DIArray GVs = CUNode.getGlobalVariables(); - for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i) - CU.getOrCreateGlobalVariableDIE(DIGlobalVariable(GVs.getElement(i))); - DIArray SPs = CUNode.getSubprograms(); - for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) - SPMap.insert(std::make_pair(SPs.getElement(i), &CU)); - DIArray EnumTypes = CUNode.getEnumTypes(); - for (unsigned i = 0, e = EnumTypes.getNumElements(); i != e; ++i) { - DIType Ty(EnumTypes.getElement(i)); + for (auto *GV : CUNode->getGlobalVariables()) + CU.getOrCreateGlobalVariableDIE(GV); + for (auto *SP : CUNode->getSubprograms()) + SPMap.insert(std::make_pair(SP, &CU)); + for (DIType Ty : CUNode->getEnumTypes()) { // The enum types array by design contains pointers to // MDNodes rather than DIRefs. Unique them here. - DIType UniqueTy(resolve(Ty.getRef())); + DIType UniqueTy = cast(resolve(Ty->getRef())); CU.getOrCreateTypeDIE(UniqueTy); } - DIArray RetainedTypes = CUNode.getRetainedTypes(); - for (unsigned i = 0, e = RetainedTypes.getNumElements(); i != e; ++i) { - DIType Ty(RetainedTypes.getElement(i)); + for (DIType Ty : CUNode->getRetainedTypes()) { // The retained types array by design contains pointers to // MDNodes rather than DIRefs. Unique them here. - DIType UniqueTy(resolve(Ty.getRef())); + DIType UniqueTy = cast(resolve(Ty->getRef())); CU.getOrCreateTypeDIE(UniqueTy); } // Emit imported_modules last so that the relevant context is already // available. - for (unsigned i = 0, e = ImportedEntities.getNumElements(); i != e; ++i) - constructAndAddImportedEntityDIE(CU, ImportedEntities.getElement(i)); + for (auto *IE : CUNode->getImportedEntities()) + constructAndAddImportedEntityDIE(CU, IE); } // Tell MMI that we have debug info. @@ -498,7 +489,8 @@ void DwarfDebug::finishVariableDefinitions() { // DIE::getUnit isn't simple - it walks parent pointers, etc. DwarfCompileUnit *Unit = lookupUnit(VariableDie->getUnit()); assert(Unit); - DbgVariable *AbsVar = getExistingAbstractVariable(Var->getVariable()); + DbgVariable *AbsVar = getExistingAbstractVariable( + InlinedVariable(Var->getVariable(), Var->getInlinedAt())); if (AbsVar && AbsVar->getDIE()) { Unit->addDIEEntry(*VariableDie, dwarf::DW_AT_abstract_origin, *AbsVar->getDIE()); @@ -510,7 +502,7 @@ void DwarfDebug::finishVariableDefinitions() { void DwarfDebug::finishSubprogramDefinitions() { for (const auto &P : SPMap) forBothCUs(*P.second, [&](DwarfCompileUnit &CU) { - CU.finishSubprogramDefinition(DISubprogram(P.first)); + CU.finishSubprogramDefinition(cast(P.first)); }); } @@ -521,14 +513,12 @@ void DwarfDebug::collectDeadVariables() { if (NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu")) { for (MDNode *N : CU_Nodes->operands()) { - DICompileUnit TheCU(N); + DICompileUnit TheCU = cast(N); // Construct subprogram DIE and add variables DIEs. DwarfCompileUnit *SPCU = static_cast(CUMap.lookup(TheCU)); assert(SPCU && "Unable to find Compile Unit!"); - DIArray Subprograms = TheCU.getSubprograms(); - for (unsigned i = 0, e = Subprograms.getNumElements(); i != e; ++i) { - DISubprogram SP(Subprograms.getElement(i)); + for (auto *SP : TheCU->getSubprograms()) { if (ProcessedSPNodes.count(SP) != 0) continue; SPCU->collectDeadVariables(SP); @@ -671,70 +661,71 @@ void DwarfDebug::endModule() { } // Find abstract variable, if any, associated with Var. -DbgVariable *DwarfDebug::getExistingAbstractVariable(const DIVariable &DV, +DbgVariable *DwarfDebug::getExistingAbstractVariable(InlinedVariable IV, DIVariable &Cleansed) { - LLVMContext &Ctx = DV->getContext(); // More then one inlined variable corresponds to one abstract variable. - // FIXME: This duplication of variables when inlining should probably be - // removed. It's done to allow each DIVariable to describe its location - // because the DebugLoc on the dbg.value/declare isn't accurate. We should - // make it accurate then remove this duplication/cleansing stuff. - Cleansed = cleanseInlinedVariable(DV, Ctx); + Cleansed = IV.first; auto I = AbstractVariables.find(Cleansed); if (I != AbstractVariables.end()) return I->second.get(); return nullptr; } -DbgVariable *DwarfDebug::getExistingAbstractVariable(const DIVariable &DV) { +DbgVariable *DwarfDebug::getExistingAbstractVariable(InlinedVariable IV) { DIVariable Cleansed; - return getExistingAbstractVariable(DV, Cleansed); + return getExistingAbstractVariable(IV, Cleansed); } void DwarfDebug::createAbstractVariable(const DIVariable &Var, LexicalScope *Scope) { - auto AbsDbgVariable = make_unique(Var, DIExpression(), this); + auto AbsDbgVariable = + make_unique(Var, nullptr, DIExpression(), this); InfoHolder.addScopeVariable(Scope, AbsDbgVariable.get()); AbstractVariables[Var] = std::move(AbsDbgVariable); } -void DwarfDebug::ensureAbstractVariableIsCreated(const DIVariable &DV, +void DwarfDebug::ensureAbstractVariableIsCreated(InlinedVariable IV, const MDNode *ScopeNode) { - DIVariable Cleansed = DV; - if (getExistingAbstractVariable(DV, Cleansed)) + DIVariable Cleansed; + if (getExistingAbstractVariable(IV, Cleansed)) return; - createAbstractVariable(Cleansed, LScopes.getOrCreateAbstractScope(ScopeNode)); + createAbstractVariable(Cleansed, LScopes.getOrCreateAbstractScope( + cast(ScopeNode))); } -void -DwarfDebug::ensureAbstractVariableIsCreatedIfScoped(const DIVariable &DV, - const MDNode *ScopeNode) { - DIVariable Cleansed = DV; - if (getExistingAbstractVariable(DV, Cleansed)) +void DwarfDebug::ensureAbstractVariableIsCreatedIfScoped( + InlinedVariable IV, const MDNode *ScopeNode) { + DIVariable Cleansed; + if (getExistingAbstractVariable(IV, Cleansed)) return; - if (LexicalScope *Scope = LScopes.findAbstractScope(ScopeNode)) + if (LexicalScope *Scope = + LScopes.findAbstractScope(cast_or_null(ScopeNode))) createAbstractVariable(Cleansed, Scope); } // Collect variable information from side table maintained by MMI. void DwarfDebug::collectVariableInfoFromMMITable( - SmallPtrSetImpl &Processed) { + DenseSet &Processed) { for (const auto &VI : MMI->getVariableDbgInfo()) { if (!VI.Var) continue; - Processed.insert(VI.Var); + assert(VI.Var->isValidLocationForIntrinsic(VI.Loc) && + "Expected inlined-at fields to agree"); + + InlinedVariable Var(VI.Var, VI.Loc->getInlinedAt()); + Processed.insert(Var); LexicalScope *Scope = LScopes.findLexicalScope(VI.Loc); // If variable scope is not found then skip this variable. if (!Scope) continue; - DIVariable DV(VI.Var); - DIExpression Expr(VI.Expr); - ensureAbstractVariableIsCreatedIfScoped(DV, Scope->getScopeNode()); - auto RegVar = make_unique(DV, Expr, this, VI.Slot); + DIExpression Expr = cast_or_null(VI.Expr); + ensureAbstractVariableIsCreatedIfScoped(Var, Scope->getScopeNode()); + auto RegVar = + make_unique(Var.first, Var.second, Expr, this, VI.Slot); if (InfoHolder.addScopeVariable(Scope, RegVar.get())) ConcreteVariables.push_back(std::move(RegVar)); } @@ -768,12 +759,12 @@ static DebugLocEntry::Value getDebugLocValue(const MachineInstr *MI) { /// Determine whether two variable pieces overlap. static bool piecesOverlap(DIExpression P1, DIExpression P2) { - if (!P1.isBitPiece() || !P2.isBitPiece()) + if (!P1->isBitPiece() || !P2->isBitPiece()) return true; - unsigned l1 = P1.getBitPieceOffset(); - unsigned l2 = P2.getBitPieceOffset(); - unsigned r1 = l1 + P1.getBitPieceSize(); - unsigned r2 = l2 + P2.getBitPieceSize(); + unsigned l1 = P1->getBitPieceOffset(); + unsigned l2 = P2->getBitPieceOffset(); + unsigned r1 = l1 + P1->getBitPieceSize(); + unsigned r2 = l2 + P2->getBitPieceSize(); // True where [l1,r1[ and [r1,r2[ overlap. return (l1 < r2) && (l2 < r1); } @@ -845,7 +836,7 @@ DwarfDebug::buildLocationList(SmallVectorImpl &DebugLoc, bool couldMerge = false; // If this is a piece, it may belong to the current DebugLocEntry. - if (DIExpr.isBitPiece()) { + if (DIExpr->isBitPiece()) { // Add this value to the list of open ranges. OpenRanges.push_back(Value); @@ -884,35 +875,34 @@ DwarfDebug::buildLocationList(SmallVectorImpl &DebugLoc, // Find variables for each lexical scope. -void -DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP, - SmallPtrSetImpl &Processed) { +void DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP, + DenseSet &Processed) { // Grab the variable info that was squirreled away in the MMI side-table. collectVariableInfoFromMMITable(Processed); for (const auto &I : DbgValues) { - DIVariable DV(I.first); - if (Processed.count(DV)) + InlinedVariable IV = I.first; + if (Processed.count(IV)) continue; - // Instruction ranges, specifying where DV is accessible. + // Instruction ranges, specifying where IV is accessible. const auto &Ranges = I.second; if (Ranges.empty()) continue; LexicalScope *Scope = nullptr; - if (MDNode *IA = DV.getInlinedAt()) - Scope = LScopes.findInlinedScope(DV.getContext(), IA); + if (const MDLocation *IA = IV.second) + Scope = LScopes.findInlinedScope(IV.first->getScope(), IA); else - Scope = LScopes.findLexicalScope(DV.getContext()); + Scope = LScopes.findLexicalScope(IV.first->getScope()); // If variable scope is not found then skip this variable. if (!Scope) continue; - Processed.insert(DV); + Processed.insert(IV); const MachineInstr *MInsn = Ranges.front().first; assert(MInsn->isDebugValue() && "History must begin with debug value"); - ensureAbstractVariableIsCreatedIfScoped(DV, Scope->getScopeNode()); + ensureAbstractVariableIsCreatedIfScoped(IV, Scope->getScopeNode()); ConcreteVariables.push_back(make_unique(MInsn, this)); DbgVariable *RegVar = ConcreteVariables.back().get(); InfoHolder.addScopeVariable(Scope, RegVar); @@ -937,16 +927,15 @@ DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP, } // Collect info for variables that were optimized out. - DIArray Variables = SP.getVariables(); - for (unsigned i = 0, e = Variables.getNumElements(); i != e; ++i) { - DIVariable DV(Variables.getElement(i)); - assert(DV.isVariable()); - if (!Processed.insert(DV).second) + for (DIVariable DV : SP->getVariables()) { + if (!Processed.insert(InlinedVariable(DV, nullptr)).second) continue; - if (LexicalScope *Scope = LScopes.findLexicalScope(DV.getContext())) { - ensureAbstractVariableIsCreatedIfScoped(DV, Scope->getScopeNode()); + if (LexicalScope *Scope = LScopes.findLexicalScope(DV->getScope())) { + ensureAbstractVariableIsCreatedIfScoped(InlinedVariable(DV, nullptr), + Scope->getScopeNode()); DIExpression NoExpr; - ConcreteVariables.push_back(make_unique(DV, NoExpr, this)); + ConcreteVariables.push_back( + make_unique(DV, nullptr, NoExpr, this)); InfoHolder.addScopeVariable(Scope, ConcreteVariables.back().get()); } } @@ -972,7 +961,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { if (!MI->isDebugValue()) { DebugLoc DL = MI->getDebugLoc(); if (DL != PrevInstLoc) { - if (!DL.isUnknown()) { + if (DL) { unsigned Flags = 0; PrevInstLoc = DL; if (DL == PrologEndLoc) { @@ -984,7 +973,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { Asm->OutStreamer.getContext().getCurrentDwarfLoc().getLine()) Flags |= DWARF2_FLAG_IS_STMT; - const MDNode *Scope = DL.getScope(Asm->MF->getFunction()->getContext()); + const MDNode *Scope = DL.getScope(); recordSourceLine(DL.getLine(), DL.getCol(), Scope, Flags); } else if (UnknownLocations) { PrevInstLoc = DL; @@ -1072,7 +1061,7 @@ static DebugLoc findPrologueEndLoc(const MachineFunction *MF) { for (const auto &MBB : *MF) for (const auto &MI : MBB) if (!MI.isDebugValue() && !MI.getFlag(MachineInstr::FrameSetup) && - !MI.getDebugLoc().isUnknown()) { + MI.getDebugLoc()) { // Did the target forget to set the FrameSetup flag for CFI insns? assert(!MI.isCFIInstruction() && "First non-frame-setup instruction is a CFI instruction."); @@ -1137,11 +1126,11 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { // The first mention of a function argument gets the CurrentFnBegin // label, so arguments are visible when breaking at function entry. - DIVariable DIVar(Ranges.front().first->getDebugVariable()); - if (DIVar.isVariable() && DIVar.getTag() == dwarf::DW_TAG_arg_variable && - getDISubprogram(DIVar.getContext()).describes(MF->getFunction())) { + DIVariable DIVar = Ranges.front().first->getDebugVariable(); + if (DIVar->getTag() == dwarf::DW_TAG_arg_variable && + getDISubprogram(DIVar->getScope())->describes(MF->getFunction())) { LabelsBeforeInsn[Ranges.front().first] = Asm->getFunctionBegin(); - if (Ranges.front().first->getDebugExpression().isBitPiece()) { + if (Ranges.front().first->getDebugExpression()->isBitPiece()) { // Mark all non-overlapping initial pieces. for (auto I = Ranges.begin(); I != Ranges.end(); ++I) { DIExpression Piece = I->first->getDebugExpression(); @@ -1168,15 +1157,11 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { // Record beginning of function. PrologEndLoc = findPrologueEndLoc(MF); - if (!PrologEndLoc.isUnknown()) { - DebugLoc FnStartDL = - PrologEndLoc.getFnDebugLoc(MF->getFunction()->getContext()); - + if (MDLocation *L = PrologEndLoc) { // We'd like to list the prologue as "not statements" but GDB behaves // poorly if we do that. Revisit this with caution/GDB (7.5+) testing. - recordSourceLine(FnStartDL.getLine(), FnStartDL.getCol(), - FnStartDL.getScope(MF->getFunction()->getContext()), - DWARF2_FLAG_IS_STMT); + auto *SP = L->getInlinedAtScope()->getSubprogram(); + recordSourceLine(SP->getScopeLine(), 0, SP, DWARF2_FLAG_IS_STMT); } } @@ -1199,10 +1184,10 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { Asm->OutStreamer.getContext().setDwarfCompileUnitID(0); LexicalScope *FnScope = LScopes.getCurrentFunctionScope(); - DISubprogram SP(FnScope->getScopeNode()); + DISubprogram SP = cast(FnScope->getScopeNode()); DwarfCompileUnit &TheCU = *SPMap.lookup(SP); - SmallPtrSet ProcessedVars; + DenseSet ProcessedVars; collectVariableInfo(TheCU, SP, ProcessedVars); // Add the range of this function to the list of ranges for the CU. @@ -1210,7 +1195,7 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { // Under -gmlt, skip building the subprogram if there are no inlined // subroutines inside it. - if (TheCU.getCUNode().getEmissionKind() == DIBuilder::LineTablesOnly && + if (TheCU.getCUNode()->getEmissionKind() == DIBuilder::LineTablesOnly && LScopes.getAbstractScopesList().empty() && !IsDarwin) { assert(InfoHolder.getScopeVariables().empty()); assert(DbgValues.empty()); @@ -1229,16 +1214,13 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { #endif // Construct abstract scopes. for (LexicalScope *AScope : LScopes.getAbstractScopesList()) { - DISubprogram SP(AScope->getScopeNode()); - assert(SP.isSubprogram()); + DISubprogram SP = cast(AScope->getScopeNode()); // Collect info for variables that were optimized out. - DIArray Variables = SP.getVariables(); - for (unsigned i = 0, e = Variables.getNumElements(); i != e; ++i) { - DIVariable DV(Variables.getElement(i)); - assert(DV && DV.isVariable()); - if (!ProcessedVars.insert(DV).second) + for (DIVariable DV : SP->getVariables()) { + if (!ProcessedVars.insert(InlinedVariable(DV, nullptr)).second) continue; - ensureAbstractVariableIsCreated(DV, DV.getContext()); + ensureAbstractVariableIsCreated(InlinedVariable(DV, nullptr), + DV->getScope()); assert(LScopes.getAbstractScopesList().size() == NumAbstractScopes && "ensureAbstractVariableIsCreated inserted abstract scopes"); } @@ -1270,12 +1252,11 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, StringRef Dir; unsigned Src = 1; unsigned Discriminator = 0; - if (DIScope Scope = DIScope(S)) { - assert(Scope.isScope()); - Fn = Scope.getFilename(); - Dir = Scope.getDirectory(); - if (Scope.isLexicalBlockFile()) - Discriminator = DILexicalBlockFile(S).getDiscriminator(); + if (auto *Scope = cast_or_null(S)) { + Fn = Scope->getFilename(); + Dir = Scope->getDirectory(); + if (auto *LBF = dyn_cast(Scope)) + Discriminator = LBF->getDiscriminator(); unsigned CUID = Asm->OutStreamer.getContext().getDwarfCompileUnitID(); Src = static_cast(*InfoHolder.getUnits()[CUID]) @@ -1499,23 +1480,25 @@ static void emitDebugLocValue(const AsmPrinter &AP, Streamer); // Regular entry. if (Value.isInt()) { - DIBasicType BTy(DV.getType().resolve(TypeIdentifierMap)); - if (BTy.Verify() && (BTy.getEncoding() == dwarf::DW_ATE_signed || - BTy.getEncoding() == dwarf::DW_ATE_signed_char)) + MDType *T = DV->getType().resolve(TypeIdentifierMap); + auto *B = dyn_cast(T); + if (B && (B->getEncoding() == dwarf::DW_ATE_signed || + B->getEncoding() == dwarf::DW_ATE_signed_char)) DwarfExpr.AddSignedConstant(Value.getInt()); else DwarfExpr.AddUnsignedConstant(Value.getInt()); } else if (Value.isLocation()) { MachineLocation Loc = Value.getLoc(); DIExpression Expr = Value.getExpression(); - if (!Expr || (Expr.getNumElements() == 0)) + if (!Expr || !Expr->getNumElements()) // Regular entry. AP.EmitDwarfRegOp(Streamer, Loc); else { // Complex address entry. if (Loc.getOffset()) { DwarfExpr.AddMachineRegIndirect(Loc.getReg(), Loc.getOffset()); - DwarfExpr.AddExpression(Expr.begin(), Expr.end(), PieceOffsetInBits); + DwarfExpr.AddExpression(Expr->expr_op_begin(), Expr->expr_op_end(), + PieceOffsetInBits); } else DwarfExpr.AddMachineRegExpression(Expr, Loc.getReg(), PieceOffsetInBits); @@ -1542,8 +1525,8 @@ void DebugLocEntry::finalize(const AsmPrinter &AP, unsigned Offset = 0; for (auto Piece : Values) { DIExpression Expr = Piece.getExpression(); - unsigned PieceOffset = Expr.getBitPieceOffset(); - unsigned PieceSize = Expr.getBitPieceSize(); + unsigned PieceOffset = Expr->getBitPieceOffset(); + unsigned PieceSize = Expr->getBitPieceSize(); assert(Offset <= PieceOffset && "overlapping or duplicate pieces"); if (Offset < PieceOffset) { // The DWARF spec seriously mandates pieces with no locations for gaps. @@ -1554,15 +1537,7 @@ void DebugLocEntry::finalize(const AsmPrinter &AP, Offset += PieceOffset-Offset; } Offset += PieceSize; - -#ifndef NDEBUG - DIVariable Var = Piece.getVariable(); - unsigned VarSize = Var.getSizeInBits(TypeIdentifierMap); - assert(PieceSize+PieceOffset <= VarSize - && "piece is larger than or outside of variable"); - assert(PieceSize != VarSize - && "piece covers entire variable"); -#endif + emitDebugLocValue(AP, TypeIdentifierMap, Streamer, Piece, PieceOffset); } } else { @@ -1850,7 +1825,7 @@ void DwarfDebug::emitDebugRanges() { void DwarfDebug::initSkeletonUnit(const DwarfUnit &U, DIE &Die, std::unique_ptr NewU) { NewU->addString(Die, dwarf::DW_AT_GNU_dwo_name, - U.getCUNode().getSplitDebugFilename()); + U.getCUNode()->getSplitDebugFilename()); if (!CompilationDir.empty()) NewU->addString(Die, dwarf::DW_AT_comp_dir, CompilationDir); @@ -1914,7 +1889,7 @@ MCDwarfDwoLineTable *DwarfDebug::getDwoLineTable(const DwarfCompileUnit &CU) { if (!useSplitDwarf()) return nullptr; if (SingleCU) - SplitTypeUnitFileTable.setCompilationDir(CU.getCUNode().getDirectory()); + SplitTypeUnitFileTable.setCompilationDir(CU.getCUNode()->getDirectory()); return &SplitTypeUnitFileTable; } diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h index 74db3ef..e067f41 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -21,6 +21,7 @@ #include "DwarfAccelTable.h" #include "DwarfFile.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallPtrSet.h" @@ -74,7 +75,8 @@ public: /// - Variables that are described by multiple MMI table entries have multiple /// expressions and frame indices. class DbgVariable { - DIVariable Var; /// Variable Descriptor. + DIVariable Var; /// Variable Descriptor. + DILocation IA; /// Inlined at location. SmallVector Expr; /// Complex address location expression. DIE *TheDIE; /// Variable DIE. unsigned DotDebugLocOffset; /// Offset in DotDebugLocEntries. @@ -84,11 +86,11 @@ class DbgVariable { public: /// Construct a DbgVariable from a DIVariable. - DbgVariable(DIVariable V, DIExpression E, DwarfDebug *DD, int FI = ~0) - : Var(V), Expr(1, E), TheDIE(nullptr), DotDebugLocOffset(~0U), - MInsn(nullptr), DD(DD) { + DbgVariable(DIVariable V, DILocation IA, DIExpression E, DwarfDebug *DD, + int FI = ~0) + : Var(V), IA(IA), Expr(1, E), TheDIE(nullptr), DotDebugLocOffset(~0U), + MInsn(nullptr), DD(DD) { FrameIndex.push_back(FI); - assert(Var.Verify()); assert(!E || E->isValid()); } @@ -96,6 +98,7 @@ public: /// AbstractVar may be NULL. DbgVariable(const MachineInstr *DbgValue, DwarfDebug *DD) : Var(DbgValue->getDebugVariable()), + IA(DbgValue->getDebugLoc()->getInlinedAt()), Expr(1, DbgValue->getDebugExpression()), TheDIE(nullptr), DotDebugLocOffset(~0U), MInsn(DbgValue), DD(DD) { FrameIndex.push_back(~0); @@ -103,12 +106,13 @@ public: // Accessors. DIVariable getVariable() const { return Var; } + DILocation getInlinedAt() const { return IA; } const ArrayRef getExpression() const { return Expr; } void setDIE(DIE &D) { TheDIE = &D; } DIE *getDIE() const { return TheDIE; } void setDotDebugLocOffset(unsigned O) { DotDebugLocOffset = O; } unsigned getDotDebugLocOffset() const { return DotDebugLocOffset; } - StringRef getName() const { return Var.getName(); } + StringRef getName() const { return Var->getName(); } const MachineInstr *getMInsn() const { return MInsn; } const ArrayRef getFrameIndex() const { return FrameIndex; } @@ -116,6 +120,7 @@ public: assert( DotDebugLocOffset == ~0U && !MInsn && "not an MMI entry"); assert(V.DotDebugLocOffset == ~0U && !V.MInsn && "not an MMI entry"); assert(V.Var == Var && "conflicting DIVariable"); + assert(V.IA == IA && "conflicting inlined-at location"); if (V.getFrameIndex().back() != ~0) { auto E = V.getExpression(); @@ -124,40 +129,40 @@ public: FrameIndex.append(FI.begin(), FI.end()); } assert(Expr.size() > 1 - ? std::all_of(Expr.begin(), Expr.end(), - [](DIExpression &E) { return E.isBitPiece(); }) - : (true && "conflicting locations for variable")); + ? std::all_of(Expr.begin(), Expr.end(), + [](DIExpression &E) { return E->isBitPiece(); }) + : (true && "conflicting locations for variable")); } // Translate tag to proper Dwarf tag. dwarf::Tag getTag() const { - if (Var.getTag() == dwarf::DW_TAG_arg_variable) + if (Var->getTag() == dwarf::DW_TAG_arg_variable) return dwarf::DW_TAG_formal_parameter; return dwarf::DW_TAG_variable; } /// \brief Return true if DbgVariable is artificial. bool isArtificial() const { - if (Var.isArtificial()) + if (Var->isArtificial()) return true; - if (getType().isArtificial()) + if (getType()->isArtificial()) return true; return false; } bool isObjectPointer() const { - if (Var.isObjectPointer()) + if (Var->isObjectPointer()) return true; - if (getType().isObjectPointer()) + if (getType()->isObjectPointer()) return true; return false; } bool variableHasComplexAddress() const { - assert(Var.isVariable() && "Invalid complex DbgVariable!"); + assert(Var && "Invalid complex DbgVariable!"); assert(Expr.size() == 1 && "variableHasComplexAddress() invoked on multi-FI variable"); - return Expr.back().getNumElements() > 0; + return Expr.back()->getNumElements() > 0; } bool isBlockByrefVariable() const; DIType getType() const; @@ -165,7 +170,7 @@ public: private: /// resolve - Look in the DwarfDebug map for the MDNode that /// corresponds to the reference. - template T resolve(DIRef Ref) const; + template T *resolve(TypedDebugNodeRef Ref) const; }; @@ -324,14 +329,16 @@ class DwarfDebug : public AsmPrinterHandler { return InfoHolder.getUnits(); } + typedef DbgValueHistoryMap::InlinedVariable InlinedVariable; + /// \brief Find abstract variable associated with Var. - DbgVariable *getExistingAbstractVariable(const DIVariable &DV, + DbgVariable *getExistingAbstractVariable(InlinedVariable IV, DIVariable &Cleansed); - DbgVariable *getExistingAbstractVariable(const DIVariable &DV); + DbgVariable *getExistingAbstractVariable(InlinedVariable IV); void createAbstractVariable(const DIVariable &DV, LexicalScope *Scope); - void ensureAbstractVariableIsCreated(const DIVariable &Var, + void ensureAbstractVariableIsCreated(InlinedVariable Var, const MDNode *Scope); - void ensureAbstractVariableIsCreatedIfScoped(const DIVariable &Var, + void ensureAbstractVariableIsCreatedIfScoped(InlinedVariable Var, const MDNode *Scope); /// \brief Construct a DIE for this abstract scope. @@ -461,7 +468,7 @@ class DwarfDebug : public AsmPrinterHandler { /// \brief Populate LexicalScope entries with variables' info. void collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP, - SmallPtrSetImpl &ProcessedVars); + DenseSet &ProcessedVars); /// \brief Build the location list for all DBG_VALUEs in the /// function that describe the same variable. @@ -470,7 +477,7 @@ class DwarfDebug : public AsmPrinterHandler { /// \brief Collect variable information from the side table maintained /// by MMI. - void collectVariableInfoFromMMITable(SmallPtrSetImpl &P); + void collectVariableInfoFromMMITable(DenseSet &P); /// \brief Ensure that a label will be emitted before MI. void requestLabelBeforeInsn(const MachineInstr *MI) { @@ -567,7 +574,7 @@ public: void emitDebugLocEntryLocation(const DebugLocEntry &Entry); /// Find the MDNode for the given reference. - template T resolve(DIRef Ref) const { + template T *resolve(TypedDebugNodeRef Ref) const { return Ref.resolve(TypeIdentifierMap); } diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h index 6eaf707..a4fd36f 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.h +++ b/lib/CodeGen/AsmPrinter/DwarfException.h @@ -48,7 +48,7 @@ public: // Main entry points. // DwarfCFIException(AsmPrinter *A); - virtual ~DwarfCFIException(); + ~DwarfCFIException() override; /// Emit all exception information that should come after the content. void endModule() override; @@ -70,7 +70,7 @@ public: // Main entry points. // ARMException(AsmPrinter *A); - virtual ~ARMException(); + ~ARMException() override; /// Emit all exception information that should come after the content. void endModule() override; diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp index 489e455..e576c93 100644 --- a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp @@ -195,27 +195,27 @@ static unsigned getOffsetOrZero(unsigned OffsetInBits, bool DwarfExpression::AddMachineRegExpression(DIExpression Expr, unsigned MachineReg, unsigned PieceOffsetInBits) { - auto I = Expr.begin(); - auto E = Expr.end(); + auto I = Expr->expr_op_begin(); + auto E = Expr->expr_op_end(); if (I == E) return AddMachineRegPiece(MachineReg); // Pattern-match combinations for which more efficient representations exist // first. bool ValidReg = false; - switch (*I) { + switch (I->getOp()) { case dwarf::DW_OP_bit_piece: { - unsigned OffsetInBits = I->getArg(1); - unsigned SizeInBits = I->getArg(2); + unsigned OffsetInBits = I->getArg(0); + unsigned SizeInBits = I->getArg(1); // Piece always comes at the end of the expression. return AddMachineRegPiece(MachineReg, SizeInBits, getOffsetOrZero(OffsetInBits, PieceOffsetInBits)); } case dwarf::DW_OP_plus: { // [DW_OP_reg,Offset,DW_OP_plus,DW_OP_deref] --> [DW_OP_breg,Offset]. - auto N = I->getNext(); - if ((N != E) && (*N == dwarf::DW_OP_deref)) { - unsigned Offset = I->getArg(1); + auto N = I.getNext(); + if (N != E && N->getOp() == dwarf::DW_OP_deref) { + unsigned Offset = I->getArg(0); ValidReg = AddMachineRegIndirect(MachineReg, Offset); std::advance(I, 2); break; @@ -240,20 +240,20 @@ bool DwarfExpression::AddMachineRegExpression(DIExpression Expr, return true; } -void DwarfExpression::AddExpression(DIExpression::iterator I, - DIExpression::iterator E, +void DwarfExpression::AddExpression(MDExpression::expr_op_iterator I, + MDExpression::expr_op_iterator E, unsigned PieceOffsetInBits) { for (; I != E; ++I) { - switch (*I) { + switch (I->getOp()) { case dwarf::DW_OP_bit_piece: { - unsigned OffsetInBits = I->getArg(1); - unsigned SizeInBits = I->getArg(2); + unsigned OffsetInBits = I->getArg(0); + unsigned SizeInBits = I->getArg(1); AddOpPiece(SizeInBits, getOffsetOrZero(OffsetInBits, PieceOffsetInBits)); break; } case dwarf::DW_OP_plus: EmitOp(dwarf::DW_OP_plus_uconst); - EmitUnsigned(I->getArg(1)); + EmitUnsigned(I->getArg(0)); break; case dwarf::DW_OP_deref: EmitOp(dwarf::DW_OP_deref); diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.h b/lib/CodeGen/AsmPrinter/DwarfExpression.h index 985d52c..a8b65f5 100644 --- a/lib/CodeGen/AsmPrinter/DwarfExpression.h +++ b/lib/CodeGen/AsmPrinter/DwarfExpression.h @@ -97,7 +97,8 @@ public: /// Emit a the operations remaining the DIExpressionIterator I. /// \param PieceOffsetInBits If this is one piece out of a fragmented /// location, this is the offset of the piece inside the entire variable. - void AddExpression(DIExpression::iterator I, DIExpression::iterator E, + void AddExpression(MDExpression::expr_op_iterator I, + MDExpression::expr_op_iterator E, unsigned PieceOffsetInBits = 0); }; diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.cpp b/lib/CodeGen/AsmPrinter/DwarfFile.cpp index 60acc58e..32adb40 100644 --- a/lib/CodeGen/AsmPrinter/DwarfFile.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfFile.cpp @@ -139,7 +139,7 @@ bool DwarfFile::addScopeVariable(LexicalScope *LS, DbgVariable *Var) { SmallVectorImpl &Vars = ScopeVariables[LS]; DIVariable DV = Var->getVariable(); // Variables with positive arg numbers are parameters. - if (unsigned ArgNum = DV.getArgNumber()) { + if (unsigned ArgNum = DV->getArg()) { // Keep all parameters in order at the start of the variable list to ensure // function types are correct (no out-of-order parameters) // @@ -149,7 +149,7 @@ bool DwarfFile::addScopeVariable(LexicalScope *LS, DbgVariable *Var) { // rather than linear search. auto I = Vars.begin(); while (I != Vars.end()) { - unsigned CurNum = (*I)->getVariable().getArgNumber(); + unsigned CurNum = (*I)->getVariable()->getArg(); // A local (non-parameter) variable has been found, insert immediately // before it. if (CurNum == 0) diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp index f6af73f..9154652 100644 --- a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -175,8 +175,8 @@ static bool isShareableAcrossCUs(DIDescriptor D) { // level already) but may be implementable for some value in projects // building multiple independent libraries with LTO and then linking those // together. - return (D.isType() || - (D.isSubprogram() && !DISubprogram(D).isDefinition())) && + return (isa(D) || + (isa(D) && !cast(D)->isDefinition())) && !GenerateDwarfTypeUnits; } @@ -397,52 +397,48 @@ void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, StringRef File, /// addSourceLine - Add location information to specified debug information /// entry. void DwarfUnit::addSourceLine(DIE &Die, DIVariable V) { - assert(V.isVariable()); + assert(V); - addSourceLine(Die, V.getLineNumber(), V.getContext().getFilename(), - V.getContext().getDirectory()); + addSourceLine(Die, V->getLine(), V->getScope()->getFilename(), + V->getScope()->getDirectory()); } /// addSourceLine - Add location information to specified debug information /// entry. void DwarfUnit::addSourceLine(DIE &Die, DIGlobalVariable G) { - assert(G.isGlobalVariable()); + assert(G); - addSourceLine(Die, G.getLineNumber(), G.getFilename(), G.getDirectory()); + addSourceLine(Die, G->getLine(), G->getFilename(), G->getDirectory()); } /// addSourceLine - Add location information to specified debug information /// entry. void DwarfUnit::addSourceLine(DIE &Die, DISubprogram SP) { - assert(SP.isSubprogram()); + assert(SP); - addSourceLine(Die, SP.getLineNumber(), SP.getFilename(), SP.getDirectory()); + addSourceLine(Die, SP->getLine(), SP->getFilename(), SP->getDirectory()); } /// addSourceLine - Add location information to specified debug information /// entry. void DwarfUnit::addSourceLine(DIE &Die, DIType Ty) { - assert(Ty.isType()); + assert(Ty); - addSourceLine(Die, Ty.getLineNumber(), Ty.getFilename(), Ty.getDirectory()); + addSourceLine(Die, Ty->getLine(), Ty->getFilename(), Ty->getDirectory()); } /// addSourceLine - Add location information to specified debug information /// entry. void DwarfUnit::addSourceLine(DIE &Die, DIObjCProperty Ty) { - assert(Ty.isObjCProperty()); + assert(Ty); - DIFile File = Ty.getFile(); - addSourceLine(Die, Ty.getLineNumber(), File.getFilename(), - File.getDirectory()); + addSourceLine(Die, Ty->getLine(), Ty->getFilename(), Ty->getDirectory()); } /// addSourceLine - Add location information to specified debug information /// entry. void DwarfUnit::addSourceLine(DIE &Die, DINameSpace NS) { - assert(NS.Verify()); - - addSourceLine(Die, NS.getLineNumber(), NS.getFilename(), NS.getDirectory()); + addSourceLine(Die, NS->getLine(), NS->getFilename(), NS->getDirectory()); } /// addRegisterOp - Add register operand. @@ -525,28 +521,26 @@ void DwarfUnit::addBlockByrefAddress(const DbgVariable &DV, DIE &Die, const MachineLocation &Location) { DIType Ty = DV.getType(); DIType TmpTy = Ty; - uint16_t Tag = Ty.getTag(); + uint16_t Tag = Ty->getTag(); bool isPointer = false; StringRef varName = DV.getName(); if (Tag == dwarf::DW_TAG_pointer_type) { - DIDerivedType DTy(Ty); - TmpTy = resolve(DTy.getTypeDerivedFrom()); + DIDerivedType DTy = cast(Ty); + TmpTy = resolve(DTy->getBaseType()); isPointer = true; } - DICompositeType blockStruct(TmpTy); - // Find the __forwarding field and the variable field in the __Block_byref // struct. - DIArray Fields = blockStruct.getElements(); + DIArray Fields = cast(TmpTy)->getElements(); DIDerivedType varField; DIDerivedType forwardingField; - for (unsigned i = 0, N = Fields.getNumElements(); i < N; ++i) { - DIDerivedType DT(Fields.getElement(i)); - StringRef fieldName = DT.getName(); + for (unsigned i = 0, N = Fields.size(); i < N; ++i) { + DIDerivedType DT = cast(Fields[i]); + StringRef fieldName = DT->getName(); if (fieldName == "__forwarding") forwardingField = DT; else if (fieldName == varName) @@ -554,8 +548,8 @@ void DwarfUnit::addBlockByrefAddress(const DbgVariable &DV, DIE &Die, } // Get the offsets for the forwarding field and the variable field. - unsigned forwardingFieldOffset = forwardingField.getOffsetInBits() >> 3; - unsigned varFieldOffset = varField.getOffsetInBits() >> 2; + unsigned forwardingFieldOffset = forwardingField->getOffsetInBits() >> 3; + unsigned varFieldOffset = varField->getOffsetInBits() >> 2; // Decode the original location, and use that as the start of the byref // variable's location. @@ -601,9 +595,8 @@ void DwarfUnit::addBlockByrefAddress(const DbgVariable &DV, DIE &Die, /// Return true if type encoding is unsigned. static bool isUnsignedDIType(DwarfDebug *DD, DIType Ty) { - DIDerivedType DTy(Ty); - if (DTy.isDerivedType()) { - dwarf::Tag T = (dwarf::Tag)Ty.getTag(); + if (DIDerivedType DTy = dyn_cast(Ty)) { + dwarf::Tag T = (dwarf::Tag)Ty->getTag(); // Encode pointer constants as unsigned bytes. This is used at least for // null pointer constant emission. // (Pieces of) aggregate types that get hacked apart by SROA may also be @@ -624,56 +617,55 @@ static bool isUnsignedDIType(DwarfDebug *DD, DIType Ty) { T == dwarf::DW_TAG_volatile_type || T == dwarf::DW_TAG_restrict_type || T == dwarf::DW_TAG_enumeration_type); - if (DITypeRef Deriv = DTy.getTypeDerivedFrom()) + if (DITypeRef Deriv = DTy->getBaseType()) return isUnsignedDIType(DD, DD->resolve(Deriv)); // FIXME: Enums without a fixed underlying type have unknown signedness // here, leading to incorrectly emitted constants. - assert(DTy.getTag() == dwarf::DW_TAG_enumeration_type); + assert(DTy->getTag() == dwarf::DW_TAG_enumeration_type); return false; } - DIBasicType BTy(Ty); - assert(BTy.isBasicType()); - unsigned Encoding = BTy.getEncoding(); + DIBasicType BTy = cast(Ty); + unsigned Encoding = BTy->getEncoding(); assert((Encoding == dwarf::DW_ATE_unsigned || Encoding == dwarf::DW_ATE_unsigned_char || Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char || - Encoding == dwarf::DW_ATE_float || - Encoding == dwarf::DW_ATE_UTF || Encoding == dwarf::DW_ATE_boolean || - (Ty.getTag() == dwarf::DW_TAG_unspecified_type && - Ty.getName() == "decltype(nullptr)")) && + Encoding == dwarf::DW_ATE_float || Encoding == dwarf::DW_ATE_UTF || + Encoding == dwarf::DW_ATE_boolean || + (Ty->getTag() == dwarf::DW_TAG_unspecified_type && + Ty->getName() == "decltype(nullptr)")) && "Unsupported encoding"); - return (Encoding == dwarf::DW_ATE_unsigned || - Encoding == dwarf::DW_ATE_unsigned_char || - Encoding == dwarf::DW_ATE_UTF || Encoding == dwarf::DW_ATE_boolean || - Ty.getTag() == dwarf::DW_TAG_unspecified_type); + return Encoding == dwarf::DW_ATE_unsigned || + Encoding == dwarf::DW_ATE_unsigned_char || + Encoding == dwarf::DW_ATE_UTF || Encoding == dwarf::DW_ATE_boolean || + Ty->getTag() == dwarf::DW_TAG_unspecified_type; } /// If this type is derived from a base type then return base type size. static uint64_t getBaseTypeSize(DwarfDebug *DD, DIDerivedType Ty) { - unsigned Tag = Ty.getTag(); + unsigned Tag = Ty->getTag(); if (Tag != dwarf::DW_TAG_member && Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type && Tag != dwarf::DW_TAG_volatile_type && Tag != dwarf::DW_TAG_restrict_type) - return Ty.getSizeInBits(); + return Ty->getSizeInBits(); - DIType BaseType = DD->resolve(Ty.getTypeDerivedFrom()); + auto *BaseType = DD->resolve(Ty->getBaseType()); - assert(BaseType.isValid() && "Unexpected invalid base type"); + assert(BaseType && "Unexpected invalid base type"); // If this is a derived type, go ahead and get the base type, unless it's a // reference then it's just the size of the field. Pointer types have no need // of this since they're a different type of qualification on the type. - if (BaseType.getTag() == dwarf::DW_TAG_reference_type || - BaseType.getTag() == dwarf::DW_TAG_rvalue_reference_type) - return Ty.getSizeInBits(); + if (BaseType->getTag() == dwarf::DW_TAG_reference_type || + BaseType->getTag() == dwarf::DW_TAG_rvalue_reference_type) + return Ty->getSizeInBits(); - if (BaseType.isDerivedType()) - return getBaseTypeSize(DD, DIDerivedType(BaseType)); + if (auto *DT = dyn_cast(BaseType)) + return getBaseTypeSize(DD, DT); - return BaseType.getSizeInBits(); + return BaseType->getSizeInBits(); } /// addConstantFPValue - Add constant value entry in variable DIE. @@ -771,39 +763,37 @@ void DwarfUnit::addLinkageName(DIE &Die, StringRef LinkageName) { /// addTemplateParams - Add template parameters into buffer. void DwarfUnit::addTemplateParams(DIE &Buffer, DIArray TParams) { // Add template parameters. - for (unsigned i = 0, e = TParams.getNumElements(); i != e; ++i) { - DIDescriptor Element = TParams.getElement(i); - if (Element.isTemplateTypeParameter()) - constructTemplateTypeParameterDIE(Buffer, - DITemplateTypeParameter(Element)); - else if (Element.isTemplateValueParameter()) - constructTemplateValueParameterDIE(Buffer, - DITemplateValueParameter(Element)); + for (unsigned i = 0, e = TParams.size(); i != e; ++i) { + DIDescriptor Element = TParams[i]; + if (auto *TTP = dyn_cast(Element)) + constructTemplateTypeParameterDIE(Buffer, TTP); + else if (auto *TVP = dyn_cast(Element)) + constructTemplateValueParameterDIE(Buffer, TVP); } } /// getOrCreateContextDIE - Get context owner's DIE. DIE *DwarfUnit::getOrCreateContextDIE(DIScope Context) { - if (!Context || Context.isFile()) + if (!Context || isa(Context)) return &getUnitDie(); - if (Context.isType()) - return getOrCreateTypeDIE(DIType(Context)); - if (Context.isNameSpace()) - return getOrCreateNameSpace(DINameSpace(Context)); - if (Context.isSubprogram()) - return getOrCreateSubprogramDIE(DISubprogram(Context)); + if (auto *T = dyn_cast(Context)) + return getOrCreateTypeDIE(T); + if (auto *NS = dyn_cast(Context)) + return getOrCreateNameSpace(NS); + if (auto *SP = dyn_cast(Context)) + return getOrCreateSubprogramDIE(SP); return getDIE(Context); } DIE *DwarfUnit::createTypeDIE(DICompositeType Ty) { - DIScope Context = resolve(Ty.getContext()); + DIScope Context = resolve(Ty->getScope()); DIE *ContextDIE = getOrCreateContextDIE(Context); if (DIE *TyDIE = getDIE(Ty)) return TyDIE; // Create new type. - DIE &TyDIE = createAndAddDIE(Ty.getTag(), *ContextDIE, Ty); + DIE &TyDIE = createAndAddDIE(Ty->getTag(), *ContextDIE, Ty); constructTypeDIE(TyDIE, Ty); @@ -817,18 +807,18 @@ DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) { if (!TyNode) return nullptr; - DIType Ty(TyNode); - assert(Ty.isType()); - assert(Ty == resolve(Ty.getRef()) && + auto *Ty = cast(TyNode); + assert(Ty == resolve(Ty->getRef()) && "type was not uniqued, possible ODR violation."); // DW_TAG_restrict_type is not supported in DWARF2 - if (Ty.getTag() == dwarf::DW_TAG_restrict_type && DD->getDwarfVersion() <= 2) - return getOrCreateTypeDIE(resolve(DIDerivedType(Ty).getTypeDerivedFrom())); + if (Ty->getTag() == dwarf::DW_TAG_restrict_type && DD->getDwarfVersion() <= 2) + return getOrCreateTypeDIE( + resolve(DITypeRef(cast(Ty)->getBaseType()))); // Construct the context before querying for the existence of the DIE in case // such construction creates the DIE. - DIScope Context = resolve(Ty.getContext()); + DIScope Context = resolve(Ty->getScope()); DIE *ContextDIE = getOrCreateContextDIE(Context); assert(ContextDIE); @@ -836,24 +826,22 @@ DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) { return TyDIE; // Create new type. - DIE &TyDIE = createAndAddDIE(Ty.getTag(), *ContextDIE, Ty); + DIE &TyDIE = createAndAddDIE(Ty->getTag(), *ContextDIE, Ty); updateAcceleratorTables(Context, Ty, TyDIE); - if (Ty.isBasicType()) - constructTypeDIE(TyDIE, DIBasicType(Ty)); - else if (Ty.isCompositeType()) { - DICompositeType CTy(Ty); - if (GenerateDwarfTypeUnits && !Ty.isForwardDecl()) - if (MDString *TypeId = CTy.getIdentifier()) { + if (auto *BT = dyn_cast(Ty)) + constructTypeDIE(TyDIE, BT); + else if (DICompositeType CTy = dyn_cast(Ty)) { + if (GenerateDwarfTypeUnits && !Ty->isForwardDecl()) + if (MDString *TypeId = CTy->getRawIdentifier()) { DD->addDwarfTypeUnitType(getCU(), TypeId->getString(), TyDIE, CTy); // Skip updating the accelerator tables since this is not the full type. return &TyDIE; } constructTypeDIE(TyDIE, CTy); } else { - assert(Ty.isDerivedType() && "Unknown kind of DIType"); - constructTypeDIE(TyDIE, DIDerivedType(Ty)); + constructTypeDIE(TyDIE, cast(Ty)); } return &TyDIE; @@ -861,19 +849,18 @@ DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) { void DwarfUnit::updateAcceleratorTables(DIScope Context, DIType Ty, const DIE &TyDIE) { - if (!Ty.getName().empty() && !Ty.isForwardDecl()) { + if (!Ty->getName().empty() && !Ty->isForwardDecl()) { bool IsImplementation = 0; - if (Ty.isCompositeType()) { - DICompositeType CT(Ty); + if (auto *CT = dyn_cast(Ty)) { // A runtime language of 0 actually means C/C++ and that any // non-negative value is some version of Objective-C/C++. - IsImplementation = (CT.getRunTimeLang() == 0) || CT.isObjcClassComplete(); + IsImplementation = CT->getRuntimeLang() == 0 || CT->isObjcClassComplete(); } unsigned Flags = IsImplementation ? dwarf::DW_FLAG_type_implementation : 0; - DD->addAccelType(Ty.getName(), TyDIE, Flags); + DD->addAccelType(Ty->getName(), TyDIE, Flags); - if (!Context || Context.isCompileUnit() || Context.isFile() || - Context.isNameSpace()) + if (!Context || isa(Context) || isa(Context) || + isa(Context)) addGlobalType(Ty, TyDIE, Context); } } @@ -914,10 +901,10 @@ std::string DwarfUnit::getParentContextString(DIScope Context) const { std::string CS; SmallVector Parents; - while (!Context.isCompileUnit()) { + while (!isa(Context)) { Parents.push_back(Context); - if (Context.getContext()) - Context = resolve(Context.getContext()); + if (Context->getScope()) + Context = resolve(Context->getScope()); else // Structure, etc types will have a NULL context if they're at the top // level. @@ -929,9 +916,9 @@ std::string DwarfUnit::getParentContextString(DIScope Context) const { for (SmallVectorImpl::reverse_iterator I = Parents.rbegin(), E = Parents.rend(); I != E; ++I) { - DIScope Ctx = *I; - StringRef Name = Ctx.getName(); - if (Name.empty() && Ctx.isNameSpace()) + const MDScope *Ctx = *I; + StringRef Name = Ctx->getName(); + if (Name.empty() && isa(Ctx)) Name = "(anonymous namespace)"; if (!Name.empty()) { CS += Name; @@ -944,31 +931,31 @@ std::string DwarfUnit::getParentContextString(DIScope Context) const { /// constructTypeDIE - Construct basic type die from DIBasicType. void DwarfUnit::constructTypeDIE(DIE &Buffer, DIBasicType BTy) { // Get core information. - StringRef Name = BTy.getName(); + StringRef Name = BTy->getName(); // Add name if not anonymous or intermediate type. if (!Name.empty()) addString(Buffer, dwarf::DW_AT_name, Name); // An unspecified type only has a name attribute. - if (BTy.getTag() == dwarf::DW_TAG_unspecified_type) + if (BTy->getTag() == dwarf::DW_TAG_unspecified_type) return; addUInt(Buffer, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, - BTy.getEncoding()); + BTy->getEncoding()); - uint64_t Size = BTy.getSizeInBits() >> 3; + uint64_t Size = BTy->getSizeInBits() >> 3; addUInt(Buffer, dwarf::DW_AT_byte_size, None, Size); } /// constructTypeDIE - Construct derived type die from DIDerivedType. void DwarfUnit::constructTypeDIE(DIE &Buffer, DIDerivedType DTy) { // Get core information. - StringRef Name = DTy.getName(); - uint64_t Size = DTy.getSizeInBits() >> 3; + StringRef Name = DTy->getName(); + uint64_t Size = DTy->getSizeInBits() >> 3; uint16_t Tag = Buffer.getTag(); // Map to main type, void will not have a type. - DIType FromTy = resolve(DTy.getTypeDerivedFrom()); + DIType FromTy = resolve(DTy->getBaseType()); if (FromTy) addType(Buffer, FromTy); @@ -982,24 +969,25 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DIDerivedType DTy) { addUInt(Buffer, dwarf::DW_AT_byte_size, None, Size); if (Tag == dwarf::DW_TAG_ptr_to_member_type) - addDIEEntry(Buffer, dwarf::DW_AT_containing_type, - *getOrCreateTypeDIE(resolve(DTy.getClassType()))); + addDIEEntry( + Buffer, dwarf::DW_AT_containing_type, + *getOrCreateTypeDIE(resolve(cast(DTy)->getClassType()))); // Add source line info if available and TyDesc is not a forward declaration. - if (!DTy.isForwardDecl()) + if (!DTy->isForwardDecl()) addSourceLine(Buffer, DTy); } /// constructSubprogramArguments - Construct function argument DIEs. void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeArray Args) { - for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) { - DIType Ty = resolve(Args.getElement(i)); + for (unsigned i = 1, N = Args.size(); i < N; ++i) { + DIType Ty = resolve(Args[i]); if (!Ty) { assert(i == N-1 && "Unspecified parameter must be the last argument"); createAndAddDIE(dwarf::DW_TAG_unspecified_parameters, Buffer); } else { DIE &Arg = createAndAddDIE(dwarf::DW_TAG_formal_parameter, Buffer); addType(Arg, Ty); - if (Ty.isArtificial()) + if (Ty->isArtificial()) addFlag(Arg, dwarf::DW_AT_artificial); } } @@ -1008,9 +996,9 @@ void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeArray Args) { /// constructTypeDIE - Construct type DIE from DICompositeType. void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { // Add name if not anonymous or intermediate type. - StringRef Name = CTy.getName(); + StringRef Name = CTy->getName(); - uint64_t Size = CTy.getSizeInBits() >> 3; + uint64_t Size = CTy->getSizeInBits() >> 3; uint16_t Tag = Buffer.getTag(); switch (Tag) { @@ -1022,14 +1010,13 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { break; case dwarf::DW_TAG_subroutine_type: { // Add return type. A void return won't have a type. - DITypeArray Elements = DISubroutineType(CTy).getTypeArray(); - DIType RTy(resolve(Elements.getElement(0))); - if (RTy) - addType(Buffer, RTy); + auto Elements = cast(CTy)->getTypeArray(); + if (Elements.size()) + if (auto RTy = resolve(Elements[0])) + addType(Buffer, RTy); bool isPrototyped = true; - if (Elements.getNumElements() == 2 && - !Elements.getElement(1)) + if (Elements.size() == 2 && !Elements[1]) isPrototyped = false; constructSubprogramArguments(Buffer, Elements); @@ -1042,60 +1029,46 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { Language == dwarf::DW_LANG_ObjC)) addFlag(Buffer, dwarf::DW_AT_prototyped); - if (CTy.isLValueReference()) + if (CTy->isLValueReference()) addFlag(Buffer, dwarf::DW_AT_reference); - if (CTy.isRValueReference()) + if (CTy->isRValueReference()) addFlag(Buffer, dwarf::DW_AT_rvalue_reference); } break; case dwarf::DW_TAG_structure_type: case dwarf::DW_TAG_union_type: case dwarf::DW_TAG_class_type: { // Add elements to structure type. - DIArray Elements = CTy.getElements(); - for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { - DIDescriptor Element = Elements.getElement(i); - if (Element.isSubprogram()) - getOrCreateSubprogramDIE(DISubprogram(Element)); - else if (Element.isDerivedType()) { - DIDerivedType DDTy(Element); - if (DDTy.getTag() == dwarf::DW_TAG_friend) { + DIArray Elements = CTy->getElements(); + for (unsigned i = 0, N = Elements.size(); i < N; ++i) { + DIDescriptor Element = Elements[i]; + if (!Element) + continue; + if (auto *SP = dyn_cast(Element)) + getOrCreateSubprogramDIE(SP); + else if (DIDerivedType DDTy = dyn_cast(Element)) { + if (DDTy->getTag() == dwarf::DW_TAG_friend) { DIE &ElemDie = createAndAddDIE(dwarf::DW_TAG_friend, Buffer); - addType(ElemDie, resolve(DDTy.getTypeDerivedFrom()), - dwarf::DW_AT_friend); - } else if (DDTy.isStaticMember()) { + addType(ElemDie, resolve(DDTy->getBaseType()), dwarf::DW_AT_friend); + } else if (DDTy->isStaticMember()) { getOrCreateStaticMemberDIE(DDTy); } else { constructMemberDIE(Buffer, DDTy); } - } else if (Element.isObjCProperty()) { - DIObjCProperty Property(Element); - DIE &ElemDie = createAndAddDIE(Property.getTag(), Buffer); - StringRef PropertyName = Property.getObjCPropertyName(); + } else if (DIObjCProperty Property = dyn_cast(Element)) { + DIE &ElemDie = createAndAddDIE(Property->getTag(), Buffer); + StringRef PropertyName = Property->getName(); addString(ElemDie, dwarf::DW_AT_APPLE_property_name, PropertyName); - if (Property.getType()) - addType(ElemDie, Property.getType()); + if (Property->getType()) + addType(ElemDie, Property->getType()); addSourceLine(ElemDie, Property); - StringRef GetterName = Property.getObjCPropertyGetterName(); + StringRef GetterName = Property->getGetterName(); if (!GetterName.empty()) addString(ElemDie, dwarf::DW_AT_APPLE_property_getter, GetterName); - StringRef SetterName = Property.getObjCPropertySetterName(); + StringRef SetterName = Property->getSetterName(); if (!SetterName.empty()) addString(ElemDie, dwarf::DW_AT_APPLE_property_setter, SetterName); - unsigned PropertyAttributes = 0; - if (Property.isReadOnlyObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_readonly; - if (Property.isReadWriteObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_readwrite; - if (Property.isAssignObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_assign; - if (Property.isRetainObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_retain; - if (Property.isCopyObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_copy; - if (Property.isNonAtomicObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_nonatomic; - if (PropertyAttributes) + if (unsigned PropertyAttributes = Property->getAttributes()) addUInt(ElemDie, dwarf::DW_AT_APPLE_property_attribute, None, PropertyAttributes); @@ -1104,28 +1077,27 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { Entry = createDIEEntry(ElemDie); insertDIEEntry(Element, Entry); } - } else - continue; + } } - if (CTy.isAppleBlockExtension()) + if (CTy->isAppleBlockExtension()) addFlag(Buffer, dwarf::DW_AT_APPLE_block); // This is outside the DWARF spec, but GDB expects a DW_AT_containing_type // inside C++ composite types to point to the base class with the vtable. - DICompositeType ContainingType(resolve(CTy.getContainingType())); - if (ContainingType) + if (DICompositeType ContainingType = + dyn_cast_or_null(resolve(CTy->getVTableHolder()))) addDIEEntry(Buffer, dwarf::DW_AT_containing_type, *getOrCreateTypeDIE(ContainingType)); - if (CTy.isObjcClassComplete()) + if (CTy->isObjcClassComplete()) addFlag(Buffer, dwarf::DW_AT_APPLE_objc_complete_type); // Add template parameters to a class, structure or union types. // FIXME: The support isn't in the metadata for this yet. if (Tag == dwarf::DW_TAG_class_type || Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type) - addTemplateParams(Buffer, CTy.getTemplateParams()); + addTemplateParams(Buffer, CTy->getTemplateParams()); break; } @@ -1144,20 +1116,20 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { // TODO: Do we care about size for enum forward declarations? if (Size) addUInt(Buffer, dwarf::DW_AT_byte_size, None, Size); - else if (!CTy.isForwardDecl()) + else if (!CTy->isForwardDecl()) // Add zero size if it is not a forward declaration. addUInt(Buffer, dwarf::DW_AT_byte_size, None, 0); // If we're a forward decl, say so. - if (CTy.isForwardDecl()) + if (CTy->isForwardDecl()) addFlag(Buffer, dwarf::DW_AT_declaration); // Add source line info if available. - if (!CTy.isForwardDecl()) + if (!CTy->isForwardDecl()) addSourceLine(Buffer, CTy); // No harm in adding the runtime language to the declaration. - unsigned RLang = CTy.getRunTimeLang(); + unsigned RLang = CTy->getRuntimeLang(); if (RLang) addUInt(Buffer, dwarf::DW_AT_APPLE_runtime_class, dwarf::DW_FORM_data1, RLang); @@ -1171,10 +1143,10 @@ void DwarfUnit::constructTemplateTypeParameterDIE(DIE &Buffer, DIE &ParamDIE = createAndAddDIE(dwarf::DW_TAG_template_type_parameter, Buffer); // Add the type if it exists, it could be void and therefore no type. - if (TP.getType()) - addType(ParamDIE, resolve(TP.getType())); - if (!TP.getName().empty()) - addString(ParamDIE, dwarf::DW_AT_name, TP.getName()); + if (TP->getType()) + addType(ParamDIE, resolve(TP->getType())); + if (!TP->getName().empty()) + addString(ParamDIE, dwarf::DW_AT_name, TP->getName()); } /// constructTemplateValueParameterDIE - Construct new DIE for the given @@ -1182,17 +1154,17 @@ void DwarfUnit::constructTemplateTypeParameterDIE(DIE &Buffer, void DwarfUnit::constructTemplateValueParameterDIE(DIE &Buffer, DITemplateValueParameter VP) { - DIE &ParamDIE = createAndAddDIE(VP.getTag(), Buffer); + DIE &ParamDIE = createAndAddDIE(VP->getTag(), Buffer); // Add the type if there is one, template template and template parameter // packs will not have a type. - if (VP.getTag() == dwarf::DW_TAG_template_value_parameter) - addType(ParamDIE, resolve(VP.getType())); - if (!VP.getName().empty()) - addString(ParamDIE, dwarf::DW_AT_name, VP.getName()); - if (Metadata *Val = VP.getValue()) { + if (VP->getTag() == dwarf::DW_TAG_template_value_parameter) + addType(ParamDIE, resolve(VP->getType())); + if (!VP->getName().empty()) + addString(ParamDIE, dwarf::DW_AT_name, VP->getName()); + if (Metadata *Val = VP->getValue()) { if (ConstantInt *CI = mdconst::dyn_extract(Val)) - addConstantValue(ParamDIE, CI, resolve(VP.getType())); + addConstantValue(ParamDIE, CI, resolve(VP->getType())); else if (GlobalValue *GV = mdconst::dyn_extract(Val)) { // For declaration non-type template parameters (such as global values and // functions) @@ -1202,14 +1174,12 @@ DwarfUnit::constructTemplateValueParameterDIE(DIE &Buffer, // parameter, rather than a pointer to it. addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_stack_value); addBlock(ParamDIE, dwarf::DW_AT_location, Loc); - } else if (VP.getTag() == dwarf::DW_TAG_GNU_template_template_param) { + } else if (VP->getTag() == dwarf::DW_TAG_GNU_template_template_param) { assert(isa(Val)); addString(ParamDIE, dwarf::DW_AT_GNU_template_name, cast(Val)->getString()); - } else if (VP.getTag() == dwarf::DW_TAG_GNU_template_parameter_pack) { - assert(isa(Val)); - DIArray A(cast(Val)); - addTemplateParams(ParamDIE, A); + } else if (VP->getTag() == dwarf::DW_TAG_GNU_template_parameter_pack) { + addTemplateParams(ParamDIE, cast(Val)); } } } @@ -1218,19 +1188,19 @@ DwarfUnit::constructTemplateValueParameterDIE(DIE &Buffer, DIE *DwarfUnit::getOrCreateNameSpace(DINameSpace NS) { // Construct the context before querying for the existence of the DIE in case // such construction creates the DIE. - DIE *ContextDIE = getOrCreateContextDIE(NS.getContext()); + DIE *ContextDIE = getOrCreateContextDIE(NS->getScope()); if (DIE *NDie = getDIE(NS)) return NDie; DIE &NDie = createAndAddDIE(dwarf::DW_TAG_namespace, *ContextDIE, NS); - StringRef Name = NS.getName(); + StringRef Name = NS->getName(); if (!Name.empty()) - addString(NDie, dwarf::DW_AT_name, NS.getName()); + addString(NDie, dwarf::DW_AT_name, NS->getName()); else Name = "(anonymous namespace)"; DD->addAccelNamespace(Name, NDie); - addGlobalName(Name, NDie, NS.getContext()); + addGlobalName(Name, NDie, NS->getScope()); addSourceLine(NDie, NS); return &NDie; } @@ -1241,12 +1211,12 @@ DIE *DwarfUnit::getOrCreateSubprogramDIE(DISubprogram SP, bool Minimal) { // such construction creates the DIE (as is the case for member function // declarations). DIE *ContextDIE = - Minimal ? &getUnitDie() : getOrCreateContextDIE(resolve(SP.getContext())); + Minimal ? &getUnitDie() : getOrCreateContextDIE(resolve(SP->getScope())); if (DIE *SPDie = getDIE(SP)) return SPDie; - if (DISubprogram SPDecl = SP.getFunctionDeclaration()) { + if (auto *SPDecl = SP->getDeclaration()) { if (!Minimal) { // Add subprogram definitions to the CU die directly. ContextDIE = &getUnitDie(); @@ -1260,7 +1230,7 @@ DIE *DwarfUnit::getOrCreateSubprogramDIE(DISubprogram SP, bool Minimal) { // Stop here and fill this in later, depending on whether or not this // subprogram turns out to have inlined instances or not. - if (SP.isDefinition()) + if (SP->isDefinition()) return &SPDie; applySubprogramAttributes(SP, SPDie); @@ -1271,19 +1241,19 @@ bool DwarfUnit::applySubprogramDefinitionAttributes(DISubprogram SP, DIE &SPDie) { DIE *DeclDie = nullptr; StringRef DeclLinkageName; - if (DISubprogram SPDecl = SP.getFunctionDeclaration()) { + if (auto *SPDecl = SP->getDeclaration()) { DeclDie = getDIE(SPDecl); assert(DeclDie && "This DIE should've already been constructed when the " "definition DIE was created in " "getOrCreateSubprogramDIE"); - DeclLinkageName = SPDecl.getLinkageName(); + DeclLinkageName = SPDecl->getLinkageName(); } // Add function template parameters. - addTemplateParams(SPDie, SP.getTemplateParams()); + addTemplateParams(SPDie, SP->getTemplateParams()); // Add the linkage name if we have one and it isn't in the Decl. - StringRef LinkageName = SP.getLinkageName(); + StringRef LinkageName = SP->getLinkageName(); assert(((LinkageName.empty() || DeclLinkageName.empty()) || LinkageName == DeclLinkageName) && "decl has a linkage name and it is different"); @@ -1306,8 +1276,8 @@ void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie, return; // Constructors and operators for anonymous aggregates do not have names. - if (!SP.getName().empty()) - addString(SPDie, dwarf::DW_AT_name, SP.getName()); + if (!SP->getName().empty()) + addString(SPDie, dwarf::DW_AT_name, SP->getName()); // Skip the rest of the attributes under -gmlt to save space. if (Minimal) @@ -1318,33 +1288,34 @@ void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie, // Add the prototype if we have a prototype and we have a C like // language. uint16_t Language = getLanguage(); - if (SP.isPrototyped() && + if (SP->isPrototyped() && (Language == dwarf::DW_LANG_C89 || Language == dwarf::DW_LANG_C99 || Language == dwarf::DW_LANG_ObjC)) addFlag(SPDie, dwarf::DW_AT_prototyped); - DISubroutineType SPTy = SP.getType(); - assert(SPTy.getTag() == dwarf::DW_TAG_subroutine_type && + DISubroutineType SPTy = SP->getType(); + assert(SPTy->getTag() == dwarf::DW_TAG_subroutine_type && "the type of a subprogram should be a subroutine"); - DITypeArray Args = SPTy.getTypeArray(); + auto Args = SPTy->getTypeArray(); // Add a return type. If this is a type like a C/C++ void type we don't add a // return type. - if (resolve(Args.getElement(0))) - addType(SPDie, DIType(resolve(Args.getElement(0)))); + if (Args.size()) + if (auto Ty = resolve(Args[0])) + addType(SPDie, Ty); - unsigned VK = SP.getVirtuality(); + unsigned VK = SP->getVirtuality(); if (VK) { addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_data1, VK); DIELoc *Block = getDIELoc(); addUInt(*Block, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); - addUInt(*Block, dwarf::DW_FORM_udata, SP.getVirtualIndex()); + addUInt(*Block, dwarf::DW_FORM_udata, SP->getVirtualIndex()); addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, Block); ContainingTypeMap.insert( - std::make_pair(&SPDie, resolve(SP.getContainingType()))); + std::make_pair(&SPDie, resolve(SP->getContainingType()))); } - if (!SP.isDefinition()) { + if (!SP->isDefinition()) { addFlag(SPDie, dwarf::DW_AT_declaration); // Add arguments. Do not add arguments for subprogram definition. They will @@ -1352,35 +1323,35 @@ void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie, constructSubprogramArguments(SPDie, Args); } - if (SP.isArtificial()) + if (SP->isArtificial()) addFlag(SPDie, dwarf::DW_AT_artificial); - if (!SP.isLocalToUnit()) + if (!SP->isLocalToUnit()) addFlag(SPDie, dwarf::DW_AT_external); - if (SP.isOptimized()) + if (SP->isOptimized()) addFlag(SPDie, dwarf::DW_AT_APPLE_optimized); if (unsigned isa = Asm->getISAEncoding()) addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa); - if (SP.isLValueReference()) + if (SP->isLValueReference()) addFlag(SPDie, dwarf::DW_AT_reference); - if (SP.isRValueReference()) + if (SP->isRValueReference()) addFlag(SPDie, dwarf::DW_AT_rvalue_reference); - if (SP.isProtected()) + if (SP->isProtected()) addUInt(SPDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_protected); - else if (SP.isPrivate()) + else if (SP->isPrivate()) addUInt(SPDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); - else if (SP.isPublic()) + else if (SP->isPublic()) addUInt(SPDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); - if (SP.isExplicit()) + if (SP->isExplicit()) addFlag(SPDie, dwarf::DW_AT_explicit); } @@ -1393,9 +1364,9 @@ void DwarfUnit::constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy) { // C/C++. The Count value is the number of elements. Values are 64 bit. If // Count == -1 then the array is unbounded and we do not emit // DW_AT_lower_bound and DW_AT_count attributes. - int64_t LowerBound = SR.getLo(); + int64_t LowerBound = SR->getLowerBound(); int64_t DefaultLowerBound = getDefaultLowerBound(); - int64_t Count = SR.getCount(); + int64_t Count = SR->getCount(); if (DefaultLowerBound == -1 || LowerBound != DefaultLowerBound) addUInt(DW_Subrange, dwarf::DW_AT_lower_bound, None, LowerBound); @@ -1420,11 +1391,11 @@ DIE *DwarfUnit::getIndexTyDie() { /// constructArrayTypeDIE - Construct array type DIE from DICompositeType. void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, DICompositeType CTy) { - if (CTy.isVector()) + if (CTy->isVector()) addFlag(Buffer, dwarf::DW_AT_GNU_vector); // Emit the element type. - addType(Buffer, resolve(CTy.getTypeDerivedFrom())); + addType(Buffer, resolve(CTy->getBaseType())); // Get an anonymous type for index type. // FIXME: This type should be passed down from the front end @@ -1432,31 +1403,32 @@ void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, DICompositeType CTy) { DIE *IdxTy = getIndexTyDie(); // Add subranges to array type. - DIArray Elements = CTy.getElements(); - for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { - DIDescriptor Element = Elements.getElement(i); - if (Element.getTag() == dwarf::DW_TAG_subrange_type) - constructSubrangeDIE(Buffer, DISubrange(Element), IdxTy); + DIArray Elements = CTy->getElements(); + for (unsigned i = 0, N = Elements.size(); i < N; ++i) { + // FIXME: Should this really be such a loose cast? + if (auto *Element = dyn_cast_or_null(Elements[i])) + if (Element->getTag() == dwarf::DW_TAG_subrange_type) + constructSubrangeDIE(Buffer, cast(Element), IdxTy); } } /// constructEnumTypeDIE - Construct an enum type DIE from DICompositeType. void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, DICompositeType CTy) { - DIArray Elements = CTy.getElements(); + DIArray Elements = CTy->getElements(); // Add enumerators to enumeration type. - for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { - DIEnumerator Enum(Elements.getElement(i)); - if (Enum.isEnumerator()) { + for (unsigned i = 0, N = Elements.size(); i < N; ++i) { + auto *Enum = dyn_cast_or_null(Elements[i]); + if (Enum) { DIE &Enumerator = createAndAddDIE(dwarf::DW_TAG_enumerator, Buffer); - StringRef Name = Enum.getName(); + StringRef Name = Enum->getName(); addString(Enumerator, dwarf::DW_AT_name, Name); - int64_t Value = Enum.getEnumValue(); + int64_t Value = Enum->getValue(); addSInt(Enumerator, dwarf::DW_AT_const_value, dwarf::DW_FORM_sdata, Value); } } - DIType DTy = resolve(CTy.getTypeDerivedFrom()); + DIType DTy = resolve(CTy->getBaseType()); if (DTy) { addType(Buffer, DTy); addFlag(Buffer, dwarf::DW_AT_enum_class); @@ -1481,17 +1453,20 @@ void DwarfUnit::constructContainingTypeDIEs() { } /// constructMemberDIE - Construct member DIE from DIDerivedType. -void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT) { - DIE &MemberDie = createAndAddDIE(DT.getTag(), Buffer); - StringRef Name = DT.getName(); +void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT_) { + // Downcast to MDDerivedType. + const MDDerivedType *DT = cast(DT_); + + DIE &MemberDie = createAndAddDIE(DT->getTag(), Buffer); + StringRef Name = DT->getName(); if (!Name.empty()) addString(MemberDie, dwarf::DW_AT_name, Name); - addType(MemberDie, resolve(DT.getTypeDerivedFrom())); + addType(MemberDie, resolve(DT->getBaseType())); addSourceLine(MemberDie, DT); - if (DT.getTag() == dwarf::DW_TAG_inheritance && DT.isVirtual()) { + if (DT->getTag() == dwarf::DW_TAG_inheritance && DT->isVirtual()) { // For C++, virtual base classes are not at fixed offset. Use following // expression to extract appropriate offset from vtable. @@ -1501,14 +1476,14 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT) { addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_dup); addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_deref); addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); - addUInt(*VBaseLocationDie, dwarf::DW_FORM_udata, DT.getOffsetInBits()); + addUInt(*VBaseLocationDie, dwarf::DW_FORM_udata, DT->getOffsetInBits()); addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_minus); addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_deref); addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_plus); addBlock(MemberDie, dwarf::DW_AT_data_member_location, VBaseLocationDie); } else { - uint64_t Size = DT.getSizeInBits(); + uint64_t Size = DT->getSizeInBits(); uint64_t FieldSize = getBaseTypeSize(DD, DT); uint64_t OffsetInBytes; @@ -1517,8 +1492,8 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT) { addUInt(MemberDie, dwarf::DW_AT_byte_size, None, FieldSize/8); addUInt(MemberDie, dwarf::DW_AT_bit_size, None, Size); - uint64_t Offset = DT.getOffsetInBits(); - uint64_t AlignMask = ~(DT.getAlignInBits() - 1); + uint64_t Offset = DT->getOffsetInBits(); + uint64_t AlignMask = ~(DT->getAlignInBits() - 1); uint64_t HiMark = (Offset + FieldSize) & AlignMask; uint64_t FieldOffset = (HiMark - FieldSize); Offset -= FieldOffset; @@ -1533,7 +1508,7 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT) { OffsetInBytes = FieldOffset >> 3; } else // This is not a bitfield. - OffsetInBytes = DT.getOffsetInBits() >> 3; + OffsetInBytes = DT->getOffsetInBits() >> 3; if (DD->getDwarfVersion() <= 2) { DIELoc *MemLocationDie = new (DIEValueAllocator) DIELoc(); @@ -1545,49 +1520,50 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT) { OffsetInBytes); } - if (DT.isProtected()) + if (DT->isProtected()) addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_protected); - else if (DT.isPrivate()) + else if (DT->isPrivate()) addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); // Otherwise C++ member and base classes are considered public. - else if (DT.isPublic()) + else if (DT->isPublic()) addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); - if (DT.isVirtual()) + if (DT->isVirtual()) addUInt(MemberDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_data1, dwarf::DW_VIRTUALITY_virtual); // Objective-C properties. - if (MDNode *PNode = DT.getObjCProperty()) + if (MDNode *PNode = DT->getObjCProperty()) if (DIEEntry *PropertyDie = getDIEEntry(PNode)) MemberDie.addValue(dwarf::DW_AT_APPLE_property, dwarf::DW_FORM_ref4, PropertyDie); - if (DT.isArtificial()) + if (DT->isArtificial()) addFlag(MemberDie, dwarf::DW_AT_artificial); } /// getOrCreateStaticMemberDIE - Create new DIE for C++ static member. -DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT) { - if (!DT.Verify()) +DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT_) { + const MDDerivedType *DT = cast_or_null(DT_); + if (!DT) return nullptr; // Construct the context before querying for the existence of the DIE in case // such construction creates the DIE. - DIE *ContextDIE = getOrCreateContextDIE(resolve(DT.getContext())); + DIE *ContextDIE = getOrCreateContextDIE(resolve(DT->getScope())); assert(dwarf::isType(ContextDIE->getTag()) && "Static member should belong to a type."); if (DIE *StaticMemberDIE = getDIE(DT)) return StaticMemberDIE; - DIE &StaticMemberDIE = createAndAddDIE(DT.getTag(), *ContextDIE, DT); + DIE &StaticMemberDIE = createAndAddDIE(DT->getTag(), *ContextDIE, DT); - DIType Ty = resolve(DT.getTypeDerivedFrom()); + DIType Ty = resolve(DT->getBaseType()); - addString(StaticMemberDIE, dwarf::DW_AT_name, DT.getName()); + addString(StaticMemberDIE, dwarf::DW_AT_name, DT->getName()); addType(StaticMemberDIE, Ty); addSourceLine(StaticMemberDIE, DT); addFlag(StaticMemberDIE, dwarf::DW_AT_external); @@ -1595,19 +1571,19 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT) { // FIXME: We could omit private if the parent is a class_type, and // public if the parent is something else. - if (DT.isProtected()) + if (DT->isProtected()) addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_protected); - else if (DT.isPrivate()) + else if (DT->isPrivate()) addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); - else if (DT.isPublic()) + else if (DT->isPublic()) addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); - if (const ConstantInt *CI = dyn_cast_or_null(DT.getConstant())) + if (const ConstantInt *CI = dyn_cast_or_null(DT->getConstant())) addConstantValue(StaticMemberDIE, CI, Ty); - if (const ConstantFP *CFP = dyn_cast_or_null(DT.getConstant())) + if (const ConstantFP *CFP = dyn_cast_or_null(DT->getConstant())) addConstantFPValue(StaticMemberDIE, CFP); return &StaticMemberDIE; diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.h b/lib/CodeGen/AsmPrinter/DwarfUnit.h index 81c5821..b354255 100644 --- a/lib/CodeGen/AsmPrinter/DwarfUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfUnit.h @@ -141,7 +141,7 @@ public: // Accessors. AsmPrinter* getAsmPrinter() const { return Asm; } unsigned getUniqueID() const { return UniqueID; } - uint16_t getLanguage() const { return CUNode.getLanguage(); } + uint16_t getLanguage() const { return CUNode->getSourceLanguage(); } DICompileUnit getCUNode() const { return CUNode; } DIE &getUnitDie() { return UnitDie; } @@ -342,7 +342,7 @@ protected: /// resolve - Look in the DwarfDebug map for the MDNode that /// corresponds to the reference. - template T resolve(DIRef Ref) const { + template T *resolve(TypedDebugNodeRef Ref) const { return DD->resolve(Ref); } diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/lib/CodeGen/AsmPrinter/EHStreamer.cpp index 14df4c9..6f64d8f 100644 --- a/lib/CodeGen/AsmPrinter/EHStreamer.cpp +++ b/lib/CodeGen/AsmPrinter/EHStreamer.cpp @@ -188,20 +188,12 @@ bool EHStreamer::callToNoUnwindFunction(const MachineInstr *MI) { return MarkedNoUnwind; } -/// Compute the call-site table. The entry for an invoke has a try-range -/// containing the call, a non-zero landing pad, and an appropriate action. The -/// entry for an ordinary call has a try-range containing the call and zero for -/// the landing pad and the action. Calls marked 'nounwind' have no entry and -/// must not be contained in the try-range of any entry - they form gaps in the -/// table. Entries must be ordered by try-range address. -void EHStreamer:: -computeCallSiteTable(SmallVectorImpl &CallSites, - const SmallVectorImpl &LandingPads, - const SmallVectorImpl &FirstActions) { +void EHStreamer::computePadMap( + const SmallVectorImpl &LandingPads, + RangeMapType &PadMap) { // Invokes and nounwind calls have entries in PadMap (due to being bracketed // by try-range labels when lowered). Ordinary calls do not, so appropriate // try-ranges for them need be deduced so we can put them in the LSDA. - RangeMapType PadMap; for (unsigned i = 0, N = LandingPads.size(); i != N; ++i) { const LandingPadInfo *LandingPad = LandingPads[i]; for (unsigned j = 0, E = LandingPad->BeginLabels.size(); j != E; ++j) { @@ -211,6 +203,20 @@ computeCallSiteTable(SmallVectorImpl &CallSites, PadMap[BeginLabel] = P; } } +} + +/// Compute the call-site table. The entry for an invoke has a try-range +/// containing the call, a non-zero landing pad, and an appropriate action. The +/// entry for an ordinary call has a try-range containing the call and zero for +/// the landing pad and the action. Calls marked 'nounwind' have no entry and +/// must not be contained in the try-range of any entry - they form gaps in the +/// table. Entries must be ordered by try-range address. +void EHStreamer:: +computeCallSiteTable(SmallVectorImpl &CallSites, + const SmallVectorImpl &LandingPads, + const SmallVectorImpl &FirstActions) { + RangeMapType PadMap; + computePadMap(LandingPads, PadMap); // The end label of the previous invoke or nounwind try-range. MCSymbol *LastLabel = nullptr; diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.h b/lib/CodeGen/AsmPrinter/EHStreamer.h index 94d0585..65973fa 100644 --- a/lib/CodeGen/AsmPrinter/EHStreamer.h +++ b/lib/CodeGen/AsmPrinter/EHStreamer.h @@ -80,13 +80,15 @@ protected: /// `false' otherwise. bool callToNoUnwindFunction(const MachineInstr *MI); + void computePadMap(const SmallVectorImpl &LandingPads, + RangeMapType &PadMap); + /// Compute the call-site table. The entry for an invoke has a try-range /// containing the call, a non-zero landing pad and an appropriate action. /// The entry for an ordinary call has a try-range containing the call and /// zero for the landing pad and the action. Calls marked 'nounwind' have /// no entry and must not be contained in the try-range of any entry - they /// form gaps in the table. Entries must be ordered by try-range address. - void computeCallSiteTable(SmallVectorImpl &CallSites, const SmallVectorImpl &LPs, const SmallVectorImpl &FirstActions); @@ -123,7 +125,7 @@ protected: public: EHStreamer(AsmPrinter *A); - virtual ~EHStreamer(); + ~EHStreamer() override; // Unused. void setSymbolSize(const MCSymbol *Sym, uint64_t Size) override {} diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/lib/CodeGen/AsmPrinter/Win64Exception.cpp index 7d76ead..f89d364 100644 --- a/lib/CodeGen/AsmPrinter/Win64Exception.cpp +++ b/lib/CodeGen/AsmPrinter/Win64Exception.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" @@ -67,6 +68,27 @@ void Win64Exception::beginFunction(const MachineFunction *MF) { shouldEmitLSDA = shouldEmitPersonality && LSDAEncoding != dwarf::DW_EH_PE_omit; + + // If this was an outlined handler, we need to define the label corresponding + // to the offset of the parent frame relative to the stack pointer after the + // prologue. + const Function *F = MF->getFunction(); + const Function *ParentF = MMI->getWinEHParent(F); + if (F != ParentF) { + WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(ParentF); + auto I = FuncInfo.CatchHandlerParentFrameObjOffset.find(F); + if (I != FuncInfo.CatchHandlerParentFrameObjOffset.end()) { + MCSymbol *HandlerTypeParentFrameOffset = + Asm->OutContext.getOrCreateParentFrameOffsetSymbol( + GlobalValue::getRealLinkageName(F->getName())); + + // Emit a symbol assignment. + Asm->OutStreamer.EmitAssignment( + HandlerTypeParentFrameOffset, + MCConstantExpr::Create(I->second, Asm->OutContext)); + } + } + if (!shouldEmitPersonality && !shouldEmitMoves) return; @@ -82,12 +104,17 @@ void Win64Exception::beginFunction(const MachineFunction *MF) { /// endFunction - Gather and emit post-function exception information. /// -void Win64Exception::endFunction(const MachineFunction *) { +void Win64Exception::endFunction(const MachineFunction *MF) { if (!shouldEmitPersonality && !shouldEmitMoves) return; - // Map all labels and get rid of any dead landing pads. - MMI->TidyLandingPads(); + EHPersonality Per = MMI->getPersonalityType(); + + // Get rid of any dead landing pads if we're not using a Windows EH scheme. In + // Windows EH schemes, the landing pad is not actually reachable. It only + // exists so that we can emit the right table data. + if (!isMSVCEHPersonality(Per)) + MMI->TidyLandingPads(); if (shouldEmitPersonality) { Asm->OutStreamer.PushSection(); @@ -97,9 +124,10 @@ void Win64Exception::endFunction(const MachineFunction *) { // Emit the tables appropriate to the personality function in use. If we // don't recognize the personality, assume it uses an Itanium-style LSDA. - EHPersonality Per = MMI->getPersonalityType(); if (Per == EHPersonality::MSVC_Win64SEH) emitCSpecificHandlerTable(); + else if (Per == EHPersonality::MSVC_CXX) + emitCXXFrameHandler3Table(MF); else emitExceptionTable(); @@ -108,11 +136,19 @@ void Win64Exception::endFunction(const MachineFunction *) { Asm->OutStreamer.EmitWinCFIEndProc(); } -const MCSymbolRefExpr *Win64Exception::createImageRel32(const MCSymbol *Value) { +const MCExpr *Win64Exception::createImageRel32(const MCSymbol *Value) { + if (!Value) + return MCConstantExpr::Create(0, Asm->OutContext); return MCSymbolRefExpr::Create(Value, MCSymbolRefExpr::VK_COFF_IMGREL32, Asm->OutContext); } +const MCExpr *Win64Exception::createImageRel32(const GlobalValue *GV) { + if (!GV) + return MCConstantExpr::Create(0, Asm->OutContext); + return createImageRel32(Asm->getSymbol(GV)); +} + /// Emit the language-specific data that __C_specific_handler expects. This /// handler lives in the x64 Microsoft C runtime and allows catching or cleaning /// up after faults with __try, __except, and __finally. The typeinfo values @@ -237,3 +273,231 @@ void Win64Exception::emitCSpecificHandlerTable() { } } } + +void Win64Exception::emitCXXFrameHandler3Table(const MachineFunction *MF) { + const Function *F = MF->getFunction(); + const Function *ParentF = MMI->getWinEHParent(F); + auto &OS = Asm->OutStreamer; + WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(ParentF); + + StringRef ParentLinkageName = + GlobalValue::getRealLinkageName(ParentF->getName()); + + MCSymbol *FuncInfoXData = + Asm->OutContext.GetOrCreateSymbol(Twine("$cppxdata$", ParentLinkageName)); + OS.EmitValue(createImageRel32(FuncInfoXData), 4); + + // The Itanium LSDA table sorts similar landing pads together to simplify the + // actions table, but we don't need that. + SmallVector LandingPads; + const std::vector &PadInfos = MMI->getLandingPads(); + LandingPads.reserve(PadInfos.size()); + for (const auto &LP : PadInfos) + LandingPads.push_back(&LP); + + RangeMapType PadMap; + computePadMap(LandingPads, PadMap); + + // The end label of the previous invoke or nounwind try-range. + MCSymbol *LastLabel = Asm->getFunctionBegin(); + + // Whether there is a potentially throwing instruction (currently this means + // an ordinary call) between the end of the previous try-range and now. + bool SawPotentiallyThrowing = false; + + int LastEHState = -2; + + // The parent function and the catch handlers contribute to the 'ip2state' + // table. + for (const auto &MBB : *MF) { + for (const auto &MI : MBB) { + if (!MI.isEHLabel()) { + if (MI.isCall()) + SawPotentiallyThrowing |= !callToNoUnwindFunction(&MI); + continue; + } + + // End of the previous try-range? + MCSymbol *BeginLabel = MI.getOperand(0).getMCSymbol(); + if (BeginLabel == LastLabel) + SawPotentiallyThrowing = false; + + // Beginning of a new try-range? + RangeMapType::const_iterator L = PadMap.find(BeginLabel); + if (L == PadMap.end()) + // Nope, it was just some random label. + continue; + + const PadRange &P = L->second; + const LandingPadInfo *LandingPad = LandingPads[P.PadIndex]; + assert(BeginLabel == LandingPad->BeginLabels[P.RangeIndex] && + "Inconsistent landing pad map!"); + + if (SawPotentiallyThrowing) { + FuncInfo.IPToStateList.push_back(std::make_pair(LastLabel, -1)); + SawPotentiallyThrowing = false; + LastEHState = -1; + } + + if (LandingPad->WinEHState != LastEHState) + FuncInfo.IPToStateList.push_back( + std::make_pair(BeginLabel, LandingPad->WinEHState)); + LastEHState = LandingPad->WinEHState; + LastLabel = LandingPad->EndLabels[P.RangeIndex]; + } + } + + // Defer emission until we've visited the parent function and all the catch + // handlers. Cleanups don't contribute to the ip2state table yet, so don't + // count them. + if (ParentF != F && !FuncInfo.CatchHandlerMaxState.count(F)) + return; + ++FuncInfo.NumIPToStateFuncsVisited; + if (FuncInfo.NumIPToStateFuncsVisited != FuncInfo.CatchHandlerMaxState.size()) + return; + + MCSymbol *UnwindMapXData = nullptr; + MCSymbol *TryBlockMapXData = nullptr; + MCSymbol *IPToStateXData = nullptr; + if (!FuncInfo.UnwindMap.empty()) + UnwindMapXData = Asm->OutContext.GetOrCreateSymbol( + Twine("$stateUnwindMap$", ParentLinkageName)); + if (!FuncInfo.TryBlockMap.empty()) + TryBlockMapXData = Asm->OutContext.GetOrCreateSymbol( + Twine("$tryMap$", ParentLinkageName)); + if (!FuncInfo.IPToStateList.empty()) + IPToStateXData = Asm->OutContext.GetOrCreateSymbol( + Twine("$ip2state$", ParentLinkageName)); + + // FuncInfo { + // uint32_t MagicNumber + // int32_t MaxState; + // UnwindMapEntry *UnwindMap; + // uint32_t NumTryBlocks; + // TryBlockMapEntry *TryBlockMap; + // uint32_t IPMapEntries; + // IPToStateMapEntry *IPToStateMap; + // uint32_t UnwindHelp; // (x64/ARM only) + // ESTypeList *ESTypeList; + // int32_t EHFlags; + // } + // EHFlags & 1 -> Synchronous exceptions only, no async exceptions. + // EHFlags & 2 -> ??? + // EHFlags & 4 -> The function is noexcept(true), unwinding can't continue. + OS.EmitLabel(FuncInfoXData); + OS.EmitIntValue(0x19930522, 4); // MagicNumber + OS.EmitIntValue(FuncInfo.UnwindMap.size(), 4); // MaxState + OS.EmitValue(createImageRel32(UnwindMapXData), 4); // UnwindMap + OS.EmitIntValue(FuncInfo.TryBlockMap.size(), 4); // NumTryBlocks + OS.EmitValue(createImageRel32(TryBlockMapXData), 4); // TryBlockMap + OS.EmitIntValue(FuncInfo.IPToStateList.size(), 4); // IPMapEntries + OS.EmitValue(createImageRel32(IPToStateXData), 4); // IPToStateMap + OS.EmitIntValue(FuncInfo.UnwindHelpFrameOffset, 4); // UnwindHelp + OS.EmitIntValue(0, 4); // ESTypeList + OS.EmitIntValue(1, 4); // EHFlags + + // UnwindMapEntry { + // int32_t ToState; + // void (*Action)(); + // }; + if (UnwindMapXData) { + OS.EmitLabel(UnwindMapXData); + for (const WinEHUnwindMapEntry &UME : FuncInfo.UnwindMap) { + OS.EmitIntValue(UME.ToState, 4); // ToState + OS.EmitValue(createImageRel32(UME.Cleanup), 4); // Action + } + } + + // TryBlockMap { + // int32_t TryLow; + // int32_t TryHigh; + // int32_t CatchHigh; + // int32_t NumCatches; + // HandlerType *HandlerArray; + // }; + if (TryBlockMapXData) { + OS.EmitLabel(TryBlockMapXData); + SmallVector HandlerMaps; + for (size_t I = 0, E = FuncInfo.TryBlockMap.size(); I != E; ++I) { + WinEHTryBlockMapEntry &TBME = FuncInfo.TryBlockMap[I]; + MCSymbol *HandlerMapXData = nullptr; + + if (!TBME.HandlerArray.empty()) + HandlerMapXData = + Asm->OutContext.GetOrCreateSymbol(Twine("$handlerMap$") + .concat(Twine(I)) + .concat("$") + .concat(ParentLinkageName)); + + HandlerMaps.push_back(HandlerMapXData); + + int CatchHigh = -1; + for (WinEHHandlerType &HT : TBME.HandlerArray) + CatchHigh = + std::max(CatchHigh, FuncInfo.CatchHandlerMaxState[HT.Handler]); + + assert(TBME.TryLow <= TBME.TryHigh); + assert(CatchHigh > TBME.TryHigh); + OS.EmitIntValue(TBME.TryLow, 4); // TryLow + OS.EmitIntValue(TBME.TryHigh, 4); // TryHigh + OS.EmitIntValue(CatchHigh, 4); // CatchHigh + OS.EmitIntValue(TBME.HandlerArray.size(), 4); // NumCatches + OS.EmitValue(createImageRel32(HandlerMapXData), 4); // HandlerArray + } + + for (size_t I = 0, E = FuncInfo.TryBlockMap.size(); I != E; ++I) { + WinEHTryBlockMapEntry &TBME = FuncInfo.TryBlockMap[I]; + MCSymbol *HandlerMapXData = HandlerMaps[I]; + if (!HandlerMapXData) + continue; + // HandlerType { + // int32_t Adjectives; + // TypeDescriptor *Type; + // int32_t CatchObjOffset; + // void (*Handler)(); + // int32_t ParentFrameOffset; // x64 only + // }; + OS.EmitLabel(HandlerMapXData); + for (const WinEHHandlerType &HT : TBME.HandlerArray) { + MCSymbol *ParentFrameOffset = + Asm->OutContext.getOrCreateParentFrameOffsetSymbol( + GlobalValue::getRealLinkageName(HT.Handler->getName())); + const MCSymbolRefExpr *ParentFrameOffsetRef = MCSymbolRefExpr::Create( + ParentFrameOffset, MCSymbolRefExpr::VK_None, Asm->OutContext); + + // Get the frame escape label with the offset of the catch object. If + // the index is -1, then there is no catch object, and we should emit an + // offset of zero, indicating that no copy will occur. + const MCExpr *FrameAllocOffsetRef = nullptr; + if (HT.CatchObjRecoverIdx >= 0) { + MCSymbol *FrameAllocOffset = + Asm->OutContext.getOrCreateFrameAllocSymbol( + GlobalValue::getRealLinkageName(ParentF->getName()), + HT.CatchObjRecoverIdx); + FrameAllocOffsetRef = MCSymbolRefExpr::Create( + FrameAllocOffset, MCSymbolRefExpr::VK_None, Asm->OutContext); + } else { + FrameAllocOffsetRef = MCConstantExpr::Create(0, Asm->OutContext); + } + + OS.EmitIntValue(HT.Adjectives, 4); // Adjectives + OS.EmitValue(createImageRel32(HT.TypeDescriptor), 4); // Type + OS.EmitValue(FrameAllocOffsetRef, 4); // CatchObjOffset + OS.EmitValue(createImageRel32(HT.Handler), 4); // Handler + OS.EmitValue(ParentFrameOffsetRef, 4); // ParentFrameOffset + } + } + } + + // IPToStateMapEntry { + // void *IP; + // int32_t State; + // }; + if (IPToStateXData) { + OS.EmitLabel(IPToStateXData); + for (auto &IPStatePair : FuncInfo.IPToStateList) { + OS.EmitValue(createImageRel32(IPStatePair.first), 4); // IP + OS.EmitIntValue(IPStatePair.second, 4); // State + } + } +} diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.h b/lib/CodeGen/AsmPrinter/Win64Exception.h index b2d5d1b..5f4237f 100644 --- a/lib/CodeGen/AsmPrinter/Win64Exception.h +++ b/lib/CodeGen/AsmPrinter/Win64Exception.h @@ -17,7 +17,9 @@ #include "EHStreamer.h" namespace llvm { +class GlobalValue; class MachineFunction; +class MCExpr; class Win64Exception : public EHStreamer { /// Per-function flag to indicate if personality info should be emitted. @@ -31,14 +33,17 @@ class Win64Exception : public EHStreamer { void emitCSpecificHandlerTable(); - const MCSymbolRefExpr *createImageRel32(const MCSymbol *Value); + void emitCXXFrameHandler3Table(const MachineFunction *MF); + + const MCExpr *createImageRel32(const MCSymbol *Value); + const MCExpr *createImageRel32(const GlobalValue *GV); public: //===--------------------------------------------------------------------===// // Main entry points. // Win64Exception(AsmPrinter *A); - virtual ~Win64Exception(); + ~Win64Exception() override; /// Emit all exception information that should come after the content. void endModule() override; diff --git a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp index d2b4eec..276e7df 100644 --- a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp +++ b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp @@ -20,14 +20,13 @@ namespace llvm { StringRef WinCodeViewLineTables::getFullFilepath(const MDNode *S) { assert(S); - DIDescriptor D(S); - assert((D.isCompileUnit() || D.isFile() || D.isSubprogram() || - D.isLexicalBlockFile() || D.isLexicalBlock()) && + assert((isa(S) || isa(S) || isa(S) || + isa(S)) && "Unexpected scope info"); - DIScope Scope(S); - StringRef Dir = Scope.getDirectory(), - Filename = Scope.getFilename(); + auto *Scope = cast(S); + StringRef Dir = Scope->getDirectory(), + Filename = Scope->getFilename(); char *&Result = DirAndFilenameToFilepathMap[std::make_pair(Dir, Filename)]; if (Result) return Result; @@ -40,7 +39,7 @@ StringRef WinCodeViewLineTables::getFullFilepath(const MDNode *S) { if (Filename.find(':') == 1) Filepath = Filename; else - Filepath = (Dir + Twine("\\") + Filename).str(); + Filepath = (Dir + "\\" + Filename).str(); // Canonicalize the path. We have to do it textually because we may no longer // have access the file in the filesystem. @@ -81,7 +80,7 @@ StringRef WinCodeViewLineTables::getFullFilepath(const MDNode *S) { void WinCodeViewLineTables::maybeRecordLocation(DebugLoc DL, const MachineFunction *MF) { - const MDNode *Scope = DL.getScope(MF->getFunction()->getContext()); + const MDNode *Scope = DL.getScope(); if (!Scope) return; StringRef Filename = getFullFilepath(Scope); @@ -193,7 +192,7 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) { StringRef GVName = GV->getName(); StringRef FuncName; if (DISubprogram SP = getDISubprogram(GV)) - FuncName = SP.getDisplayName(); + FuncName = SP->getDisplayName(); // FIXME Clang currently sets DisplayName to "bar" for a C++ // "namespace_foo::bar" function, see PR21528. Luckily, dbghelp.dll is trying @@ -330,7 +329,7 @@ void WinCodeViewLineTables::beginFunction(const MachineFunction *MF) { DebugLoc PrologEndLoc; bool EmptyPrologue = true; for (const auto &MBB : *MF) { - if (!PrologEndLoc.isUnknown()) + if (PrologEndLoc) break; for (const auto &MI : MBB) { if (MI.isDebugValue()) @@ -339,8 +338,7 @@ void WinCodeViewLineTables::beginFunction(const MachineFunction *MF) { // First known non-DBG_VALUE and non-frame setup location marks // the beginning of the function body. // FIXME: do we need the first subcondition? - if (!MI.getFlag(MachineInstr::FrameSetup) && - (!MI.getDebugLoc().isUnknown())) { + if (!MI.getFlag(MachineInstr::FrameSetup) && MI.getDebugLoc()) { PrologEndLoc = MI.getDebugLoc(); break; } @@ -348,9 +346,8 @@ void WinCodeViewLineTables::beginFunction(const MachineFunction *MF) { } } // Record beginning of function if we have a non-empty prologue. - if (!PrologEndLoc.isUnknown() && !EmptyPrologue) { - DebugLoc FnStartDL = - PrologEndLoc.getFnDebugLoc(MF->getFunction()->getContext()); + if (PrologEndLoc && !EmptyPrologue) { + DebugLoc FnStartDL = PrologEndLoc.getFnDebugLoc(); maybeRecordLocation(FnStartDL, MF); } } @@ -377,7 +374,7 @@ void WinCodeViewLineTables::beginInstruction(const MachineInstr *MI) { if (!Asm || MI->isDebugValue() || MI->getFlag(MachineInstr::FrameSetup)) return; DebugLoc DL = MI->getDebugLoc(); - if (DL == PrevInstLoc || DL.isUnknown()) + if (DL == PrevInstLoc || !DL) return; maybeRecordLocation(DL, Asm->MF); } diff --git a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h index 8492eac..c66d141 100644 --- a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h +++ b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h @@ -114,7 +114,7 @@ class WinCodeViewLineTables : public AsmPrinterHandler { public: WinCodeViewLineTables(AsmPrinter *Asm); - ~WinCodeViewLineTables() { + ~WinCodeViewLineTables() override { for (DirAndFilenameToFilepathMapTy::iterator I = DirAndFilenameToFilepathMap.begin(), E = DirAndFilenameToFilepathMap.end(); diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp index 6c9d048..35376e1 100644 --- a/lib/CodeGen/CodeGenPrepare.cpp +++ b/lib/CodeGen/CodeGenPrepare.cpp @@ -693,11 +693,11 @@ static bool SinkCast(CastInst *CI) { InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0), CI->getType(), "", InsertPt); - MadeChange = true; } // Replace a use of the cast with a use of the new cast. TheUse = InsertedCast; + MadeChange = true; ++NumCastUses; } @@ -747,13 +747,60 @@ static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI){ return SinkCast(CI); } -/// OptimizeCmpExpression - sink the given CmpInst into user blocks to reduce +/// CombineUAddWithOverflow - try to combine CI into a call to the +/// llvm.uadd.with.overflow intrinsic if possible. +/// +/// Return true if any changes were made. +static bool CombineUAddWithOverflow(CmpInst *CI) { + Value *A, *B; + Instruction *AddI; + if (!match(CI, + m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI)))) + return false; + + Type *Ty = AddI->getType(); + if (!isa(Ty)) + return false; + + // We don't want to move around uses of condition values this late, so we we + // check if it is legal to create the call to the intrinsic in the basic + // block containing the icmp: + + if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse()) + return false; + +#ifndef NDEBUG + // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption + // for now: + if (AddI->hasOneUse()) + assert(*AddI->user_begin() == CI && "expected!"); +#endif + + Module *M = CI->getParent()->getParent()->getParent(); + Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty); + + auto *InsertPt = AddI->hasOneUse() ? CI : AddI; + + auto *UAddWithOverflow = + CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt); + auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt); + auto *Overflow = + ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt); + + CI->replaceAllUsesWith(Overflow); + AddI->replaceAllUsesWith(UAdd); + CI->eraseFromParent(); + AddI->eraseFromParent(); + return true; +} + +/// SinkCmpExpression - Sink the given CmpInst into user blocks to reduce /// the number of virtual registers that must be created and coalesced. This is /// a clear win except on targets with multiple condition code registers /// (PowerPC), where it might lose; some adjustment may be wanted there. /// /// Return true if any changes are made. -static bool OptimizeCmpExpression(CmpInst *CI) { +static bool SinkCmpExpression(CmpInst *CI) { BasicBlock *DefBB = CI->getParent(); /// InsertedCmp - Only insert a cmp in each block once. @@ -787,21 +834,33 @@ static bool OptimizeCmpExpression(CmpInst *CI) { CmpInst::Create(CI->getOpcode(), CI->getPredicate(), CI->getOperand(0), CI->getOperand(1), "", InsertPt); - MadeChange = true; } // Replace a use of the cmp with a use of the new cmp. TheUse = InsertedCmp; + MadeChange = true; ++NumCmpUses; } // If we removed all uses, nuke the cmp. - if (CI->use_empty()) + if (CI->use_empty()) { CI->eraseFromParent(); + MadeChange = true; + } return MadeChange; } +static bool OptimizeCmpExpression(CmpInst *CI) { + if (SinkCmpExpression(CI)) + return true; + + if (CombineUAddWithOverflow(CI)) + return true; + + return false; +} + /// isExtractBitsCandidateUse - Check if the candidates could /// be combined with shift instruction, which includes: /// 1. Truncate instruction @@ -1081,8 +1140,9 @@ static void ScalarizeMaskedLoad(CallInst *CI) { // CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load"); Builder.SetInsertPoint(InsertPt); - - Value* Gep = Builder.CreateInBoundsGEP(FirstEltPtr, Builder.getInt32(Idx)); + + Value *Gep = + Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); LoadInst* Load = Builder.CreateLoad(Gep, false); VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx)); @@ -1176,7 +1236,8 @@ static void ScalarizeMaskedStore(CallInst *CI) { Builder.SetInsertPoint(InsertPt); Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx)); - Value* Gep = Builder.CreateInBoundsGEP(FirstEltPtr, Builder.getInt32(Idx)); + Value *Gep = + Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); Builder.CreateStore(OneElt, Gep); // Create "else" block, fill it in the next iteration @@ -1227,13 +1288,25 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) { cast(Arg->getType())->getAddressSpace()), 0); Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*TD, Offset); uint64_t Offset2 = Offset.getLimitedValue(); + if ((Offset2 & (PrefAlign-1)) != 0) + continue; AllocaInst *AI; - if ((Offset2 & (PrefAlign-1)) == 0 && - (AI = dyn_cast(Val)) && + if ((AI = dyn_cast(Val)) && AI->getAlignment() < PrefAlign && TD->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2) AI->setAlignment(PrefAlign); - // TODO: Also align GlobalVariables + // Global variables can only be aligned if they are defined in this + // object (i.e. they are uniquely initialized in this object), and + // over-aligning global variables that have an explicit section is + // forbidden. + GlobalVariable *GV; + if ((GV = dyn_cast(Val)) && + GV->hasUniqueInitializer() && + !GV->hasSection() && + GV->getAlignment() < PrefAlign && + TD->getTypeAllocSize( + GV->getType()->getElementType()) >= MinSize + Offset2) + GV->setAlignment(PrefAlign); } // If this is a memcpy (or similar) then we may be able to improve the // alignment @@ -1841,7 +1914,7 @@ class TypePromotionTransaction { Inst->removeFromParent(); } - ~InstructionRemover() { delete Replacer; } + ~InstructionRemover() override { delete Replacer; } /// \brief Really remove the instruction. void commit() override { delete Inst; } @@ -3233,7 +3306,8 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr, return false; } else { Type *I8PtrTy = - Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace()); + Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace()); + Type *I8Ty = Builder.getInt8Ty(); // Start with the base register. Do this first so that subsequent address // matching finds it last, which will prevent it from trying to match it @@ -3285,7 +3359,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr, // SDAG consecutive load/store merging. if (ResultPtr->getType() != I8PtrTy) ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy); - ResultPtr = Builder.CreateGEP(ResultPtr, ResultIndex, "sunkaddr"); + ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); } ResultIndex = V; @@ -3296,7 +3370,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr, } else { if (ResultPtr->getType() != I8PtrTy) ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy); - SunkAddr = Builder.CreateGEP(ResultPtr, ResultIndex, "sunkaddr"); + SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); } if (SunkAddr->getType() != Addr->getType()) diff --git a/lib/CodeGen/CriticalAntiDepBreaker.h b/lib/CodeGen/CriticalAntiDepBreaker.h index ceef74d..af011a0 100644 --- a/lib/CodeGen/CriticalAntiDepBreaker.h +++ b/lib/CodeGen/CriticalAntiDepBreaker.h @@ -69,7 +69,7 @@ class TargetRegisterInfo; public: CriticalAntiDepBreaker(MachineFunction& MFi, const RegisterClassInfo&); - ~CriticalAntiDepBreaker(); + ~CriticalAntiDepBreaker() override; /// Initialize anti-dep breaking for a new basic block. void StartBlock(MachineBasicBlock *BB) override; diff --git a/lib/CodeGen/EarlyIfConversion.cpp b/lib/CodeGen/EarlyIfConversion.cpp index 8f74271..6cde4c2 100644 --- a/lib/CodeGen/EarlyIfConversion.cpp +++ b/lib/CodeGen/EarlyIfConversion.cpp @@ -797,9 +797,8 @@ bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) { // if-conversion in a single pass. The tryConvertIf() function may erase // blocks, but only blocks dominated by the head block. This makes it safe to // update the dominator tree while the post-order iterator is still active. - for (po_iterator - I = po_begin(DomTree), E = po_end(DomTree); I != E; ++I) - if (tryConvertIf(I->getBlock())) + for (auto DomNode : post_order(DomTree)) + if (tryConvertIf(DomNode->getBlock())) Changed = true; return Changed; diff --git a/lib/CodeGen/GCMetadata.cpp b/lib/CodeGen/GCMetadata.cpp index a2c5fce..16cd9e8 100644 --- a/lib/CodeGen/GCMetadata.cpp +++ b/lib/CodeGen/GCMetadata.cpp @@ -99,10 +99,6 @@ void Printer::getAnalysisUsage(AnalysisUsage &AU) const { static const char *DescKind(GC::PointKind Kind) { switch (Kind) { - case GC::Loop: - return "loop"; - case GC::Return: - return "return"; case GC::PreCall: return "pre-call"; case GC::PostCall: diff --git a/lib/CodeGen/GCRootLowering.cpp b/lib/CodeGen/GCRootLowering.cpp index 9d38e4c..ac35165 100644 --- a/lib/CodeGen/GCRootLowering.cpp +++ b/lib/CodeGen/GCRootLowering.cpp @@ -332,19 +332,22 @@ bool GCMachineCodeAnalysis::runOnMachineFunction(MachineFunction &MF) { return false; FI = &getAnalysis().getFunctionInfo(*MF.getFunction()); - if (!FI->getStrategy().needsSafePoints()) - return false; - MMI = &getAnalysis(); TII = MF.getSubtarget().getInstrInfo(); - // Find the size of the stack frame. - FI->setFrameSize(MF.getFrameInfo()->getStackSize()); + // Find the size of the stack frame. There may be no correct static frame + // size, we use UINT64_MAX to represent this. + const MachineFrameInfo *MFI = MF.getFrameInfo(); + const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); + const bool DynamicFrameSize = MFI->hasVarSizedObjects() || + RegInfo->needsStackRealignment(MF); + FI->setFrameSize(DynamicFrameSize ? UINT64_MAX : MFI->getStackSize()); // Find all safe points. - FindSafePoints(MF); + if (FI->getStrategy().needsSafePoints()) + FindSafePoints(MF); - // Find the stack offsets for all roots. + // Find the concrete stack offsets for all roots (stack slots) FindStackOffsets(MF); return false; diff --git a/lib/CodeGen/GlobalMerge.cpp b/lib/CodeGen/GlobalMerge.cpp index 4188e5d..153ba1a 100644 --- a/lib/CodeGen/GlobalMerge.cpp +++ b/lib/CodeGen/GlobalMerge.cpp @@ -73,9 +73,10 @@ using namespace llvm; #define DEBUG_TYPE "global-merge" +// FIXME: This is only useful as a last-resort way to disable the pass. cl::opt EnableGlobalMerge("enable-global-merge", cl::Hidden, - cl::desc("Enable global merge pass"), + cl::desc("Enable the global merge pass"), cl::init(true)); static cl::opt @@ -222,7 +223,8 @@ bool GlobalMerge::doMerge(SmallVectorImpl &Globals, ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, k-i) }; - Constant *GEP = ConstantExpr::getInBoundsGetElementPtr(MergedGV, Idx); + Constant *GEP = + ConstantExpr::getInBoundsGetElementPtr(MergedTy, MergedGV, Idx); Globals[k]->replaceAllUsesWith(GEP); Globals[k]->eraseFromParent(); diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index f0d407f..c7e7e58 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -135,7 +135,7 @@ private: // Dead defs generated during spilling. SmallVector DeadDefs; - ~InlineSpiller() {} + ~InlineSpiller() override {} public: InlineSpiller(MachineFunctionPass &pass, MachineFunction &mf, VirtRegMap &vrm) @@ -1232,6 +1232,8 @@ void InlineSpiller::spillAroundUses(unsigned Reg) { DebugLoc DL = MI->getDebugLoc(); DEBUG(dbgs() << "Modifying debug info due to spill:" << "\t" << *MI); MachineBasicBlock *MBB = MI->getParent(); + assert(cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); BuildMI(*MBB, MBB->erase(MI), DL, TII.get(TargetOpcode::DBG_VALUE)) .addFrameIndex(StackSlot) .addImm(Offset) diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 0fb0c46..61d68f6 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -140,12 +140,9 @@ static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM, return &MMI->getContext(); } -bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, - formatted_raw_ostream &Out, - CodeGenFileType FileType, - bool DisableVerify, - AnalysisID StartAfter, - AnalysisID StopAfter) { +bool LLVMTargetMachine::addPassesToEmitFile( + PassManagerBase &PM, raw_pwrite_stream &Out, CodeGenFileType FileType, + bool DisableVerify, AnalysisID StartAfter, AnalysisID StopAfter) { // Add common CodeGen passes. MCContext *Context = addPassesToGenerateCode(this, PM, DisableVerify, StartAfter, StopAfter); @@ -175,7 +172,7 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, switch (FileType) { case CGFT_AssemblyFile: { MCInstPrinter *InstPrinter = getTarget().createMCInstPrinter( - MAI.getAssemblerDialect(), MAI, MII, MRI, STI); + Triple(getTargetTriple()), MAI.getAssemblerDialect(), MAI, MII, MRI); // Create a code emitter if asked to show the encoding. MCCodeEmitter *MCE = nullptr; @@ -184,8 +181,9 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(), TargetCPU); + auto FOut = llvm::make_unique(Out); MCStreamer *S = getTarget().createAsmStreamer( - *Context, Out, Options.MCOptions.AsmVerbose, + *Context, std::move(FOut), Options.MCOptions.AsmVerbose, Options.MCOptions.MCUseDwarfDirectory, InstPrinter, MCE, MAB, Options.MCOptions.ShowMCInst); AsmStreamer.reset(S); @@ -229,9 +227,8 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, /// code is not supported. It fills the MCContext Ctx pointer which can be /// used to build custom MCStreamer. /// -bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, - MCContext *&Ctx, - raw_ostream &Out, +bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx, + raw_pwrite_stream &Out, bool DisableVerify) { // Add common CodeGen passes. Ctx = addPassesToGenerateCode(this, PM, DisableVerify, nullptr, nullptr); diff --git a/lib/CodeGen/LexicalScopes.cpp b/lib/CodeGen/LexicalScopes.cpp index 9eaf7da..d6998d6 100644 --- a/lib/CodeGen/LexicalScopes.cpp +++ b/lib/CodeGen/LexicalScopes.cpp @@ -59,11 +59,11 @@ void LexicalScopes::extractLexicalScopes( for (const auto &MBB : *MF) { const MachineInstr *RangeBeginMI = nullptr; const MachineInstr *PrevMI = nullptr; - DebugLoc PrevDL; + const MDLocation *PrevDL = nullptr; for (const auto &MInsn : MBB) { // Check if instruction has valid location information. - const DebugLoc MIDL = MInsn.getDebugLoc(); - if (MIDL.isUnknown()) { + const MDLocation *MIDL = MInsn.getDebugLoc(); + if (!MIDL) { PrevMI = &MInsn; continue; } @@ -96,7 +96,7 @@ void LexicalScopes::extractLexicalScopes( } // Create last instruction range. - if (RangeBeginMI && PrevMI && !PrevDL.isUnknown()) { + if (RangeBeginMI && PrevMI && PrevDL) { InsnRange R(RangeBeginMI, PrevMI); MIRanges.push_back(R); MI2ScopeMap[RangeBeginMI] = getOrCreateLexicalScope(PrevDL); @@ -106,20 +106,17 @@ void LexicalScopes::extractLexicalScopes( /// findLexicalScope - Find lexical scope, either regular or inlined, for the /// given DebugLoc. Return NULL if not found. -LexicalScope *LexicalScopes::findLexicalScope(DebugLoc DL) { - MDNode *Scope = nullptr; - MDNode *IA = nullptr; - DL.getScopeAndInlinedAt(Scope, IA, MF->getFunction()->getContext()); +LexicalScope *LexicalScopes::findLexicalScope(const MDLocation *DL) { + MDLocalScope *Scope = DL->getScope(); if (!Scope) return nullptr; // The scope that we were created with could have an extra file - which // isn't what we care about in this case. - DIDescriptor D = DIDescriptor(Scope); - if (D.isLexicalBlockFile()) - Scope = DILexicalBlockFile(Scope).getScope(); + if (auto *File = dyn_cast(Scope)) + Scope = File->getScope(); - if (IA) { + if (auto *IA = DL->getInlinedAt()) { auto I = InlinedLexicalScopeMap.find(std::make_pair(Scope, IA)); return I != InlinedLexicalScopeMap.end() ? &I->second : nullptr; } @@ -128,46 +125,39 @@ LexicalScope *LexicalScopes::findLexicalScope(DebugLoc DL) { /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If /// not available then create new lexical scope. -LexicalScope *LexicalScopes::getOrCreateLexicalScope(DebugLoc DL) { - if (DL.isUnknown()) - return nullptr; - MDNode *Scope = nullptr; - MDNode *InlinedAt = nullptr; - DL.getScopeAndInlinedAt(Scope, InlinedAt, MF->getFunction()->getContext()); - - if (InlinedAt) { +LexicalScope *LexicalScopes::getOrCreateLexicalScope(const MDLocalScope *Scope, + const MDLocation *IA) { + if (IA) { // Create an abstract scope for inlined function. getOrCreateAbstractScope(Scope); // Create an inlined scope for inlined function. - return getOrCreateInlinedScope(Scope, InlinedAt); + return getOrCreateInlinedScope(Scope, IA); } return getOrCreateRegularScope(Scope); } /// getOrCreateRegularScope - Find or create a regular lexical scope. -LexicalScope *LexicalScopes::getOrCreateRegularScope(MDNode *Scope) { - DIDescriptor D = DIDescriptor(Scope); - if (D.isLexicalBlockFile()) { - Scope = DILexicalBlockFile(Scope).getScope(); - D = DIDescriptor(Scope); - } +LexicalScope * +LexicalScopes::getOrCreateRegularScope(const MDLocalScope *Scope) { + if (auto *File = dyn_cast(Scope)) + Scope = File->getScope(); auto I = LexicalScopeMap.find(Scope); if (I != LexicalScopeMap.end()) return &I->second; + // FIXME: Should the following dyn_cast be MDLexicalBlock? LexicalScope *Parent = nullptr; - if (D.isLexicalBlock()) - Parent = getOrCreateLexicalScope(DebugLoc::getFromDILexicalBlock(Scope)); + if (auto *Block = dyn_cast(Scope)) + Parent = getOrCreateLexicalScope(Block->getScope()); I = LexicalScopeMap.emplace(std::piecewise_construct, std::forward_as_tuple(Scope), - std::forward_as_tuple(Parent, DIDescriptor(Scope), - nullptr, false)).first; + std::forward_as_tuple(Parent, Scope, nullptr, + false)).first; if (!Parent) { - assert(DIDescriptor(Scope).isSubprogram()); - assert(DISubprogram(Scope).describes(MF->getFunction())); + assert(cast(Scope)->describes(MF->getFunction())); assert(!CurrentFnLexicalScope); CurrentFnLexicalScope = &I->second; } @@ -176,19 +166,19 @@ LexicalScope *LexicalScopes::getOrCreateRegularScope(MDNode *Scope) { } /// getOrCreateInlinedScope - Find or create an inlined lexical scope. -LexicalScope *LexicalScopes::getOrCreateInlinedScope(MDNode *ScopeNode, - MDNode *InlinedAt) { - std::pair P(ScopeNode, InlinedAt); +LexicalScope * +LexicalScopes::getOrCreateInlinedScope(const MDLocalScope *Scope, + const MDLocation *InlinedAt) { + std::pair P(Scope, InlinedAt); auto I = InlinedLexicalScopeMap.find(P); if (I != InlinedLexicalScopeMap.end()) return &I->second; LexicalScope *Parent; - DILexicalBlock Scope(ScopeNode); - if (Scope.isSubprogram()) - Parent = getOrCreateLexicalScope(DebugLoc::getFromDILocation(InlinedAt)); + if (auto *Block = dyn_cast(Scope)) + Parent = getOrCreateInlinedScope(Block->getScope(), InlinedAt); else - Parent = getOrCreateInlinedScope(Scope.getContext(), InlinedAt); + Parent = getOrCreateLexicalScope(InlinedAt); I = InlinedLexicalScopeMap.emplace(std::piecewise_construct, std::forward_as_tuple(P), @@ -199,27 +189,26 @@ LexicalScope *LexicalScopes::getOrCreateInlinedScope(MDNode *ScopeNode, } /// getOrCreateAbstractScope - Find or create an abstract lexical scope. -LexicalScope *LexicalScopes::getOrCreateAbstractScope(const MDNode *N) { - assert(N && "Invalid Scope encoding!"); +LexicalScope * +LexicalScopes::getOrCreateAbstractScope(const MDLocalScope *Scope) { + assert(Scope && "Invalid Scope encoding!"); - DIDescriptor Scope(N); - if (Scope.isLexicalBlockFile()) - Scope = DILexicalBlockFile(Scope).getScope(); + if (auto *File = dyn_cast(Scope)) + Scope = File->getScope(); auto I = AbstractScopeMap.find(Scope); if (I != AbstractScopeMap.end()) return &I->second; + // FIXME: Should the following isa be MDLexicalBlock? LexicalScope *Parent = nullptr; - if (Scope.isLexicalBlock()) { - DILexicalBlock DB(Scope); - DIDescriptor ParentDesc = DB.getContext(); - Parent = getOrCreateAbstractScope(ParentDesc); - } + if (auto *Block = dyn_cast(Scope)) + Parent = getOrCreateAbstractScope(Block->getScope()); + I = AbstractScopeMap.emplace(std::piecewise_construct, std::forward_as_tuple(Scope), std::forward_as_tuple(Parent, Scope, nullptr, true)).first; - if (Scope.isSubprogram()) + if (isa(Scope)) AbstractScopesList.push_back(&I->second); return &I->second; } @@ -280,7 +269,7 @@ void LexicalScopes::assignInstructionRanges( /// have machine instructions that belong to lexical scope identified by /// DebugLoc. void LexicalScopes::getMachineBasicBlocks( - DebugLoc DL, SmallPtrSetImpl &MBBs) { + const MDLocation *DL, SmallPtrSetImpl &MBBs) { MBBs.clear(); LexicalScope *Scope = getOrCreateLexicalScope(DL); if (!Scope) @@ -303,7 +292,7 @@ void LexicalScopes::getMachineBasicBlocks( /// dominates - Return true if DebugLoc's lexical scope dominates at least one /// machine instruction's lexical scope in a given machine basic block. -bool LexicalScopes::dominates(DebugLoc DL, MachineBasicBlock *MBB) { +bool LexicalScopes::dominates(const MDLocation *DL, MachineBasicBlock *MBB) { LexicalScope *Scope = getOrCreateLexicalScope(DL); if (!Scope) return false; @@ -315,12 +304,10 @@ bool LexicalScopes::dominates(DebugLoc DL, MachineBasicBlock *MBB) { bool Result = false; for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E; ++I) { - DebugLoc IDL = I->getDebugLoc(); - if (IDL.isUnknown()) - continue; - if (LexicalScope *IScope = getOrCreateLexicalScope(IDL)) - if (Scope->dominates(IScope)) - return true; + if (const MDLocation *IDL = I->getDebugLoc()) + if (LexicalScope *IScope = getOrCreateLexicalScope(IDL)) + if (Scope->dominates(IScope)) + return true; } return Result; } diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp index e3791be..c2993db 100644 --- a/lib/CodeGen/LiveDebugVariables.cpp +++ b/lib/CodeGen/LiveDebugVariables.cpp @@ -158,10 +158,10 @@ public: UserValue *getNext() const { return next; } /// match - Does this UserValue match the parameters? - bool match(const MDNode *Var, const MDNode *Expr, unsigned Offset, - bool indirect) const { - return Var == Variable && Expr == Expression && Offset == offset && - indirect == IsIndirect; + bool match(const MDNode *Var, const MDNode *Expr, const MDLocation *IA, + unsigned Offset, bool indirect) const { + return Var == Variable && Expr == Expression && dl->getInlinedAt() == IA && + Offset == offset && indirect == IsIndirect; } /// merge - Merge equivalence classes. @@ -269,12 +269,6 @@ public: void emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS, const TargetInstrInfo &TRI); - /// findDebugLoc - Return DebugLoc used for this DBG_VALUE instruction. A - /// variable may have more than one corresponding DBG_VALUE instructions. - /// Only first one needs DebugLoc to identify variable's lexical scope - /// in source file. - DebugLoc findDebugLoc(); - /// getDebugLoc - Return DebugLoc of this UserValue. DebugLoc getDebugLoc() { return dl;} void print(raw_ostream &, const TargetRegisterInfo *); @@ -363,10 +357,47 @@ public: }; } // namespace +static void printDebugLoc(DebugLoc DL, raw_ostream &CommentOS, + const LLVMContext &Ctx) { + if (!DL) + return; + + auto *Scope = cast(DL.getScope()); + // Omit the directory, because it's likely to be long and uninteresting. + CommentOS << Scope->getFilename(); + CommentOS << ':' << DL.getLine(); + if (DL.getCol() != 0) + CommentOS << ':' << DL.getCol(); + + DebugLoc InlinedAtDL = DL.getInlinedAt(); + if (!InlinedAtDL) + return; + + CommentOS << " @[ "; + printDebugLoc(InlinedAtDL, CommentOS, Ctx); + CommentOS << " ]"; +} + +static void printExtendedName(raw_ostream &OS, const MDLocalVariable *V, + const MDLocation *DL) { + const LLVMContext &Ctx = V->getContext(); + StringRef Res = V->getName(); + if (!Res.empty()) + OS << Res << "," << V->getLine(); + if (auto *InlinedAt = DL->getInlinedAt()) { + if (DebugLoc InlinedAtDL = InlinedAt) { + OS << " @["; + printDebugLoc(InlinedAtDL, OS, Ctx); + OS << "]"; + } + } +} + void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) { - DIVariable DV(Variable); + DIVariable DV = cast(Variable); OS << "!\""; - DV.printExtendedName(OS); + printExtendedName(OS, DV, dl); + OS << "\"\t"; if (offset) OS << '+' << offset; @@ -433,7 +464,7 @@ UserValue *LDVImpl::getUserValue(const MDNode *Var, const MDNode *Expr, UserValue *UV = Leader->getLeader(); Leader = UV; for (; UV; UV = UV->getNext()) - if (UV->match(Var, Expr, Offset, IsIndirect)) + if (UV->match(Var, Expr, DL->getInlinedAt(), Offset, IsIndirect)) return UV; } @@ -942,11 +973,6 @@ findInsertLocation(MachineBasicBlock *MBB, SlotIndex Idx, std::next(MachineBasicBlock::iterator(MI)); } -DebugLoc UserValue::findDebugLoc() { - DebugLoc D = dl; - dl = DebugLoc(); - return D; -} void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex Idx, unsigned LocNo, LiveIntervals &LIS, @@ -955,11 +981,14 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex Idx, MachineOperand &Loc = locations[LocNo]; ++NumInsertedDebugValues; + assert(cast(Variable) + ->isValidLocationForIntrinsic(getDebugLoc()) && + "Expected inlined-at fields to agree"); if (Loc.isReg()) - BuildMI(*MBB, I, findDebugLoc(), TII.get(TargetOpcode::DBG_VALUE), + BuildMI(*MBB, I, getDebugLoc(), TII.get(TargetOpcode::DBG_VALUE), IsIndirect, Loc.getReg(), offset, Variable, Expression); else - BuildMI(*MBB, I, findDebugLoc(), TII.get(TargetOpcode::DBG_VALUE)) + BuildMI(*MBB, I, getDebugLoc(), TII.get(TargetOpcode::DBG_VALUE)) .addOperand(Loc) .addImm(offset) .addMetadata(Variable) diff --git a/lib/CodeGen/LiveDebugVariables.h b/lib/CodeGen/LiveDebugVariables.h index 9748329..fe296bc 100644 --- a/lib/CodeGen/LiveDebugVariables.h +++ b/lib/CodeGen/LiveDebugVariables.h @@ -38,7 +38,7 @@ public: static char ID; // Pass identification, replacement for typeid LiveDebugVariables(); - ~LiveDebugVariables(); + ~LiveDebugVariables() override; /// renameRegister - Move any user variables in OldReg to NewReg:SubIdx. /// @param OldReg Old virtual register that is going away. diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp index 2afd7fa..d75e441 100644 --- a/lib/CodeGen/LiveInterval.cpp +++ b/lib/CodeGen/LiveInterval.cpp @@ -816,23 +816,45 @@ static VNInfo *searchForVNI(const SlotIndexes &Indexes, LiveRange &LR, static void determineMissingVNIs(const SlotIndexes &Indexes, LiveInterval &LI) { SmallPtrSet Visited; - for (LiveRange::Segment &S : LI.segments) { - if (S.valno != nullptr) - continue; - // This can only happen at the begin of a basic block. - assert(S.start.isBlock() && "valno should only be missing at block begin"); - - Visited.clear(); - const MachineBasicBlock *MBB = Indexes.getMBBFromIndex(S.start); - for (const MachineBasicBlock *Pred : MBB->predecessors()) { - VNInfo *VNI = searchForVNI(Indexes, LI, Pred, Visited); - if (VNI != nullptr) { - S.valno = VNI; - break; + + LiveRange::iterator OutIt; + VNInfo *PrevValNo = nullptr; + for (LiveRange::iterator I = LI.begin(), E = LI.end(); I != E; ++I) { + LiveRange::Segment &S = *I; + // Determine final VNI if necessary. + if (S.valno == nullptr) { + // This can only happen at the begin of a basic block. + assert(S.start.isBlock() && "valno should only be missing at block begin"); + + Visited.clear(); + const MachineBasicBlock *MBB = Indexes.getMBBFromIndex(S.start); + for (const MachineBasicBlock *Pred : MBB->predecessors()) { + VNInfo *VNI = searchForVNI(Indexes, LI, Pred, Visited); + if (VNI != nullptr) { + S.valno = VNI; + break; + } } + assert(S.valno != nullptr && "could not determine valno"); + } + // Merge with previous segment if it has the same VNI. + if (PrevValNo == S.valno && OutIt->end == S.start) { + OutIt->end = S.end; + } else { + // Didn't merge. Move OutIt to next segment. + if (PrevValNo == nullptr) + OutIt = LI.begin(); + else + ++OutIt; + + if (OutIt != I) + *OutIt = *I; + PrevValNo = S.valno; } - assert(S.valno != nullptr && "could not determine valno"); } + // If we merged some segments chop off the end. + ++OutIt; + LI.segments.erase(OutIt, LI.end()); } void LiveInterval::constructMainRangeFromSubranges( @@ -955,6 +977,12 @@ void LiveInterval::constructMainRangeFromSubranges( NeedVNIFixup = true; } + // In rare cases we can produce adjacent segments with the same value + // number (if they come from different subranges, but happen to have + // the same defining instruction). VNIFixup will fix those cases. + if (!empty() && segments.back().end == Pos && + segments.back().valno == VNI) + NeedVNIFixup = true; CurrentSegment.start = Pos; CurrentSegment.valno = VNI; ConstructingSegment = true; diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index 98359b1..34131bb 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -250,7 +250,7 @@ std::string MachineBasicBlock::getFullName() const { if (getBasicBlock()) Name += getBasicBlock()->getName(); else - Name += (Twine("BB") + Twine(getNumber())).str(); + Name += ("BB" + Twine(getNumber())).str(); return Name; } diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp index ecc50c9..2969bad 100644 --- a/lib/CodeGen/MachineBlockPlacement.cpp +++ b/lib/CodeGen/MachineBlockPlacement.cpp @@ -661,7 +661,7 @@ MachineBlockPlacement::findBestLoopExit(MachineFunction &F, MachineLoop &L, for (MachineBasicBlock *MBB : L.getBlocks()) { BlockChain &Chain = *BlockToChain[MBB]; // Ensure that this block is at the end of a chain; otherwise it could be - // mid-way through an inner loop or a successor of an analyzable branch. + // mid-way through an inner loop or a successor of an unanalyzable branch. if (MBB != *std::prev(Chain.end())) continue; @@ -715,7 +715,7 @@ MachineBlockPlacement::findBestLoopExit(MachineFunction &F, MachineLoop &L, // a frequency higher than the current exit before we consider breaking // the layout. BranchProbability Bias(100 - ExitBlockBias, 100); - if (!ExitingBB || BestExitLoopDepth < SuccLoopDepth || + if (!ExitingBB || SuccLoopDepth > BestExitLoopDepth || ExitEdgeFreq > BestExitEdgeFreq || (MBB->isLayoutSuccessor(Succ) && !(ExitEdgeFreq < BestExitEdgeFreq * Bias))) { @@ -724,8 +724,8 @@ MachineBlockPlacement::findBestLoopExit(MachineFunction &F, MachineLoop &L, } } - // Restore the old exiting state, no viable looping successor was found. if (!HasLoopingSucc) { + // Restore the old exiting state, no viable looping successor was found. ExitingBB = OldExitingBB; BestExitEdgeFreq = OldBestExitEdgeFreq; continue; diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index 6ceace8..448531f 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -380,7 +380,7 @@ namespace llvm { DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {} static std::string getGraphName(const MachineFunction *F) { - return "CFG for '" + F->getName().str() + "' function"; + return ("CFG for '" + F->getName() + "' function").str(); } std::string getNodeLabel(const MachineBasicBlock *Node, @@ -468,7 +468,7 @@ MCSymbol *MachineFunction::getJTISymbol(unsigned JTI, MCContext &Ctx, SmallString<60> Name; raw_svector_ostream(Name) << Prefix << "JTI" << getFunctionNumber() << '_' << JTI; - return Ctx.GetOrCreateSymbol(Name.str()); + return Ctx.GetOrCreateSymbol(Name); } /// getPICBaseSymbol - Return a function-local symbol to represent the PIC diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 1240efb..d154110 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -881,8 +881,8 @@ bool MachineInstr::isIdenticalTo(const MachineInstr *Other, } // If DebugLoc does not match then two dbg.values are not identical. if (isDebugValue()) - if (!getDebugLoc().isUnknown() && !Other->getDebugLoc().isUnknown() - && getDebugLoc() != Other->getDebugLoc()) + if (getDebugLoc() && Other->getDebugLoc() && + getDebugLoc() != Other->getDebugLoc()) return false; return true; } @@ -1619,12 +1619,9 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const { } if (isDebugValue() && MO.isMetadata()) { // Pretty print DBG_VALUE instructions. - const MDNode *MD = MO.getMetadata(); - DIDescriptor DI(MD); - DIVariable DIV(MD); - - if (DI.isVariable() && !DIV.getName().empty()) - OS << "!\"" << DIV.getName() << '\"'; + DIVariable DIV = dyn_cast(MO.getMetadata()); + if (DIV && !DIV->getName().empty()) + OS << "!\"" << DIV->getName() << '\"'; else MO.print(OS, TRI); } else if (TRI && (isInsertSubreg() || isRegSequence()) && MO.isImm()) { @@ -1711,13 +1708,13 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const { } // Print debug location information. - if (isDebugValue() && getOperand(e - 1).isMetadata()) { + if (isDebugValue() && getOperand(e - 2).isMetadata()) { if (!HaveSemi) OS << ";"; - DIVariable DV(getOperand(e - 1).getMetadata()); - OS << " line no:" << DV.getLineNumber(); - if (MDNode *InlinedAt = DV.getInlinedAt()) { - DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(InlinedAt); - if (!InlinedAtDL.isUnknown() && MF) { + DIVariable DV = cast(getOperand(e - 2).getMetadata()); + OS << " line no:" << DV->getLine(); + if (auto *InlinedAt = debugLoc->getInlinedAt()) { + DebugLoc InlinedAtDL(InlinedAt); + if (InlinedAtDL && MF) { OS << " inlined @[ "; InlinedAtDL.print(OS); OS << " ]"; @@ -1725,7 +1722,7 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const { } if (isIndirectDebugValue()) OS << " indirect"; - } else if (!debugLoc.isUnknown() && MF) { + } else if (debugLoc && MF) { if (!HaveSemi) OS << ";"; OS << " dbg:"; debugLoc.print(OS); diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 2f65a2e..9756f13 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -10,10 +10,6 @@ // This pass performs loop invariant code motion on machine instructions. We // attempt to remove as much code from the body of a loop as possible. // -// This pass does not attempt to throttle itself to limit register pressure. -// The register allocation phases are expected to perform rematerialization -// to recover when register pressure is high. -// // This pass is not intended to be a replacement or a complete alternative // for the LLVM-IR-level LICM pass. It is only designed to hoist simple // constructs that are not exposed before lowering and instruction selection. @@ -104,7 +100,7 @@ namespace { SmallSet RegSeen; SmallVector RegPressure; - // Register pressure "limit" per register class. If the pressure + // Register pressure "limit" per register pressure set. If the pressure // is higher than the limit, then it's considered high. SmallVector RegLimit; @@ -214,7 +210,8 @@ namespace { /// CanCauseHighRegPressure - Visit BBs from header to current BB, /// check if hoisting an instruction of the given cost matrix can cause high /// register pressure. - bool CanCauseHighRegPressure(DenseMap &Cost, bool Cheap); + bool CanCauseHighRegPressure(const DenseMap &Cost, + bool Cheap); /// UpdateBackTraceRegPressure - Traverse the back trace from header to /// the current block and update their register pressures to reflect the @@ -254,21 +251,25 @@ namespace { /// if there is little to no overhead moving instructions into loops. void SinkIntoLoop(); - /// getRegisterClassIDAndCost - For a given MI, register, and the operand - /// index, return the ID and cost of its representative register class by - /// reference. - void getRegisterClassIDAndCost(const MachineInstr *MI, - unsigned Reg, unsigned OpIdx, - unsigned &RCId, unsigned &RCCost) const; - /// InitRegPressure - Find all virtual register references that are liveout /// of the preheader to initialize the starting "register pressure". Note /// this does not count live through (livein but not used) registers. void InitRegPressure(MachineBasicBlock *BB); + /// calcRegisterCost - Calculate the additional register pressure that the + /// registers used in MI cause. + /// + /// If 'ConsiderSeen' is true, updates 'RegSeen' and uses the information to + /// figure out which usages are live-ins. + /// FIXME: Figure out a way to consider 'RegSeen' from all code paths. + DenseMap calcRegisterCost(const MachineInstr *MI, + bool ConsiderSeen, + bool ConsiderUnseenAsDef); + /// UpdateRegPressure - Update estimate of register pressure after the /// specified instruction. - void UpdateRegPressure(const MachineInstr *MI); + void UpdateRegPressure(const MachineInstr *MI, + bool ConsiderUnseenAsDef = false); /// ExtractHoistableLoad - Unfold a load from the given machineinstr if /// the load itself could be hoisted. Return the unfolded and hoistable @@ -354,13 +355,12 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { if (PreRegAlloc) { // Estimate register pressure during pre-regalloc pass. - unsigned NumRC = TRI->getNumRegClasses(); - RegPressure.resize(NumRC); + unsigned NumRPS = TRI->getNumRegPressureSets(); + RegPressure.resize(NumRPS); std::fill(RegPressure.begin(), RegPressure.end(), 0); - RegLimit.resize(NumRC); - for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), - E = TRI->regclass_end(); I != E; ++I) - RegLimit[(*I)->getID()] = TRI->getRegPressureLimit(*I, MF); + RegLimit.resize(NumRPS); + for (unsigned i = 0, e = NumRPS; i != e; ++i) + RegLimit[i] = TRI->getRegPressureSetLimit(MF, i); } // Get our Loop information... @@ -836,23 +836,6 @@ static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) { return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg()); } -/// getRegisterClassIDAndCost - For a given MI, register, and the operand -/// index, return the ID and cost of its representative register class. -void -MachineLICM::getRegisterClassIDAndCost(const MachineInstr *MI, - unsigned Reg, unsigned OpIdx, - unsigned &RCId, unsigned &RCCost) const { - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - MVT VT = *RC->vt_begin(); - if (VT == MVT::Untyped) { - RCId = RC->getID(); - RCCost = 1; - } else { - RCId = TLI->getRepRegClassFor(VT)->getID(); - RCCost = TLI->getRepRegClassCostFor(VT); - } -} - /// InitRegPressure - Find all virtual register references that are liveout of /// the preheader to initialize the starting "register pressure". Note this /// does not count live through (livein but not used) registers. @@ -870,41 +853,30 @@ void MachineLICM::InitRegPressure(MachineBasicBlock *BB) { InitRegPressure(*BB->pred_begin()); } - for (MachineBasicBlock::iterator MII = BB->begin(), E = BB->end(); - MII != E; ++MII) { - MachineInstr *MI = &*MII; - for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || MO.isImplicit()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - bool isNew = RegSeen.insert(Reg).second; - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (MO.isDef()) - RegPressure[RCId] += RCCost; - else { - bool isKill = isOperandKill(MO, MRI); - if (isNew && !isKill) - // Haven't seen this, it must be a livein. - RegPressure[RCId] += RCCost; - else if (!isNew && isKill) - RegPressure[RCId] -= RCCost; - } - } - } + for (const MachineInstr &MI : *BB) + UpdateRegPressure(&MI, /*ConsiderUnseenAsDef=*/true); } /// UpdateRegPressure - Update estimate of register pressure after the /// specified instruction. -void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { - if (MI->isImplicitDef()) - return; +void MachineLICM::UpdateRegPressure(const MachineInstr *MI, + bool ConsiderUnseenAsDef) { + auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/true, ConsiderUnseenAsDef); + for (const auto &RPIdAndCost : Cost) { + unsigned Class = RPIdAndCost.first; + if (static_cast(RegPressure[Class]) < -RPIdAndCost.second) + RegPressure[Class] = 0; + else + RegPressure[Class] += RPIdAndCost.second; + } +} - SmallVector Defs; +DenseMap +MachineLICM::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen, + bool ConsiderUnseenAsDef) { + DenseMap Cost; + if (MI->isImplicitDef()) + return Cost; for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI->getOperand(i); if (!MO.isReg() || MO.isImplicit()) @@ -913,27 +885,33 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - bool isNew = RegSeen.insert(Reg).second; + // FIXME: It seems bad to use RegSeen only for some of these calculations. + bool isNew = ConsiderSeen ? RegSeen.insert(Reg).second : false; + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + + RegClassWeight W = TRI->getRegClassWeight(RC); + int RCCost = 0; if (MO.isDef()) - Defs.push_back(Reg); - else if (!isNew && isOperandKill(MO, MRI)) { - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (RCCost > RegPressure[RCId]) - RegPressure[RCId] = 0; + RCCost = W.RegWeight; + else { + bool isKill = isOperandKill(MO, MRI); + if (isNew && !isKill && ConsiderUnseenAsDef) + // Haven't seen this, it must be a livein. + RCCost = W.RegWeight; + else if (!isNew && isKill) + RCCost = -W.RegWeight; + } + if (RCCost == 0) + continue; + const int *PS = TRI->getRegClassPressureSets(RC); + for (; *PS != -1; ++PS) { + if (Cost.find(*PS) == Cost.end()) + Cost[*PS] = RCCost; else - RegPressure[RCId] -= RCCost; + Cost[*PS] += RCCost; } } - - unsigned Idx = 0; - while (!Defs.empty()) { - unsigned Reg = Defs.pop_back_val(); - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, Idx, RCId, RCCost); - RegPressure[RCId] += RCCost; - ++Idx; - } + return Cost; } /// isLoadFromGOTOrConstantPool - Return true if this machine instruction @@ -1125,27 +1103,23 @@ bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const { /// CanCauseHighRegPressure - Visit BBs from header to current BB, check /// if hoisting an instruction of the given cost matrix can cause high /// register pressure. -bool MachineLICM::CanCauseHighRegPressure(DenseMap &Cost, +bool MachineLICM::CanCauseHighRegPressure(const DenseMap& Cost, bool CheapInstr) { - for (DenseMap::iterator CI = Cost.begin(), CE = Cost.end(); - CI != CE; ++CI) { - if (CI->second <= 0) + for (const auto &RPIdAndCost : Cost) { + if (RPIdAndCost.second <= 0) continue; - unsigned RCId = CI->first; - unsigned Limit = RegLimit[RCId]; - int Cost = CI->second; + unsigned Class = RPIdAndCost.first; + int Limit = RegLimit[Class]; // Don't hoist cheap instructions if they would increase register pressure, // even if we're under the limit. if (CheapInstr && !HoistCheapInsts) return true; - for (unsigned i = BackTrace.size(); i != 0; --i) { - SmallVectorImpl &RP = BackTrace[i-1]; - if (RP[RCId] + Cost >= Limit) + for (const auto &RP : BackTrace) + if (static_cast(RP[Class]) + RPIdAndCost.second >= Limit) return true; - } } return false; @@ -1155,46 +1129,15 @@ bool MachineLICM::CanCauseHighRegPressure(DenseMap &Cost, /// current block and update their register pressures to reflect the effect /// of hoisting MI from the current block to the preheader. void MachineLICM::UpdateBackTraceRegPressure(const MachineInstr *MI) { - if (MI->isImplicitDef()) - return; - // First compute the 'cost' of the instruction, i.e. its contribution // to register pressure. - DenseMap Cost; - for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || MO.isImplicit()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (MO.isDef()) { - DenseMap::iterator CI = Cost.find(RCId); - if (CI != Cost.end()) - CI->second += RCCost; - else - Cost.insert(std::make_pair(RCId, RCCost)); - } else if (isOperandKill(MO, MRI)) { - DenseMap::iterator CI = Cost.find(RCId); - if (CI != Cost.end()) - CI->second -= RCCost; - else - Cost.insert(std::make_pair(RCId, -RCCost)); - } - } + auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/false, + /*ConsiderUnseenAsDef=*/false); // Update register pressure of blocks from loop header to current block. - for (unsigned i = 0, e = BackTrace.size(); i != e; ++i) { - SmallVectorImpl &RP = BackTrace[i]; - for (DenseMap::iterator CI = Cost.begin(), CE = Cost.end(); - CI != CE; ++CI) { - unsigned RCId = CI->first; - RP[RCId] += CI->second; - } - } + for (auto &RP : BackTrace) + for (const auto &RPIdAndCost : Cost) + RP[RPIdAndCost.first] += RPIdAndCost.second; } /// IsProfitableToHoist - Return true if it is potentially profitable to hoist @@ -1229,15 +1172,8 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { if (TII->isTriviallyReMaterializable(&MI, AA)) return true; - // Estimate register pressure to determine whether to LICM the instruction. - // In low register pressure situation, we can be more aggressive about - // hoisting. Also, favors hoisting long latency instructions even in - // moderately high pressure situation. - // Cheap instructions will only be hoisted if they don't increase register - // pressure at all. // FIXME: If there are long latency loop-invariant instructions inside the // loop at this point, why didn't the optimizer's LICM hoist them? - DenseMap Cost; for (unsigned i = 0, e = MI.getDesc().getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || MO.isImplicit()) @@ -1245,24 +1181,22 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - - unsigned RCId, RCCost; - getRegisterClassIDAndCost(&MI, Reg, i, RCId, RCCost); - if (MO.isDef()) { - if (HasHighOperandLatency(MI, i, Reg)) { - DEBUG(dbgs() << "Hoist High Latency: " << MI); - ++NumHighLatency; - return true; - } - Cost[RCId] += RCCost; - } else if (isOperandKill(MO, MRI)) { - // Is a virtual register use is a kill, hoisting it out of the loop - // may actually reduce register pressure or be register pressure - // neutral. - Cost[RCId] -= RCCost; + if (MO.isDef() && HasHighOperandLatency(MI, i, Reg)) { + DEBUG(dbgs() << "Hoist High Latency: " << MI); + ++NumHighLatency; + return true; } } + // Estimate register pressure to determine whether to LICM the instruction. + // In low register pressure situation, we can be more aggressive about + // hoisting. Also, favors hoisting long latency instructions even in + // moderately high pressure situation. + // Cheap instructions will only be hoisted if they don't increase register + // pressure at all. + auto Cost = calcRegisterCost(&MI, /*ConsiderSeen=*/false, + /*ConsiderUnseenAsDef=*/false); + // Visit BBs from header to current BB, if hoisting this doesn't cause // high register pressure, then it's safe to proceed. if (!CanCauseHighRegPressure(Cost, CheapInstr)) { diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp index fca7df0..e8bd1f8 100644 --- a/lib/CodeGen/MachineModuleInfo.cpp +++ b/lib/CodeGen/MachineModuleInfo.cpp @@ -14,6 +14,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalVariable.h" @@ -425,6 +426,12 @@ void MachineModuleInfo::addPersonality(MachineBasicBlock *LandingPad, Personalities.push_back(Personality); } +void MachineModuleInfo::addWinEHState(MachineBasicBlock *LandingPad, + int State) { + LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad); + LP.WinEHState = State; +} + /// addCatchTypeInfo - Provide the catch typeinfo for a landing pad. /// void MachineModuleInfo:: @@ -563,10 +570,13 @@ const Function *MachineModuleInfo::getPersonality() const { } EHPersonality MachineModuleInfo::getPersonalityType() { - if (PersonalityTypeCache == EHPersonality::Unknown) - PersonalityTypeCache = classifyEHPersonality(getPersonality()); + if (PersonalityTypeCache == EHPersonality::Unknown) { + if (const Function *F = getPersonality()) + PersonalityTypeCache = classifyEHPersonality(F); + } return PersonalityTypeCache; } + /// getPersonalityIndex - Return unique index for current personality /// function. NULL/first personality function should always get zero index. unsigned MachineModuleInfo::getPersonalityIndex() const { @@ -588,3 +598,18 @@ unsigned MachineModuleInfo::getPersonalityIndex() const { // in the zero index. return 0; } + +const Function *MachineModuleInfo::getWinEHParent(const Function *F) const { + StringRef WinEHParentName = + F->getFnAttribute("wineh-parent").getValueAsString(); + if (WinEHParentName.empty() || WinEHParentName == F->getName()) + return F; + return F->getParent()->getFunction(WinEHParentName); +} + +WinEHFuncInfo &MachineModuleInfo::getWinEHFuncInfo(const Function *F) { + auto &Ptr = FuncInfoMap[getWinEHParent(F)]; + if (!Ptr) + Ptr.reset(new WinEHFuncInfo); + return *Ptr; +} diff --git a/lib/CodeGen/MachineModuleInfoImpls.cpp b/lib/CodeGen/MachineModuleInfoImpls.cpp index a1c7e9f..22d519e 100644 --- a/lib/CodeGen/MachineModuleInfoImpls.cpp +++ b/lib/CodeGen/MachineModuleInfoImpls.cpp @@ -31,15 +31,14 @@ static int SortSymbolPair(const void *LHS, const void *RHS) { return LHSS->getName().compare(RHSS->getName()); } -/// GetSortedStubs - Return the entries from a DenseMap in a deterministic -/// sorted orer. -MachineModuleInfoImpl::SymbolListTy -MachineModuleInfoImpl::GetSortedStubs(const DenseMap&Map) { +MachineModuleInfoImpl::SymbolListTy MachineModuleInfoImpl::getSortedStubs( + DenseMap &Map) { MachineModuleInfoImpl::SymbolListTy List(Map.begin(), Map.end()); if (!List.empty()) qsort(&List[0], List.size(), sizeof(List[0]), SortSymbolPair); + + Map.clear(); return List; } diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp index 7a3c80b..a52d05f 100644 --- a/lib/CodeGen/MachineScheduler.cpp +++ b/lib/CodeGen/MachineScheduler.cpp @@ -1036,8 +1036,6 @@ void ScheduleDAGMILive::schedule() { scheduleMI(SU, IsTopNode); - updateQueues(SU, IsTopNode); - if (DFSResult) { unsigned SubtreeID = DFSResult->getSubtreeID(SU); if (!ScheduledTrees.test(SubtreeID)) { @@ -1049,6 +1047,8 @@ void ScheduleDAGMILive::schedule() { // Notify the scheduling strategy after updating the DAG. SchedImpl->schedNode(SU, IsTopNode); + + updateQueues(SU, IsTopNode); } assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone."); diff --git a/lib/CodeGen/MachineTraceMetrics.cpp b/lib/CodeGen/MachineTraceMetrics.cpp index 8aacd1f..5dc7c21 100644 --- a/lib/CodeGen/MachineTraceMetrics.cpp +++ b/lib/CodeGen/MachineTraceMetrics.cpp @@ -463,13 +463,11 @@ void MachineTraceMetrics::Ensemble::computeTrace(const MachineBasicBlock *MBB) { // Run an upwards post-order search for the trace start. Bounds.Downward = false; Bounds.Visited.clear(); - typedef ipo_ext_iterator UpwardPO; - for (UpwardPO I = ipo_ext_begin(MBB, Bounds), E = ipo_ext_end(MBB, Bounds); - I != E; ++I) { + for (auto I : inverse_post_order_ext(MBB, Bounds)) { DEBUG(dbgs() << " pred for BB#" << I->getNumber() << ": "); TraceBlockInfo &TBI = BlockInfo[I->getNumber()]; // All the predecessors have been visited, pick the preferred one. - TBI.Pred = pickTracePred(*I); + TBI.Pred = pickTracePred(I); DEBUG({ if (TBI.Pred) dbgs() << "BB#" << TBI.Pred->getNumber() << '\n'; @@ -477,19 +475,17 @@ void MachineTraceMetrics::Ensemble::computeTrace(const MachineBasicBlock *MBB) { dbgs() << "null\n"; }); // The trace leading to I is now known, compute the depth resources. - computeDepthResources(*I); + computeDepthResources(I); } // Run a downwards post-order search for the trace end. Bounds.Downward = true; Bounds.Visited.clear(); - typedef po_ext_iterator DownwardPO; - for (DownwardPO I = po_ext_begin(MBB, Bounds), E = po_ext_end(MBB, Bounds); - I != E; ++I) { + for (auto I : post_order_ext(MBB, Bounds)) { DEBUG(dbgs() << " succ for BB#" << I->getNumber() << ": "); TraceBlockInfo &TBI = BlockInfo[I->getNumber()]; // All the successors have been visited, pick the preferred one. - TBI.Succ = pickTraceSucc(*I); + TBI.Succ = pickTraceSucc(I); DEBUG({ if (TBI.Succ) dbgs() << "BB#" << TBI.Succ->getNumber() << '\n'; @@ -497,7 +493,7 @@ void MachineTraceMetrics::Ensemble::computeTrace(const MachineBasicBlock *MBB) { dbgs() << "null\n"; }); // The trace leaving I is now known, compute the height resources. - computeHeightResources(*I); + computeHeightResources(I); } } diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index ad59fc9..55f08e4 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -141,7 +141,7 @@ namespace { TargetSubtargetInfo::AntiDepBreakMode AntiDepMode, SmallVectorImpl &CriticalPathRCs); - ~SchedulePostRATDList(); + ~SchedulePostRATDList() override; /// startBlock - Initialize register live-range state for scheduling in /// this block. diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index e073e6a..5334a63 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -30,6 +30,7 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/LLVMContext.h" @@ -752,6 +753,25 @@ void PEI::replaceFrameIndices(MachineFunction &Fn) { const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering(); if (!TFI.needsFrameIndexResolution(Fn)) return; + MachineModuleInfo &MMI = Fn.getMMI(); + const Function *F = Fn.getFunction(); + const Function *ParentF = MMI.getWinEHParent(F); + unsigned FrameReg; + if (F == ParentF) { + WinEHFuncInfo &FuncInfo = MMI.getWinEHFuncInfo(Fn.getFunction()); + // FIXME: This should be unconditional but we have bugs in the preparation + // pass. + if (FuncInfo.UnwindHelpFrameIdx != INT_MAX) + FuncInfo.UnwindHelpFrameOffset = TFI.getFrameIndexReferenceFromSP( + Fn, FuncInfo.UnwindHelpFrameIdx, FrameReg); + } else if (MMI.hasWinEHFuncInfo(F)) { + WinEHFuncInfo &FuncInfo = MMI.getWinEHFuncInfo(Fn.getFunction()); + auto I = FuncInfo.CatchHandlerParentFrameObjIdx.find(F); + if (I != FuncInfo.CatchHandlerParentFrameObjIdx.end()) + FuncInfo.CatchHandlerParentFrameObjOffset[F] = + TFI.getFrameIndexReferenceFromSP(Fn, I->second, FrameReg); + } + // Store SPAdj at exit of a basic block. SmallVector SPState; SPState.resize(Fn.getNumBlockIDs()); diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp index c621414..c311c7b 100644 --- a/lib/CodeGen/RegAllocFast.cpp +++ b/lib/CodeGen/RegAllocFast.cpp @@ -301,13 +301,9 @@ void RAFast::spillVirtReg(MachineBasicBlock::iterator MI, const MDNode *Expr = DBG->getDebugExpression(); bool IsIndirect = DBG->isIndirectDebugValue(); uint64_t Offset = IsIndirect ? DBG->getOperand(1).getImm() : 0; - DebugLoc DL; - if (MI == MBB->end()) { - // If MI is at basic block end then use last instruction's location. - MachineBasicBlock::iterator EI = MI; - DL = (--EI)->getDebugLoc(); - } else - DL = MI->getDebugLoc(); + DebugLoc DL = DBG->getDebugLoc(); + assert(cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); MachineInstr *NewDV = BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::DBG_VALUE)) .addFrameIndex(FI) @@ -877,6 +873,9 @@ void RAFast::AllocateBasicBlock() { const MDNode *Expr = MI->getDebugExpression(); DebugLoc DL = MI->getDebugLoc(); MachineBasicBlock *MBB = MI->getParent(); + assert( + cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); MachineInstr *NewDV = BuildMI(*MBB, MBB->erase(MI), DL, TII->get(TargetOpcode::DBG_VALUE)) .addFrameIndex(SS) diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp index e94f1bb..26f42c9 100644 --- a/lib/CodeGen/RegAllocGreedy.cpp +++ b/lib/CodeGen/RegAllocGreedy.cpp @@ -538,8 +538,9 @@ void RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) { // Giant live ranges fall back to the global assignment heuristic, which // prevents excessive spilling in pathological cases. bool ReverseLocal = TRI->reverseLocalAssignment(); + const TargetRegisterClass &RC = *MRI->getRegClass(Reg); bool ForceGlobal = !ReverseLocal && - (Size / SlotIndex::InstrDist) > (2 * MRI->getRegClass(Reg)->getNumRegs()); + (Size / SlotIndex::InstrDist) > (2 * RC.getNumRegs()); if (ExtraRegInfo[Reg].Stage == RS_Assign && !ForceGlobal && !LI->empty() && LIS->intervalIsInOneMBB(*LI)) { @@ -552,10 +553,10 @@ void RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) { // Allocating bottom up may allow many short LRGs to be assigned first // to one of the cheap registers. This could be much faster for very // large blocks on targets with many physical registers. - Prio = Indexes->getZeroIndex().getInstrDistance(LI->beginIndex()); + Prio = Indexes->getZeroIndex().getInstrDistance(LI->endIndex()); } - } - else { + Prio |= RC.AllocationPriority << 24; + } else { // Allocate global and split ranges in long->short order. Long ranges that // don't fit should be spilled (or split) ASAP so they don't create // interference. Mark a bit to prioritize global above local ranges. diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp index 9e3cf41..5d958a6 100644 --- a/lib/CodeGen/RegisterCoalescer.cpp +++ b/lib/CodeGen/RegisterCoalescer.cpp @@ -2731,15 +2731,19 @@ bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const { assert(Copy.isCopyLike()); if (!UseTerminalRule) return false; + unsigned DstReg, DstSubReg, SrcReg, SrcSubReg; + isMoveInstr(*TRI, &Copy, SrcReg, DstReg, SrcSubReg, DstSubReg); // Check if the destination of this copy has any other affinity. - unsigned DstReg = Copy.getOperand(0).getReg(); if (TargetRegisterInfo::isPhysicalRegister(DstReg) || + // If SrcReg is a physical register, the copy won't be coalesced. + // Ignoring it may have other side effect (like missing + // rematerialization). So keep it. + TargetRegisterInfo::isPhysicalRegister(SrcReg) || !isTerminalReg(DstReg, Copy, MRI)) return false; // DstReg is a terminal node. Check if it inteferes with any other // copy involving SrcReg. - unsigned SrcReg = Copy.getOperand(1).getReg(); const MachineBasicBlock *OrigBB = Copy.getParent(); const LiveInterval &DstLI = LIS->getInterval(DstReg); for (const MachineInstr &MI : MRI->reg_nodbg_instructions(SrcReg)) { @@ -2751,9 +2755,11 @@ bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const { // For now, just consider the copies that are in the same block. if (&MI == &Copy || !MI.isCopyLike() || MI.getParent() != OrigBB) continue; - unsigned OtherReg = MI.getOperand(0).getReg(); + unsigned OtherReg, OtherSubReg, OtherSrcReg, OtherSrcSubReg; + isMoveInstr(*TRI, &Copy, OtherSrcReg, OtherReg, OtherSrcSubReg, + OtherSubReg); if (OtherReg == SrcReg) - OtherReg = MI.getOperand(1).getReg(); + OtherReg = OtherSrcReg; // Check if OtherReg is a non-terminal. if (TargetRegisterInfo::isPhysicalRegister(OtherReg) || isTerminalReg(OtherReg, MI, MRI)) @@ -2775,25 +2781,45 @@ RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) { // yet, it might invalidate the iterator. const unsigned PrevSize = WorkList.size(); if (JoinGlobalCopies) { + SmallVector LocalTerminals; + SmallVector GlobalTerminals; // Coalesce copies bottom-up to coalesce local defs before local uses. They // are not inherently easier to resolve, but slightly preferable until we // have local live range splitting. In particular this is required by // cmp+jmp macro fusion. for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) { - if (!MII->isCopyLike() || applyTerminalRule(*MII)) + if (!MII->isCopyLike()) continue; - if (isLocalCopy(&(*MII), LIS)) - LocalWorkList.push_back(&(*MII)); - else - WorkList.push_back(&(*MII)); + bool ApplyTerminalRule = applyTerminalRule(*MII); + if (isLocalCopy(&(*MII), LIS)) { + if (ApplyTerminalRule) + LocalTerminals.push_back(&(*MII)); + else + LocalWorkList.push_back(&(*MII)); + } else { + if (ApplyTerminalRule) + GlobalTerminals.push_back(&(*MII)); + else + WorkList.push_back(&(*MII)); + } } + // Append the copies evicted by the terminal rule at the end of the list. + LocalWorkList.append(LocalTerminals.begin(), LocalTerminals.end()); + WorkList.append(GlobalTerminals.begin(), GlobalTerminals.end()); } else { + SmallVector Terminals; for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) - if (MII->isCopyLike() && !applyTerminalRule(*MII)) - WorkList.push_back(MII); + if (MII->isCopyLike()) { + if (applyTerminalRule(*MII)) + Terminals.push_back(&(*MII)); + else + WorkList.push_back(MII); + } + // Append the copies evicted by the terminal rule at the end of the list. + WorkList.append(Terminals.begin(), Terminals.end()); } // Try coalescing the collected copies immediately, and remove the nulls. // This prevents the WorkList from getting too large since most copies are diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index a1c84c5..22fd6d6 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -705,16 +705,8 @@ static bool isConstantSplatVector(SDNode *N, APInt& SplatValue) { EltVT.getSizeInBits() >= SplatBitSize); } -// \brief Returns the SDNode if it is a constant BuildVector or constant. -static SDNode *isConstantBuildVectorOrConstantInt(SDValue N) { - if (isa(N)) - return N.getNode(); - BuildVectorSDNode *BV = dyn_cast(N); - if (BV && BV->isConstant()) - return BV; - return nullptr; -} - +// \brief Returns the SDNode if it is a constant integer BuildVector +// or constant integer. static SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) { if (isa(N)) return N.getNode(); @@ -723,6 +715,8 @@ static SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) { return nullptr; } +// \brief Returns the SDNode if it is a constant float BuildVector +// or constant float. static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) { if (isa(N)) return N.getNode(); @@ -773,8 +767,8 @@ SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL, SDValue N0, SDValue N1) { EVT VT = N0.getValueType(); if (N0.getOpcode() == Opc) { - if (SDNode *L = isConstantBuildVectorOrConstantInt(N0.getOperand(1))) { - if (SDNode *R = isConstantBuildVectorOrConstantInt(N1)) { + if (SDNode *L = isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) { + if (SDNode *R = isConstantIntBuildVectorOrConstantInt(N1)) { // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2)) if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, VT, L, R)) return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); @@ -793,8 +787,8 @@ SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL, } if (N1.getOpcode() == Opc) { - if (SDNode *R = isConstantBuildVectorOrConstantInt(N1.getOperand(1))) { - if (SDNode *L = isConstantBuildVectorOrConstantInt(N0)) { + if (SDNode *R = isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) { + if (SDNode *L = isConstantIntBuildVectorOrConstantInt(N0)) { // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2)) if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, VT, R, L)) return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode); @@ -1583,8 +1577,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) { // fold vector ops if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (add x, 0) -> x, vector edition if (ISD::isBuildVectorAllZeros(N1.getNode())) @@ -1604,7 +1598,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C); // canonicalize constant to RHS - if (N0C && !N1C) + if (isConstantIntBuildVectorOrConstantInt(N0) && + !isConstantIntBuildVectorOrConstantInt(N1)) return DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, N0); // fold (add x, 0) -> x if (N1C && N1C->isNullValue()) @@ -1624,8 +1619,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) { N0C->getAPIntValue(), VT), N0.getOperand(1)); // reassociate add - SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1); - if (RADD.getNode()) + if (SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1)) return RADD; // fold ((0-A) + B) -> B-A if (N0.getOpcode() == ISD::SUB && isa(N0.getOperand(0)) && @@ -1828,8 +1822,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { // fold vector ops if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (sub x, 0) -> x, vector edition if (ISD::isBuildVectorAllZeros(N1.getNode())) @@ -1984,26 +1978,27 @@ SDValue DAGCombiner::visitMUL(SDNode *N) { APInt ConstValue0, ConstValue1; // fold vector ops if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; N0IsConst = isConstantSplatVector(N0.getNode(), ConstValue0); N1IsConst = isConstantSplatVector(N1.getNode(), ConstValue1); } else { - N0IsConst = dyn_cast(N0) != nullptr; - ConstValue0 = N0IsConst ? (dyn_cast(N0))->getAPIntValue() - : APInt(); - N1IsConst = dyn_cast(N1) != nullptr; - ConstValue1 = N1IsConst ? (dyn_cast(N1))->getAPIntValue() - : APInt(); + N0IsConst = isa(N0); + if (N0IsConst) + ConstValue0 = cast(N0)->getAPIntValue(); + N1IsConst = isa(N1); + if (N1IsConst) + ConstValue1 = cast(N1)->getAPIntValue(); } // fold (mul c1, c2) -> c1*c2 if (N0IsConst && N1IsConst) return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0.getNode(), N1.getNode()); - // canonicalize constant to RHS - if (N0IsConst && !N1IsConst) + // canonicalize constant to RHS (vector doesn't have to splat) + if (isConstantIntBuildVectorOrConstantInt(N0) && + !isConstantIntBuildVectorOrConstantInt(N1)) return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0); // fold (mul x, 0) -> 0 if (N1IsConst && ConstValue1 == 0) @@ -2083,8 +2078,7 @@ SDValue DAGCombiner::visitMUL(SDNode *N) { N0.getOperand(1), N1)); // reassociate mul - SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1); - if (RMUL.getNode()) + if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1)) return RMUL; return SDValue(); @@ -2096,10 +2090,9 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) { EVT VT = N->getValueType(0); // fold vector ops - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } + if (VT.isVector()) + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (sdiv c1, c2) -> c1/c2 ConstantSDNode *N0C = isConstOrConstSplat(N0); @@ -2163,7 +2156,7 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) { return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), SRA); } - // if integer divide is expensive and we satisfy the requirements, emit an + // If integer divide is expensive and we satisfy the requirements, emit an // alternate sequence. if (N1C && !TLI.isIntDivCheap()) { SDValue Op = BuildSDIV(N); @@ -2186,10 +2179,9 @@ SDValue DAGCombiner::visitUDIV(SDNode *N) { EVT VT = N->getValueType(0); // fold vector ops - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } + if (VT.isVector()) + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (udiv c1, c2) -> c1/c2 ConstantSDNode *N0C = isConstOrConstSplat(N0); @@ -2459,8 +2451,8 @@ SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) { EVT VT = N->getValueType(0); SDLoc DL(N); - // If the type twice as wide is legal, transform the mulhu to a wider multiply - // plus a shift. + // If the type is twice as wide is legal, transform the mulhu to a wider + // multiply plus a shift. if (VT.isSimple() && !VT.isVector()) { MVT Simple = VT.getSimpleVT(); unsigned SimpleSize = Simple.getSizeInBits(); @@ -2489,8 +2481,8 @@ SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) { EVT VT = N->getValueType(0); SDLoc DL(N); - // If the type twice as wide is legal, transform the mulhu to a wider multiply - // plus a shift. + // If the type is twice as wide is legal, transform the mulhu to a wider + // multiply plus a shift. if (VT.isSimple() && !VT.isVector()) { MVT Simple = VT.getSimpleVT(); unsigned SimpleSize = Simple.getSizeInBits(); @@ -2809,8 +2801,8 @@ SDValue DAGCombiner::visitAND(SDNode *N) { // fold vector ops if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (and x, 0) -> 0, vector edition if (ISD::isBuildVectorAllZeros(N0.getNode())) @@ -2839,7 +2831,8 @@ SDValue DAGCombiner::visitAND(SDNode *N) { if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C); // canonicalize constant to RHS - if (N0C && !N1C) + if (isConstantIntBuildVectorOrConstantInt(N0) && + !isConstantIntBuildVectorOrConstantInt(N1)) return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); // fold (and x, -1) -> x if (N1C && N1C->isAllOnesValue()) @@ -2850,8 +2843,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) { APInt::getAllOnesValue(BitWidth))) return DAG.getConstant(0, VT); // reassociate and - SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1); - if (RAND.getNode()) + if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1)) return RAND; // fold (and (or x, C), D) -> D if (C & D) == D if (N1C && N0.getOpcode() == ISD::OR) @@ -3470,8 +3462,8 @@ SDValue DAGCombiner::visitOR(SDNode *N) { // fold vector ops if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (or x, 0) -> x, vector edition if (ISD::isBuildVectorAllZeros(N0.getNode())) @@ -3556,7 +3548,8 @@ SDValue DAGCombiner::visitOR(SDNode *N) { if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C); // canonicalize constant to RHS - if (N0C && !N1C) + if (isConstantIntBuildVectorOrConstantInt(N0) && + !isConstantIntBuildVectorOrConstantInt(N1)) return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); // fold (or x, 0) -> x if (N1C && N1C->isNullValue()) @@ -3580,8 +3573,7 @@ SDValue DAGCombiner::visitOR(SDNode *N) { return BSwap; // reassociate or - SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1); - if (ROR.getNode()) + if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1)) return ROR; // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2) // iff (c1 & c2) == 0. @@ -3875,8 +3867,8 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { // fold vector ops if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (xor x, 0) -> x, vector edition if (ISD::isBuildVectorAllZeros(N0.getNode())) @@ -3899,14 +3891,14 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C); // canonicalize constant to RHS - if (N0C && !N1C) + if (isConstantIntBuildVectorOrConstantInt(N0) && + !isConstantIntBuildVectorOrConstantInt(N1)) return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0); // fold (xor x, 0) -> x if (N1C && N1C->isNullValue()) return N0; // reassociate xor - SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1); - if (RXOR.getNode()) + if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1)) return RXOR; // fold !(x cc y) -> (x !cc y) @@ -4152,8 +4144,8 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { // fold vector ops ConstantSDNode *N1C = dyn_cast(N1); if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; BuildVectorSDNode *N1CV = dyn_cast(N1); // If setcc produces all-one true value then: @@ -4332,8 +4324,8 @@ SDValue DAGCombiner::visitSRA(SDNode *N) { // fold vector ops ConstantSDNode *N1C = dyn_cast(N1); if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; N1C = isConstOrConstSplat(N1); } @@ -4478,8 +4470,8 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { // fold vector ops ConstantSDNode *N1C = dyn_cast(N1); if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; N1C = isConstOrConstSplat(N1); } @@ -4889,7 +4881,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { SDValue N1_0 = N1->getOperand(0); SDValue N1_1 = N1->getOperand(1); SDValue N1_2 = N1->getOperand(2); - if (N1_2 == N2) { + if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) { // Create the actual and node if we can generate good code for it. if (!TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) { SDValue And = DAG.getNode(ISD::AND, SDLoc(N), N0.getValueType(), @@ -4908,7 +4900,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { SDValue N2_0 = N2->getOperand(0); SDValue N2_1 = N2->getOperand(1); SDValue N2_2 = N2->getOperand(2); - if (N2_1 == N1) { + if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) { // Create the actual or node if we can generate good code for it. if (!TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) { SDValue Or = DAG.getNode(ISD::OR, SDLoc(N), N0.getValueType(), @@ -7037,7 +7029,6 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) { // Finally, this must be the case where we are shrinking elements: each input // turns into multiple outputs. - bool isS2V = ISD::isScalarToVector(BV); unsigned NumOutputsPerInput = SrcBitSize/DstBitSize; EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, NumOutputsPerInput*BV->getNumOperands()); @@ -7055,10 +7046,6 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) { for (unsigned j = 0; j != NumOutputsPerInput; ++j) { APInt ThisVal = OpVal.trunc(DstBitSize); Ops.push_back(DAG.getConstant(ThisVal, DstEltVT)); - if (isS2V && i == 0 && j == 0 && ThisVal.zext(SrcBitSize) == OpVal) - // Simply turn this into a SCALAR_TO_VECTOR of the new type. - return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BV), VT, - Ops[0]); OpVal = OpVal.lshr(DstBitSize); } @@ -7206,10 +7193,9 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { const TargetOptions &Options = DAG.getTarget().Options; // fold vector ops - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } + if (VT.isVector()) + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (fadd c1, c2) -> c1 + c2 if (N0CFP && N1CFP) @@ -7400,10 +7386,9 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) { const TargetOptions &Options = DAG.getTarget().Options; // fold vector ops - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } + if (VT.isVector()) + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (fsub c1, c2) -> c1-c2 if (N0CFP && N1CFP) @@ -7552,15 +7537,8 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { // fold vector ops if (VT.isVector()) { // This just handles C1 * C2 for vectors. Other vector folds are below. - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) + if (SDValue FoldedVOp = SimplifyVBinOp(N)) return FoldedVOp; - // Canonicalize vector constant to RHS. - if (N0.getOpcode() == ISD::BUILD_VECTOR && - N1.getOpcode() != ISD::BUILD_VECTOR) - if (auto *BV0 = dyn_cast(N0)) - if (BV0->isConstant()) - return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); } // fold (fmul c1, c2) -> c1*c2 @@ -7568,7 +7546,8 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, N1); // canonicalize constant to RHS - if (N0CFP && !N1CFP) + if (isConstantFPBuildVectorOrConstantFP(N0) && + !isConstantFPBuildVectorOrConstantFP(N1)) return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1, N0); // fold (fmul A, 1.0) -> A @@ -7734,10 +7713,9 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { const TargetOptions &Options = DAG.getTarget().Options; // fold vector ops - if (VT.isVector()) { - SDValue FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.getNode()) return FoldedVOp; - } + if (VT.isVector()) + if (SDValue FoldedVOp = SimplifyVBinOp(N)) + return FoldedVOp; // fold (fdiv c1, c2) -> c1/c2 if (N0CFP && N1CFP) @@ -8216,11 +8194,10 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) { SDValue DAGCombiner::visitFCEIL(SDNode *N) { SDValue N0 = N->getOperand(0); - ConstantFPSDNode *N0CFP = dyn_cast(N0); EVT VT = N->getValueType(0); // fold (fceil c1) -> fceil(c1) - if (N0CFP) + if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); return SDValue(); @@ -8228,11 +8205,10 @@ SDValue DAGCombiner::visitFCEIL(SDNode *N) { SDValue DAGCombiner::visitFTRUNC(SDNode *N) { SDValue N0 = N->getOperand(0); - ConstantFPSDNode *N0CFP = dyn_cast(N0); EVT VT = N->getValueType(0); // fold (ftrunc c1) -> ftrunc(c1) - if (N0CFP) + if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); return SDValue(); @@ -8240,11 +8216,10 @@ SDValue DAGCombiner::visitFTRUNC(SDNode *N) { SDValue DAGCombiner::visitFFLOOR(SDNode *N) { SDValue N0 = N->getOperand(0); - ConstantFPSDNode *N0CFP = dyn_cast(N0); EVT VT = N->getValueType(0); // fold (ffloor c1) -> ffloor(c1) - if (N0CFP) + if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); return SDValue(); @@ -10080,19 +10055,19 @@ bool DAGCombiner::MergeStoresOfConstantsOrVecElts( int64_t ElementSizeBytes = MemVT.getSizeInBits() / 8; LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode; - unsigned EarliestNodeUsed = 0; + unsigned LatestNodeUsed = 0; for (unsigned i=0; i < NumElem; ++i) { // Find a chain for the new wide-store operand. Notice that some // of the store nodes that we found may not be selected for inclusion // in the wide store. The chain we use needs to be the chain of the - // earliest store node which is *used* and replaced by the wide store. - if (StoreNodes[i].SequenceNum > StoreNodes[EarliestNodeUsed].SequenceNum) - EarliestNodeUsed = i; + // latest store node which is *used* and replaced by the wide store. + if (StoreNodes[i].SequenceNum < StoreNodes[LatestNodeUsed].SequenceNum) + LatestNodeUsed = i; } - // The earliest Node in the DAG. - LSBaseSDNode *EarliestOp = StoreNodes[EarliestNodeUsed].MemNode; + // The latest Node in the DAG. + LSBaseSDNode *LatestOp = StoreNodes[LatestNodeUsed].MemNode; SDLoc DL(StoreNodes[0].MemNode); SDValue StoredVal; @@ -10151,17 +10126,17 @@ bool DAGCombiner::MergeStoresOfConstantsOrVecElts( StoredVal = DAG.getConstant(StoreInt, StoreTy); } - SDValue NewStore = DAG.getStore(EarliestOp->getChain(), DL, StoredVal, + SDValue NewStore = DAG.getStore(LatestOp->getChain(), DL, StoredVal, FirstInChain->getBasePtr(), FirstInChain->getPointerInfo(), false, false, FirstInChain->getAlignment()); - // Replace the first store with the new store - CombineTo(EarliestOp, NewStore); + // Replace the last store with the new store + CombineTo(LatestOp, NewStore); // Erase all other stores. for (unsigned i = 0; i < NumElem ; ++i) { - if (StoreNodes[i].MemNode == EarliestOp) + if (StoreNodes[i].MemNode == LatestOp) continue; StoreSDNode *St = cast(StoreNodes[i].MemNode); // ReplaceAllUsesWith will replace all uses that existed when it was @@ -10538,18 +10513,19 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { if (NumElem < 2) return false; - // The earliest Node in the DAG. - unsigned EarliestNodeUsed = 0; - LSBaseSDNode *EarliestOp = StoreNodes[EarliestNodeUsed].MemNode; + // The latest Node in the DAG. + unsigned LatestNodeUsed = 0; for (unsigned i=1; i StoreNodes[EarliestNodeUsed].SequenceNum) - EarliestNodeUsed = i; + // latest store node which is *used* and replaced by the wide store. + if (StoreNodes[i].SequenceNum < StoreNodes[LatestNodeUsed].SequenceNum) + LatestNodeUsed = i; } + LSBaseSDNode *LatestOp = StoreNodes[LatestNodeUsed].MemNode; + // Find if it is better to use vectors or integers to load and store // to memory. EVT JointMemOpVT; @@ -10571,7 +10547,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { false, false, false, FirstLoad->getAlignment()); - SDValue NewStore = DAG.getStore(EarliestOp->getChain(), StoreDL, NewLoad, + SDValue NewStore = DAG.getStore(LatestOp->getChain(), StoreDL, NewLoad, FirstInChain->getBasePtr(), FirstInChain->getPointerInfo(), false, false, FirstInChain->getAlignment()); @@ -10589,12 +10565,12 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Ld->getChain()); } - // Replace the first store with the new store. - CombineTo(EarliestOp, NewStore); + // Replace the last store with the new store. + CombineTo(LatestOp, NewStore); // Erase all other stores. for (unsigned i = 0; i < NumElem ; ++i) { // Remove all Store nodes. - if (StoreNodes[i].MemNode == EarliestOp) + if (StoreNodes[i].MemNode == LatestOp) continue; StoreSDNode *St = cast(StoreNodes[i].MemNode); DAG.ReplaceAllUsesOfValueWith(SDValue(St, 0), St->getChain()); @@ -11523,6 +11499,68 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { return SDValue(); } +static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + EVT OpVT = N->getOperand(0).getValueType(); + + // If the operands are legal vectors, leave them alone. + if (TLI.isTypeLegal(OpVT)) + return SDValue(); + + SDLoc DL(N); + EVT VT = N->getValueType(0); + SmallVector Ops; + + EVT SVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); + SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); + + // Keep track of what we encounter. + bool AnyInteger = false; + bool AnyFP = false; + for (const SDValue &Op : N->ops()) { + if (ISD::BITCAST == Op.getOpcode() && + !Op.getOperand(0).getValueType().isVector()) + Ops.push_back(Op.getOperand(0)); + else if (ISD::UNDEF == Op.getOpcode()) + Ops.push_back(ScalarUndef); + else + return SDValue(); + + // Note whether we encounter an integer or floating point scalar. + // If it's neither, bail out, it could be something weird like x86mmx. + EVT LastOpVT = Ops.back().getValueType(); + if (LastOpVT.isFloatingPoint()) + AnyFP = true; + else if (LastOpVT.isInteger()) + AnyInteger = true; + else + return SDValue(); + } + + // If any of the operands is a floating point scalar bitcast to a vector, + // use floating point types throughout, and bitcast everything. + // Replace UNDEFs by another scalar UNDEF node, of the final desired type. + if (AnyFP) { + SVT = EVT::getFloatingPointVT(OpVT.getSizeInBits()); + ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); + if (AnyInteger) { + for (SDValue &Op : Ops) { + if (Op.getValueType() == SVT) + continue; + if (Op.getOpcode() == ISD::UNDEF) + Op = ScalarUndef; + else + Op = DAG.getNode(ISD::BITCAST, DL, SVT, Op); + } + } + } + + EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SVT, + VT.getSizeInBits() / SVT.getSizeInBits()); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, Ops)); +} + SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector @@ -11538,9 +11576,10 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { if (ISD::allOperandsUndef(N)) return DAG.getUNDEF(VT); - // Optimize concat_vectors where one of the vectors is undef. - if (N->getNumOperands() == 2 && - N->getOperand(1)->getOpcode() == ISD::UNDEF) { + // Optimize concat_vectors where all but the first of the vectors are undef. + if (std::all_of(std::next(N->op_begin()), N->op_end(), [](const SDValue &Op) { + return Op.getOpcode() == ISD::UNDEF; + })) { SDValue In = N->getOperand(0); assert(In.getValueType().isVector() && "Must concat vectors"); @@ -11548,6 +11587,15 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { if (In->getOpcode() == ISD::BITCAST && !In->getOperand(0)->getValueType(0).isVector()) { SDValue Scalar = In->getOperand(0); + + // If the bitcast type isn't legal, it might be a trunc of a legal type; + // look through the trunc so we can still do the transform: + // concat_vectors(trunc(scalar), undef) -> scalar_to_vector(scalar) + if (Scalar->getOpcode() == ISD::TRUNCATE && + !TLI.isTypeLegal(Scalar.getValueType()) && + TLI.isTypeLegal(Scalar->getOperand(0).getValueType())) + Scalar = Scalar->getOperand(0); + EVT SclTy = Scalar->getValueType(0); if (!SclTy.isFloatingPoint() && !SclTy.isInteger()) @@ -11615,6 +11663,10 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Opnds); } + // Fold CONCAT_VECTORS of only bitcast scalars (or undef) to BUILD_VECTOR. + if (SDValue V = combineConcatVectorOfScalars(N, DAG)) + return V; + // Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR // nodes often generate nop CONCAT_VECTOR nodes. // Scan the CONCAT_VECTOR operands and look for a CONCAT operations that @@ -11676,7 +11728,7 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { // type. if (V->getOperand(0).getValueType() != NVT) return SDValue(); - unsigned Idx = dyn_cast(N->getOperand(1))->getZExtValue(); + unsigned Idx = N->getConstantOperandVal(1); unsigned NumElems = NVT.getVectorNumElements(); assert((Idx % NumElems) == 0 && "IDX in concat is not a multiple of the result vector length."); @@ -12001,6 +12053,43 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { return V; } + // Attempt to combine a shuffle of 2 inputs of 'scalar sources' - + // BUILD_VECTOR or SCALAR_TO_VECTOR into a single BUILD_VECTOR. + if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT)) { + SmallVector Ops; + for (int M : SVN->getMask()) { + SDValue Op = DAG.getUNDEF(VT.getScalarType()); + if (M >= 0) { + int Idx = M % NumElts; + SDValue &S = (M < (int)NumElts ? N0 : N1); + if (S.getOpcode() == ISD::BUILD_VECTOR && S.hasOneUse()) { + Op = S.getOperand(Idx); + } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR && S.hasOneUse()) { + if (Idx == 0) + Op = S.getOperand(0); + } else { + // Operand can't be combined - bail out. + break; + } + } + Ops.push_back(Op); + } + if (Ops.size() == VT.getVectorNumElements()) { + // BUILD_VECTOR requires all inputs to be of the same type, find the + // maximum type and extend them all. + EVT SVT = VT.getScalarType(); + if (SVT.isInteger()) + for (SDValue &Op : Ops) + SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); + if (SVT != VT.getScalarType()) + for (SDValue &Op : Ops) + Op = TLI.isZExtFree(Op.getValueType(), SVT) + ? DAG.getZExtOrTrunc(Op, SDLoc(N), SVT) + : DAG.getSExtOrTrunc(Op, SDLoc(N), SVT); + return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Ops); + } + } + // If this shuffle only has a single input that is a bitcasted shuffle, // attempt to merge the 2 shuffles and suitably bitcast the inputs/output // back to their original types. diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 223a149..5ffb826 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -425,7 +425,7 @@ bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) { // Check if the second operand is a constant and handle it appropriately. if (const auto *CI = dyn_cast(I->getOperand(1))) { - uint64_t Imm = CI->getZExtValue(); + uint64_t Imm = CI->getSExtValue(); // Transform "sdiv exact X, 8" -> "sra X, 3". if (ISDOpcode == ISD::SDIV && isa(I) && @@ -1079,11 +1079,16 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { // The donothing intrinsic does, well, nothing. case Intrinsic::donothing: return true; + case Intrinsic::eh_actions: { + unsigned ResultReg = getRegForValue(UndefValue::get(II->getType())); + if (!ResultReg) + return false; + updateValueMap(II, ResultReg); + return true; + } case Intrinsic::dbg_declare: { const DbgDeclareInst *DI = cast(II); - DIVariable DIVar(DI->getVariable()); - assert((!DIVar || DIVar.isVariable()) && - "Variable in DbgDeclareInst should be either null or a DIVariable."); + DIVariable DIVar = DI->getVariable(); if (!DIVar || !FuncInfo.MF->getMMI().hasDebugInfo()) { DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); return true; @@ -1124,6 +1129,8 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { false); if (Op) { + assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) && + "Expected inlined-at fields to agree"); if (Op->isReg()) { Op->setIsDebug(true); BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, @@ -1148,6 +1155,8 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { const DbgValueInst *DI = cast(II); const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); const Value *V = DI->getValue(); + assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) && + "Expected inlined-at fields to agree"); if (!V) { // Currently the optimizer can produce this; insert an undef to // help debugging. Probably the optimizer should not do this. diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index 291b583..4b8ae32 100644 --- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -20,6 +20,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" @@ -79,12 +80,35 @@ static ISD::NodeType getPreferredExtendForValue(const Value *V) { return ExtendKind; } +namespace { +struct WinEHNumbering { + WinEHNumbering(WinEHFuncInfo &FuncInfo) : FuncInfo(FuncInfo), NextState(0) {} + + WinEHFuncInfo &FuncInfo; + int NextState; + + SmallVector HandlerStack; + SmallPtrSet VisitedHandlers; + + int currentEHNumber() const { + return HandlerStack.empty() ? -1 : HandlerStack.back()->getEHState(); + } + + void createUnwindMapEntry(int ToState, ActionHandler *AH); + void createTryBlockMapEntry(int TryLow, int TryHigh, + ArrayRef Handlers); + void processCallSite(ArrayRef Actions, ImmutableCallSite CS); + void calculateStateNumbers(const Function &F); +}; +} + void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, SelectionDAG *DAG) { Fn = &fn; MF = &mf; TLI = MF->getSubtarget().getTargetLowering(); RegInfo = &MF->getRegInfo(); + MachineModuleInfo &MMI = MF->getMMI(); // Check whether the function can return without sret-demotion. SmallVector Outs; @@ -178,13 +202,8 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, // during the initial isel pass through the IR so that it is done // in a predictable order. if (const DbgDeclareInst *DI = dyn_cast(I)) { - MachineModuleInfo &MMI = MF->getMMI(); - DIVariable DIVar(DI->getVariable()); - assert((!DIVar || DIVar.isVariable()) && - "Variable in DbgDeclareInst should be either null or a DIVariable."); - if (MMI.hasDebugInfo() && - DIVar && - !DI->getDebugLoc().isUnknown()) { + DIVariable DIVar = DI->getVariable(); + if (MMI.hasDebugInfo() && DIVar && DI->getDebugLoc()) { // Don't handle byval struct arguments or VLAs, for example. // Non-byval arguments are handled here (they refer to the stack // temporary alloca at this point). @@ -252,8 +271,179 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, // Mark landing pad blocks. for (BB = Fn->begin(); BB != EB; ++BB) - if (const InvokeInst *Invoke = dyn_cast(BB->getTerminator())) + if (const auto *Invoke = dyn_cast(BB->getTerminator())) MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); + + // Calculate EH numbers for WinEH. + if (fn.hasFnAttribute("wineh-parent")) { + const Function *WinEHParentFn = MMI.getWinEHParent(&fn); + WinEHFuncInfo &FI = MMI.getWinEHFuncInfo(WinEHParentFn); + if (FI.LandingPadStateMap.empty()) { + WinEHNumbering Num(FI); + Num.calculateStateNumbers(*WinEHParentFn); + // Pop everything on the handler stack. + Num.processCallSite(None, ImmutableCallSite()); + } + } +} + +void WinEHNumbering::createUnwindMapEntry(int ToState, ActionHandler *AH) { + WinEHUnwindMapEntry UME; + UME.ToState = ToState; + if (auto *CH = dyn_cast_or_null(AH)) + UME.Cleanup = cast(CH->getHandlerBlockOrFunc()); + else + UME.Cleanup = nullptr; + FuncInfo.UnwindMap.push_back(UME); +} + +void WinEHNumbering::createTryBlockMapEntry(int TryLow, int TryHigh, + ArrayRef Handlers) { + WinEHTryBlockMapEntry TBME; + TBME.TryLow = TryLow; + TBME.TryHigh = TryHigh; + assert(TBME.TryLow <= TBME.TryHigh); + for (CatchHandler *CH : Handlers) { + WinEHHandlerType HT; + if (CH->getSelector()->isNullValue()) { + HT.Adjectives = 0x40; + HT.TypeDescriptor = nullptr; + } else { + auto *GV = cast(CH->getSelector()->stripPointerCasts()); + // Selectors are always pointers to GlobalVariables with 'struct' type. + // The struct has two fields, adjectives and a type descriptor. + auto *CS = cast(GV->getInitializer()); + HT.Adjectives = + cast(CS->getAggregateElement(0U))->getZExtValue(); + HT.TypeDescriptor = + cast(CS->getAggregateElement(1)->stripPointerCasts()); + } + HT.Handler = cast(CH->getHandlerBlockOrFunc()); + HT.CatchObjRecoverIdx = CH->getExceptionVarIndex(); + TBME.HandlerArray.push_back(HT); + } + FuncInfo.TryBlockMap.push_back(TBME); +} + +static void print_name(const Value *V) { +#ifndef NDEBUG + if (!V) { + DEBUG(dbgs() << "null"); + return; + } + + if (const auto *F = dyn_cast(V)) + DEBUG(dbgs() << F->getName()); + else + DEBUG(V->dump()); +#endif +} + +void WinEHNumbering::processCallSite(ArrayRef Actions, + ImmutableCallSite CS) { + int FirstMismatch = 0; + for (int E = std::min(HandlerStack.size(), Actions.size()); FirstMismatch < E; + ++FirstMismatch) { + if (HandlerStack[FirstMismatch]->getHandlerBlockOrFunc() != + Actions[FirstMismatch]->getHandlerBlockOrFunc()) + break; + delete Actions[FirstMismatch]; + } + + bool EnteringScope = (int)Actions.size() > FirstMismatch; + + // Don't recurse while we are looping over the handler stack. Instead, defer + // the numbering of the catch handlers until we are done popping. + SmallVector PoppedCatches; + for (int I = HandlerStack.size() - 1; I >= FirstMismatch; --I) { + if (auto *CH = dyn_cast(HandlerStack.back())) { + PoppedCatches.push_back(CH); + } else { + // Delete cleanup handlers + delete HandlerStack.back(); + } + HandlerStack.pop_back(); + } + + // We need to create a new state number if we are exiting a try scope and we + // will not push any more actions. + int TryHigh = NextState - 1; + if (!EnteringScope && !PoppedCatches.empty()) { + createUnwindMapEntry(currentEHNumber(), nullptr); + ++NextState; + } + + int LastTryLowIdx = 0; + for (int I = 0, E = PoppedCatches.size(); I != E; ++I) { + CatchHandler *CH = PoppedCatches[I]; + if (I + 1 == E || CH->getEHState() != PoppedCatches[I + 1]->getEHState()) { + int TryLow = CH->getEHState(); + auto Handlers = + makeArrayRef(&PoppedCatches[LastTryLowIdx], I - LastTryLowIdx + 1); + createTryBlockMapEntry(TryLow, TryHigh, Handlers); + LastTryLowIdx = I + 1; + } + } + + for (CatchHandler *CH : PoppedCatches) { + if (auto *F = dyn_cast(CH->getHandlerBlockOrFunc())) + calculateStateNumbers(*F); + delete CH; + } + + bool LastActionWasCatch = false; + for (size_t I = FirstMismatch; I != Actions.size(); ++I) { + // We can reuse eh states when pushing two catches for the same invoke. + bool CurrActionIsCatch = isa(Actions[I]); + // FIXME: Reenable this optimization! + if (CurrActionIsCatch && LastActionWasCatch && false) { + Actions[I]->setEHState(currentEHNumber()); + } else { + createUnwindMapEntry(currentEHNumber(), Actions[I]); + Actions[I]->setEHState(NextState); + NextState++; + DEBUG(dbgs() << "Creating unwind map entry for: ("); + print_name(Actions[I]->getHandlerBlockOrFunc()); + DEBUG(dbgs() << ", " << currentEHNumber() << ")\n"); + } + HandlerStack.push_back(Actions[I]); + LastActionWasCatch = CurrActionIsCatch; + } + + DEBUG(dbgs() << "In EHState " << currentEHNumber() << " for CallSite: "); + print_name(CS ? CS.getCalledValue() : nullptr); + DEBUG(dbgs() << '\n'); +} + +void WinEHNumbering::calculateStateNumbers(const Function &F) { + auto I = VisitedHandlers.insert(&F); + if (!I.second) + return; // We've already visited this handler, don't renumber it. + + DEBUG(dbgs() << "Calculating state numbers for: " << F.getName() << '\n'); + SmallVector ActionList; + for (const BasicBlock &BB : F) { + for (const Instruction &I : BB) { + const auto *CI = dyn_cast(&I); + if (!CI || CI->doesNotThrow()) + continue; + processCallSite(None, CI); + } + const auto *II = dyn_cast(BB.getTerminator()); + if (!II) + continue; + const LandingPadInst *LPI = II->getLandingPadInst(); + auto *ActionsCall = dyn_cast(LPI->getNextNode()); + if (!ActionsCall) + continue; + assert(ActionsCall->getIntrinsicID() == Intrinsic::eh_actions); + parseEHActions(ActionsCall, ActionList); + processCallSite(ActionList, II); + ActionList.clear(); + FuncInfo.LandingPadStateMap[LPI] = currentEHNumber(); + } + + FuncInfo.CatchHandlerMaxState[&F] = NextState - 1; } /// clear - Clear out all the function-specific state. This returns this @@ -462,8 +652,7 @@ void llvm::ComputeUsesVAFloatArgument(const CallInst &I, if (FT->isVarArg() && !MMI->usesVAFloatArgument()) { for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { Type* T = I.getArgOperand(i)->getType(); - for (po_iterator i = po_begin(T), e = po_end(T); - i != e; ++i) { + for (auto i : post_order(T)) { if (i->isFloatingPointTy()) { MMI->setUsesVAFloatArgument(true); return; diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 93699a7..64d606a 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -650,6 +650,8 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD, MDNode *Var = SD->getVariable(); MDNode *Expr = SD->getExpression(); DebugLoc DL = SD->getDebugLoc(); + assert(cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); if (SD->getKind() == SDDbgValue::FRAMEIX) { // Stack address; this needs to be lowered in target-dependent fashion. diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index ece38f3..4a28a4b 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -4033,6 +4033,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { Node->getOpcode() == ISD::SETCC) { OVT = Node->getOperand(0).getSimpleValueType(); } + if (Node->getOpcode() == ISD::BR_CC) + OVT = Node->getOperand(2).getSimpleValueType(); MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); SDLoc dl(Node); SDValue Tmp1, Tmp2, Tmp3; @@ -4188,11 +4190,28 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { Tmp1, Tmp2, Node->getOperand(2))); break; } + case ISD::BR_CC: { + unsigned ExtOp = ISD::FP_EXTEND; + if (NVT.isInteger()) { + ISD::CondCode CCCode = + cast(Node->getOperand(1))->get(); + ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + } + Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); + Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); + Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), + Node->getOperand(0), Node->getOperand(1), + Tmp1, Tmp2, Node->getOperand(4))); + break; + } case ISD::FADD: case ISD::FSUB: case ISD::FMUL: case ISD::FDIV: case ISD::FREM: + case ISD::FMINNUM: + case ISD::FMAXNUM: + case ISD::FCOPYSIGN: case ISD::FPOW: { Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); @@ -4201,10 +4220,40 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { Tmp3, DAG.getIntPtrConstant(0))); break; } - case ISD::FLOG2: - case ISD::FEXP2: + case ISD::FMA: { + Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); + Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); + Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); + Results.push_back( + DAG.getNode(ISD::FP_ROUND, dl, OVT, + DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), + DAG.getIntPtrConstant(0))); + break; + } + case ISD::FPOWI: { + Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); + Tmp2 = Node->getOperand(1); + Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); + Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, + Tmp3, DAG.getIntPtrConstant(0))); + break; + } + case ISD::FFLOOR: + case ISD::FCEIL: + case ISD::FRINT: + case ISD::FNEARBYINT: + case ISD::FROUND: + case ISD::FTRUNC: + case ISD::FNEG: + case ISD::FSQRT: + case ISD::FSIN: + case ISD::FCOS: case ISD::FLOG: - case ISD::FEXP: { + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FABS: + case ISD::FEXP: + case ISD::FEXP2: { Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index cef3fc9..9de85d7 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -593,6 +593,7 @@ private: bool SplitVectorOperand(SDNode *N, unsigned OpNo); SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo); SDValue SplitVecOp_UnaryOp(SDNode *N); + SDValue SplitVecOp_TruncateHelper(SDNode *N, unsigned TruncateOp); SDValue SplitVecOp_BITCAST(SDNode *N); SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N); @@ -600,7 +601,6 @@ private: SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo); SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo); SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N); - SDValue SplitVecOp_TRUNCATE(SDNode *N); SDValue SplitVecOp_VSETCC(SDNode *N); SDValue SplitVecOp_FP_ROUND(SDNode *N); diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 03c2734..408d5ed 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -379,8 +379,8 @@ SDValue VectorLegalizer::Promote(SDValue Op) { // There are currently two cases of vector promotion: // 1) Bitcasting a vector of integers to a different type to a vector of the - // same overall length. For example, x86 promotes ISD::AND on v2i32 to v1i64. - // 2) Extending a vector of floats to a vector of the same number oflarger + // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64. + // 2) Extending a vector of floats to a vector of the same number of larger // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32. MVT VT = Op.getSimpleValueType(); assert(Op.getNode()->getNumValues() == 1 && diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index f7e4557..f000902 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1293,7 +1293,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; - case ISD::TRUNCATE: Res = SplitVecOp_TRUNCATE(N); break; + case ISD::TRUNCATE: + Res = SplitVecOp_TruncateHelper(N, ISD::TRUNCATE); + break; case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; case ISD::STORE: Res = SplitVecOp_STORE(cast(N), OpNo); @@ -1304,20 +1306,32 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { case ISD::VSELECT: Res = SplitVecOp_VSELECT(N, OpNo); break; - case ISD::CTTZ: - case ISD::CTLZ: - case ISD::CTPOP: - case ISD::FP_EXTEND: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: + if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) + Res = SplitVecOp_TruncateHelper(N, ISD::TRUNCATE); + else + Res = SplitVecOp_UnaryOp(N); + break; case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: - case ISD::FTRUNC: + if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) + Res = SplitVecOp_TruncateHelper(N, ISD::FTRUNC); + else + Res = SplitVecOp_UnaryOp(N); + break; + case ISD::CTTZ: + case ISD::CTLZ: + case ISD::CTPOP: + case ISD::FP_EXTEND: case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: case ISD::ANY_EXTEND: Res = SplitVecOp_UnaryOp(N); break; + case ISD::FTRUNC: + Res = SplitVecOp_TruncateHelper(N, ISD::FTRUNC); + break; } } @@ -1581,7 +1595,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts); } -SDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { +SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N, + unsigned TruncateOp) { // The result type is legal, but the input type is illegal. If splitting // ends up with the result type of each half still being legal, just // do that. If, however, that would result in an illegal result type, @@ -1624,8 +1639,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements/2); - SDValue HalfLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InLoVec); - SDValue HalfHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InHiVec); + SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); + SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); // Concatenate them to get the full intermediate truncation result. EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, @@ -1634,7 +1649,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { // type. This should normally be something that ends up being legal directly, // but in theory if a target has very wide vectors and an annoyingly // restricted set of legal types, this split can chain to build things up. - return DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); + return DAG.getNode(TruncateOp, DL, OutVT, InterVec); } SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 8b54e656..fd0fa31 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -173,7 +173,7 @@ public: HazardRec = STI.getInstrInfo()->CreateTargetHazardRecognizer(&STI, this); } - ~ScheduleDAGRRList() { + ~ScheduleDAGRRList() override { delete HazardRec; delete AvailableQueue; } @@ -1423,9 +1423,10 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { // If one or more successors has been unscheduled, then the current // node is no longer available. - if (!TrySU->isAvailable) + if (!TrySU->isAvailable || !TrySU->NodeQueueId) CurSU = AvailableQueue->pop(); else { + // Available and in AvailableQueue AvailableQueue->remove(TrySU); CurSU = TrySU; } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 2cd1f4b..6351fa2 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -44,7 +44,7 @@ namespace llvm { explicit ScheduleDAGSDNodes(MachineFunction &mf); - virtual ~ScheduleDAGSDNodes() {} + ~ScheduleDAGSDNodes() override {} /// Run - perform scheduling. /// diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp index 418b58e..eee4a4b 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp @@ -76,7 +76,7 @@ public: HazardRec = STI.getInstrInfo()->CreateTargetHazardRecognizer(&STI, this); } - ~ScheduleDAGVLIW() { + ~ScheduleDAGVLIW() override { delete HazardRec; delete AvailableQueue; } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index b52f648..770f0b2 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2851,10 +2851,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, // FIXME: Entirely reasonable to perform folding of other unary // operations here as the need arises. break; + case ISD::TRUNCATE: + // Constant build vector truncation can be done with the original scalar + // operands but with a new build vector with the truncated value type. + return getNode(ISD::BUILD_VECTOR, DL, VT, BV->ops()); case ISD::FNEG: case ISD::FABS: + case ISD::FCEIL: + case ISD::FTRUNC: + case ISD::FFLOOR: case ISD::FP_EXTEND: - case ISD::TRUNCATE: case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: { // Let the above scalar folding handle the folding of each element. @@ -2870,6 +2876,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, } if (Ops.size() == VT.getVectorNumElements()) return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); + break; } } } @@ -3628,7 +3635,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, CSEMap.InsertNode(N, IP); } else { - N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact); } @@ -3791,12 +3797,27 @@ static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); } - Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); + assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); + EVT IntVT = VT.getScalarType(); + if (!IntVT.isInteger()) + IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); + + Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); if (NumBits > 8) { // Use a multiplication with 0x010101... to extend the input to the // required length. APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); - Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); + Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, + DAG.getConstant(Magic, IntVT)); + } + + if (VT != Value.getValueType() && !VT.isInteger()) + Value = DAG.getNode(ISD::BITCAST, dl, VT.getScalarType(), Value); + if (VT != Value.getValueType()) { + assert(VT.getVectorElementType() == Value.getValueType() && + "value type should be one vector element here"); + SmallVector BVOps(VT.getVectorNumElements(), Value); + Value = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, BVOps); } return Value; @@ -4276,7 +4297,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, - MachinePointerInfo DstPtrInfo, + bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) { assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); @@ -4334,15 +4355,16 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, Type::getVoidTy(*getContext()), getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), TLI->getPointerTy()), std::move(Args), 0) - .setDiscardResult(); - std::pair CallResult = TLI->LowerCallTo(CLI); + .setDiscardResult() + .setTailCall(isTailCall); + std::pair CallResult = TLI->LowerCallTo(CLI); return CallResult.second; } SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, - unsigned Align, bool isVol, + unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) { assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); @@ -4389,15 +4411,16 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, Type::getVoidTy(*getContext()), getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), TLI->getPointerTy()), std::move(Args), 0) - .setDiscardResult(); - std::pair CallResult = TLI->LowerCallTo(CLI); + .setDiscardResult() + .setTailCall(isTailCall); + std::pair CallResult = TLI->LowerCallTo(CLI); return CallResult.second; } SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, - unsigned Align, bool isVol, + unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo) { assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); @@ -4446,7 +4469,8 @@ SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, Type::getVoidTy(*getContext()), getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), TLI->getPointerTy()), std::move(Args), 0) - .setDiscardResult(); + .setDiscardResult() + .setTailCall(isTailCall); std::pair CallResult = TLI->LowerCallTo(CLI); return CallResult.second; @@ -5574,8 +5598,7 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, /// For IROrder, we keep the smaller of the two SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { DebugLoc NLoc = N->getDebugLoc(); - if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && - (OLoc.getDebugLoc() != NLoc)) { + if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) { N->setDebugLoc(DebugLoc()); } unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); @@ -5885,6 +5908,8 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, bool IsIndirect, uint64_t Off, DebugLoc DL, unsigned O) { + assert(cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O); } @@ -5892,6 +5917,8 @@ SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr, const Value *C, uint64_t Off, DebugLoc DL, unsigned O) { + assert(cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O); } @@ -5899,6 +5926,8 @@ SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr, SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, uint64_t Off, DebugLoc DL, unsigned O) { + assert(cast(Var)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 6c14e79..32d2aae 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -35,6 +35,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/StackMaps.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -997,14 +998,16 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, const DbgValueInst *DI = DDI.getDI(); DebugLoc dl = DDI.getdl(); unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); - MDNode *Variable = DI->getVariable(); - MDNode *Expr = DI->getExpression(); + MDLocalVariable *Variable = DI->getVariable(); + MDExpression *Expr = DI->getExpression(); + assert(Variable->isValidLocationForIntrinsic(dl) && + "Expected inlined-at fields to agree"); uint64_t Offset = DI->getOffset(); // A dbg.value for an alloca is always indirect. bool IsIndirect = isa(V) || Offset != 0; SDDbgValue *SDV; if (Val.getNode()) { - if (!EmitFuncArgumentDbgValue(V, Variable, Expr, Offset, IsIndirect, + if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, IsIndirect, Val)) { SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(), IsIndirect, Offset, dl, DbgSDNodeOrder); @@ -4447,11 +4450,9 @@ static unsigned getTruncatedArgReg(const SDValue &N) { /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function /// argument, create the corresponding DBG_VALUE machine instruction for it now. /// At the end of instruction selection, they will be inserted to the entry BB. -bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, - MDNode *Variable, - MDNode *Expr, int64_t Offset, - bool IsIndirect, - const SDValue &N) { +bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( + const Value *V, MDLocalVariable *Variable, MDExpression *Expr, + MDLocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) { const Argument *Arg = dyn_cast(V); if (!Arg) return false; @@ -4460,8 +4461,10 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); // Ignore inlined function arguments here. + // + // FIXME: Should we be checking DL->inlinedAt() to determine this? DIVariable DV(Variable); - if (DV.isInlinedFnArgument(MF.getFunction())) + if (!DV->getScope()->getSubprogram()->describes(MF.getFunction())) return false; Optional Op; @@ -4502,13 +4505,15 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, if (!Op) return false; + assert(Variable->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); if (Op->isReg()) FuncInfo.ArgDbgValues.push_back( - BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE), - IsIndirect, Op->getReg(), Offset, Variable, Expr)); + BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect, + Op->getReg(), Offset, Variable, Expr)); else FuncInfo.ArgDbgValues.push_back( - BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE)) + BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE)) .addOperand(*Op) .addImm(Offset) .addMetadata(Variable) @@ -4589,9 +4594,12 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (!Align) Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment. bool isVol = cast(I.getArgOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false, - MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1)))); + bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget()); + SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, + false, isTC, + MachinePointerInfo(I.getArgOperand(0)), + MachinePointerInfo(I.getArgOperand(1))); + updateDAGForMaybeTailCall(MC); return nullptr; } case Intrinsic::memset: { @@ -4608,8 +4616,10 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (!Align) Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment. bool isVol = cast(I.getArgOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, - MachinePointerInfo(I.getArgOperand(0)))); + bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget()); + SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, + isTC, MachinePointerInfo(I.getArgOperand(0))); + updateDAGForMaybeTailCall(MS); return nullptr; } case Intrinsic::memmove: { @@ -4628,19 +4638,19 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (!Align) Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment. bool isVol = cast(I.getArgOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, - MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1)))); + bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget()); + SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, + isTC, MachinePointerInfo(I.getArgOperand(0)), + MachinePointerInfo(I.getArgOperand(1))); + updateDAGForMaybeTailCall(MM); return nullptr; } case Intrinsic::dbg_declare: { const DbgDeclareInst &DI = cast(I); - MDNode *Variable = DI.getVariable(); - MDNode *Expression = DI.getExpression(); + MDLocalVariable *Variable = DI.getVariable(); + MDExpression *Expression = DI.getExpression(); const Value *Address = DI.getAddress(); - DIVariable DIVar(Variable); - assert((!DIVar || DIVar.isVariable()) && - "Variable in DbgDeclareInst should be either null or a DIVariable."); + DIVariable DIVar = Variable; if (!Address || !DIVar) { DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); return nullptr; @@ -4663,7 +4673,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Address = BCI->getOperand(0); // Parameters are handled specially. bool isParameter = - (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable || + (DIVariable(Variable)->getTag() == dwarf::DW_TAG_arg_variable || isa(Address)); const AllocaInst *AI = dyn_cast(Address); @@ -4677,7 +4687,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { else { // Address is an argument, so try to emit its dbg value using // virtual register info from the FuncInfo.ValueMap. - EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, N); + EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false, + N); return nullptr; } } else if (AI) @@ -4694,7 +4705,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } else { // If Address is an argument then try to emit its dbg value using // virtual register info from the FuncInfo.ValueMap. - if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, + if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false, N)) { // If variable is pinned by a alloca in dominating bb then // use StaticAllocaMap. @@ -4717,14 +4728,12 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } case Intrinsic::dbg_value: { const DbgValueInst &DI = cast(I); - DIVariable DIVar(DI.getVariable()); - assert((!DIVar || DIVar.isVariable()) && - "Variable in DbgValueInst should be either null or a DIVariable."); + DIVariable DIVar = DI.getVariable(); if (!DIVar) return nullptr; - MDNode *Variable = DI.getVariable(); - MDNode *Expression = DI.getExpression(); + MDLocalVariable *Variable = DI.getVariable(); + MDExpression *Expression = DI.getExpression(); uint64_t Offset = DI.getOffset(); const Value *V = DI.getValue(); if (!V) @@ -4745,7 +4754,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (N.getNode()) { // A dbg.value for an alloca is always indirect. bool IsIndirect = isa(V) || Offset != 0; - if (!EmitFuncArgumentDbgValue(V, Variable, Expression, Offset, + if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset, IsIndirect, N)) { SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), IsIndirect, Offset, dl, SDNodeOrder); @@ -5360,6 +5369,9 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } case Intrinsic::clear_cache: return TLI.getClearCacheBuiltinName(); + case Intrinsic::eh_actions: + setValue(&I, DAG.getUNDEF(TLI.getPointerTy())); + return nullptr; case Intrinsic::donothing: // ignore return nullptr; @@ -5397,14 +5409,16 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // Directly emit some FRAME_ALLOC machine instrs. Label assignment emission // is the same on all targets. for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) { - AllocaInst *Slot = - cast(I.getArgOperand(Idx)->stripPointerCasts()); + Value *Arg = I.getArgOperand(Idx)->stripPointerCasts(); + if (isa(Arg)) + continue; // Skip null pointers. They represent a hole in index space. + AllocaInst *Slot = cast(Arg); assert(FuncInfo.StaticAllocaMap.count(Slot) && "can only escape static allocas"); int FI = FuncInfo.StaticAllocaMap[Slot]; MCSymbol *FrameAllocSym = - MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName(), - Idx); + MF.getMMI().getContext().getOrCreateFrameAllocSymbol( + GlobalValue::getRealLinkageName(MF.getName()), Idx); BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, TII->get(TargetOpcode::FRAME_ALLOC)) .addSym(FrameAllocSym) @@ -5424,8 +5438,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { auto *Idx = cast(I.getArgOperand(2)); unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX)); MCSymbol *FrameAllocSym = - MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName(), - IdxVal); + MF.getMMI().getContext().getOrCreateFrameAllocSymbol( + GlobalValue::getRealLinkageName(Fn->getName()), IdxVal); // Create a TargetExternalSymbol for the label to avoid any target lowering // that would make this PC relative. @@ -5446,16 +5460,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_begincatch: case Intrinsic::eh_endcatch: llvm_unreachable("begin/end catch intrinsics not lowered in codegen"); - case Intrinsic::eh_unwindhelp: { - AllocaInst *Slot = - cast(I.getArgOperand(0)->stripPointerCasts()); - assert(FuncInfo.StaticAllocaMap.count(Slot) && - "can only use static allocas with llvm.eh.unwindhelp"); - int FI = FuncInfo.StaticAllocaMap[Slot]; - // TODO: Save this in the not-yet-existant WinEHFuncInfo struct. - (void)FI; - return nullptr; - } } } @@ -5546,6 +5550,11 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, // Skip the first return-type Attribute to get to params. Entry.setAttributes(&CS, i - CS.arg_begin() + 1); Args.push_back(Entry); + + // If we have an explicit sret argument that is an Instruction, (i.e., it + // might point to function-local memory), we can't meaningfully tail-call. + if (Entry.isSRet && isa(V)) + isTailCall = false; } // Check if target-independent constraints permit a tail call here. @@ -7183,6 +7192,10 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { Entry.Alignment = Align; CLI.getArgs().insert(CLI.getArgs().begin(), Entry); CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext()); + + // sret demotion isn't compatible with tail-calls, since the sret argument + // points into the callers stack frame. + CLI.IsTailCall = false; } else { for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { EVT VT = RetTys[I]; @@ -7790,3 +7803,17 @@ MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) { return nullptr; return I; } + +/// During lowering new call nodes can be created (such as memset, etc.). +/// Those will become new roots of the current DAG, but complications arise +/// when they are tail calls. In such cases, the call lowering will update +/// the root, but the builder still needs to know that a tail call has been +/// lowered in order to avoid generating an additional return. +void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) { + // If the node is null, we do have a tail call. + if (MaybeTC.getNode() != nullptr) + DAG.setRoot(MaybeTC); + else + HasTailCall = true; +} + diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 30240d8..a27f470 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -397,7 +397,6 @@ private: StackProtectorDescriptor() : ParentMBB(nullptr), SuccessMBB(nullptr), FailureMBB(nullptr), Guard(nullptr), GuardReg(0) { } - ~StackProtectorDescriptor() { } /// Returns true if all fields of the stack protector descriptor are /// initialized implying that we should/are ready to emit a stack protector. @@ -823,12 +822,17 @@ private: /// EmitFuncArgumentDbgValue - If V is an function argument then create /// corresponding DBG_VALUE machine instruction for it now. At the end of /// instruction selection, they will be inserted to the entry BB. - bool EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable, MDNode *Expr, + bool EmitFuncArgumentDbgValue(const Value *V, MDLocalVariable *Variable, + MDExpression *Expr, MDLocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N); /// Return the next block after MBB, or nullptr if there is none. MachineBasicBlock *NextBlock(MachineBasicBlock *MBB); + + /// Update the DAG and DAG builder with the relevant information after + /// a new root node has been created which could be a tail call. + void updateDAGForMaybeTailCall(SDValue MaybeTC); }; } // end namespace llvm diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 5898da4..636c0a7 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -520,22 +520,20 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (getNodeId() != -1) OS << " [ID=" << getNodeId() << ']'; - DebugLoc dl = getDebugLoc(); - if (G && !dl.isUnknown()) { - DIScope - Scope(dl.getScope(G->getMachineFunction().getFunction()->getContext())); - OS << " dbg:"; - assert((!Scope || Scope.isScope()) && - "Scope of a DebugLoc should be null or a DIScope."); - // Omit the directory, since it's usually long and uninteresting. - if (Scope) - OS << Scope.getFilename(); - else - OS << ""; - OS << ':' << dl.getLine(); - if (dl.getCol() != 0) - OS << ':' << dl.getCol(); - } + if (!G) + return; + + MDLocation *L = getDebugLoc(); + if (!L) + return; + + if (auto *Scope = L->getScope()) + OS << Scope->getFilename(); + else + OS << ""; + OS << ':' << L->getLine(); + if (unsigned C = L->getColumn()) + OS << ':' << C; } static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 4d2af3f..1e116dd 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -33,6 +33,7 @@ #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Function.h" @@ -500,12 +501,14 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { MachineBasicBlock::iterator InsertPos = Def; const MDNode *Variable = MI->getDebugVariable(); const MDNode *Expr = MI->getDebugExpression(); + DebugLoc DL = MI->getDebugLoc(); bool IsIndirect = MI->isIndirectDebugValue(); unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0; + assert(cast(Variable)->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); // Def is never a terminator here, so it is ok to increment InsertPos. - BuildMI(*EntryMBB, ++InsertPos, MI->getDebugLoc(), - TII->get(TargetOpcode::DBG_VALUE), IsIndirect, LDI->second, Offset, - Variable, Expr); + BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE), + IsIndirect, LDI->second, Offset, Variable, Expr); // If this vreg is directly copied into an exported register then // that COPY instructions also need DBG_VALUE, if it is the only @@ -523,9 +526,10 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { CopyUseMI = nullptr; break; } if (CopyUseMI) { + // Use MI's debug location, which describes where Variable was + // declared, rather than whatever is attached to CopyUseMI. MachineInstr *NewMI = - BuildMI(*MF, CopyUseMI->getDebugLoc(), - TII->get(TargetOpcode::DBG_VALUE), IsIndirect, + BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect, CopyUseMI->getOperand(0).getReg(), Offset, Variable, Expr); MachineBasicBlock::iterator Pos = CopyUseMI; EntryMBB->insertAfter(Pos, NewMI); @@ -670,8 +674,8 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { #endif { BlockNumber = FuncInfo->MBB->getNumber(); - BlockName = MF->getName().str() + ":" + - FuncInfo->MBB->getBasicBlock()->getName().str(); + BlockName = + (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str(); } DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumber << " '" << BlockName << "'\n"; CurDAG->dump()); @@ -929,53 +933,74 @@ void SelectionDAGISel::PrepareEHLandingPad() { const LandingPadInst *LPadInst = LLVMBB->getLandingPadInst(); MF->getMMI().addPersonality( MBB, cast(LPadInst->getPersonalityFn()->stripPointerCasts())); - if (MF->getMMI().getPersonalityType() == EHPersonality::MSVC_Win64SEH) { - // Make virtual registers and a series of labels that fill in values for the - // clauses. - auto &RI = MF->getRegInfo(); - FuncInfo->ExceptionSelectorVirtReg = RI.createVirtualRegister(PtrRC); + EHPersonality Personality = MF->getMMI().getPersonalityType(); - // Get all invoke BBs that will unwind into the clause BBs. + if (isMSVCEHPersonality(Personality)) { + SmallVector ClauseBBs; + const IntrinsicInst *Actions = + dyn_cast(LLVMBB->getFirstInsertionPt()); + // Get all invoke BBs that unwind to this landingpad. SmallVector InvokeBBs(MBB->pred_begin(), MBB->pred_end()); + if (Actions && Actions->getIntrinsicID() == Intrinsic::eh_actions) { + // If this is a call to llvm.eh.actions followed by indirectbr, then we've + // run WinEHPrepare, and we should remove this block from the machine CFG. + // Mark the targets of the indirectbr as landingpads instead. + for (const BasicBlock *LLVMSucc : successors(LLVMBB)) { + MachineBasicBlock *ClauseBB = FuncInfo->MBBMap[LLVMSucc]; + // Add the edge from the invoke to the clause. + for (MachineBasicBlock *InvokeBB : InvokeBBs) + InvokeBB->addSuccessor(ClauseBB); + } + } else { + // Otherwise, we haven't done the preparation, and we need to invent some + // clause basic blocks that branch into the landingpad. + // FIXME: Remove this code once SEH preparation works. + + // Make virtual registers and a series of labels that fill in values for + // the clauses. + auto &RI = MF->getRegInfo(); + FuncInfo->ExceptionSelectorVirtReg = RI.createVirtualRegister(PtrRC); + + // Emit separate machine basic blocks with separate labels for each clause + // before the main landing pad block. + MachineInstrBuilder SelectorPHI = BuildMI( + *MBB, MBB->begin(), SDB->getCurDebugLoc(), + TII->get(TargetOpcode::PHI), FuncInfo->ExceptionSelectorVirtReg); + for (unsigned I = 0, E = LPadInst->getNumClauses(); I != E; ++I) { + // Skip filter clauses, we can't implement them. + if (LPadInst->isFilter(I)) + continue; - // Emit separate machine basic blocks with separate labels for each clause - // before the main landing pad block. - MachineInstrBuilder SelectorPHI = BuildMI( - *MBB, MBB->begin(), SDB->getCurDebugLoc(), TII->get(TargetOpcode::PHI), - FuncInfo->ExceptionSelectorVirtReg); - for (unsigned I = 0, E = LPadInst->getNumClauses(); I != E; ++I) { - // Skip filter clauses, we can't implement them yet. - if (LPadInst->isFilter(I)) - continue; - - MachineBasicBlock *ClauseBB = MF->CreateMachineBasicBlock(LLVMBB); - MF->insert(MBB, ClauseBB); + MachineBasicBlock *ClauseBB = MF->CreateMachineBasicBlock(LLVMBB); + MF->insert(MBB, ClauseBB); - // Add the edge from the invoke to the clause. - for (MachineBasicBlock *InvokeBB : InvokeBBs) - InvokeBB->addSuccessor(ClauseBB); + // Add the edge from the invoke to the clause. + for (MachineBasicBlock *InvokeBB : InvokeBBs) + InvokeBB->addSuccessor(ClauseBB); - // Mark the clause as a landing pad or MI passes will delete it. - ClauseBB->setIsLandingPad(); + // Mark the clause as a landing pad or MI passes will delete it. + ClauseBB->setIsLandingPad(); - GlobalValue *ClauseGV = ExtractTypeInfo(LPadInst->getClause(I)); + GlobalValue *ClauseGV = ExtractTypeInfo(LPadInst->getClause(I)); - // Start the BB with a label. - MCSymbol *ClauseLabel = MF->getMMI().addClauseForLandingPad(MBB); - BuildMI(*ClauseBB, ClauseBB->begin(), SDB->getCurDebugLoc(), II) - .addSym(ClauseLabel); + // Start the BB with a label. + MCSymbol *ClauseLabel = MF->getMMI().addClauseForLandingPad(MBB); + BuildMI(*ClauseBB, ClauseBB->begin(), SDB->getCurDebugLoc(), II) + .addSym(ClauseLabel); - // Construct a simple BB that defines a register with the typeid constant. - FuncInfo->MBB = ClauseBB; - FuncInfo->InsertPt = ClauseBB->end(); - unsigned VReg = SDB->visitLandingPadClauseBB(ClauseGV, MBB); - CurDAG->setRoot(SDB->getRoot()); - SDB->clear(); - CodeGenAndEmitDAG(); + // Construct a simple BB that defines a register with the typeid + // constant. + FuncInfo->MBB = ClauseBB; + FuncInfo->InsertPt = ClauseBB->end(); + unsigned VReg = SDB->visitLandingPadClauseBB(ClauseGV, MBB); + CurDAG->setRoot(SDB->getRoot()); + SDB->clear(); + CodeGenAndEmitDAG(); - // Add the typeid virtual register to the phi in the main landing pad. - SelectorPHI.addReg(VReg).addMBB(ClauseBB); + // Add the typeid virtual register to the phi in the main landing pad. + SelectorPHI.addReg(VReg).addMBB(ClauseBB); + } } // Remove the edge from the invoke to the lpad. @@ -986,6 +1011,12 @@ void SelectionDAGISel::PrepareEHLandingPad() { // pad block. FuncInfo->MBB = MBB; FuncInfo->InsertPt = MBB->end(); + + // Transfer EH state number assigned to the IR block to the MBB. + if (Personality == EHPersonality::MSVC_CXX) { + WinEHFuncInfo &FI = MF->getMMI().getWinEHFuncInfo(MF->getFunction()); + MF->getMMI().addWinEHState(MBB, FI.LandingPadStateMap[LPadInst]); + } return; } @@ -1165,7 +1196,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Setup an EH landing-pad block. FuncInfo->ExceptionPointerVirtReg = 0; FuncInfo->ExceptionSelectorVirtReg = 0; - if (FuncInfo->MBB->isLandingPad()) + if (LLVMBB->isLandingPad()) PrepareEHLandingPad(); // Before doing SelectionDAG ISel, see if FastISel has been requested. @@ -2552,7 +2583,7 @@ public: SelectionDAG::DAGUpdateListener(DAG), RecordedNodes(RN), MatchScopes(MS) { } - void NodeDeleted(SDNode *N, SDNode *E) { + void NodeDeleted(SDNode *N, SDNode *E) override { // Some early-returns here to avoid the search if we deleted the node or // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we // do, so it's unnecessary to update matching state at that point). diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 3cc7a98..a9ffa72 100644 --- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -397,10 +397,11 @@ static void lowerIncomingStatepointValue(SDValue Incoming, Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); Ops.push_back(Builder.DAG.getTargetConstant(C->getSExtValue(), MVT::i64)); } else if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { - // This handles allocas as arguments to the statepoint - const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo(); - Ops.push_back( - Builder.DAG.getTargetFrameIndex(FI->getIndex(), TLI.getPointerTy())); + // This handles allocas as arguments to the statepoint (this is only + // really meaningful for a deopt value. For GC, we'd be trying to + // relocate the address of the alloca itself?) + Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), + Incoming.getValueType())); } else { // Otherwise, locate a spill slot and explicitly spill it so it // can be found by the runtime later. We currently do not support @@ -441,27 +442,25 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, // heap. This is basically just here to help catch errors during statepoint // insertion. TODO: This should actually be in the Verifier, but we can't get // to the GCStrategy from there (yet). - if (Builder.GFI) { - GCStrategy &S = Builder.GFI->getStrategy(); - for (const Value *V : Bases) { - auto Opt = S.isGCManagedPointer(V); - if (Opt.hasValue()) { - assert(Opt.getValue() && - "non gc managed base pointer found in statepoint"); - } + GCStrategy &S = Builder.GFI->getStrategy(); + for (const Value *V : Bases) { + auto Opt = S.isGCManagedPointer(V); + if (Opt.hasValue()) { + assert(Opt.getValue() && + "non gc managed base pointer found in statepoint"); } - for (const Value *V : Ptrs) { - auto Opt = S.isGCManagedPointer(V); - if (Opt.hasValue()) { - assert(Opt.getValue() && - "non gc managed derived pointer found in statepoint"); - } + } + for (const Value *V : Ptrs) { + auto Opt = S.isGCManagedPointer(V); + if (Opt.hasValue()) { + assert(Opt.getValue() && + "non gc managed derived pointer found in statepoint"); } - for (const Value *V : Relocations) { - auto Opt = S.isGCManagedPointer(V); - if (Opt.hasValue()) { - assert(Opt.getValue() && "non gc managed pointer relocated"); - } + } + for (const Value *V : Relocations) { + auto Opt = S.isGCManagedPointer(V); + if (Opt.hasValue()) { + assert(Opt.getValue() && "non gc managed pointer relocated"); } } #endif @@ -523,6 +522,21 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, SDValue Incoming = Builder.getValue(V); lowerIncomingStatepointValue(Incoming, Ops, Builder); } + + // If there are any explicit spill slots passed to the statepoint, record + // them, but otherwise do not do anything special. These are user provided + // allocas and give control over placement to the consumer. In this case, + // it is the contents of the slot which may get updated, not the pointer to + // the alloca + for (Value *V : StatepointSite.gc_args()) { + SDValue Incoming = Builder.getValue(V); + if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { + // This handles allocas as arguments to the statepoint + Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), + Incoming.getValueType())); + + } + } } void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) { @@ -565,10 +579,8 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, // TODO: This if should become an assert. For now, we allow the GCStrategy // to be optional for backwards compatibility. This will only last a short // period (i.e. a couple of weeks). - if (GFI) { - assert(GFI->getStrategy().useStatepoints() && - "GCStrategy does not expect to encounter statepoints"); - } + assert(GFI->getStrategy().useStatepoints() && + "GCStrategy does not expect to encounter statepoints"); #endif // Lower statepoint vmstate and gcstate arguments @@ -614,7 +626,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, // Add a leading constant argument with the Flags and the calling convention // masked together CallingConv::ID CallConv = CS.getCallingConv(); - int Flags = dyn_cast(CS.getArgument(2))->getZExtValue(); + int Flags = cast(CS.getArgument(2))->getZExtValue(); assert(Flags == 0 && "not expected to be used"); Ops.push_back(DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); Ops.push_back( diff --git a/lib/CodeGen/ShadowStackGCLowering.cpp b/lib/CodeGen/ShadowStackGCLowering.cpp index 66a6a3c..7c0b2bb 100644 --- a/lib/CodeGen/ShadowStackGCLowering.cpp +++ b/lib/CodeGen/ShadowStackGCLowering.cpp @@ -239,7 +239,7 @@ Constant *ShadowStackGCLowering::GetFrameMap(Function &F) { Constant *GEPIndices[2] = { ConstantInt::get(Type::getInt32Ty(F.getContext()), 0), ConstantInt::get(Type::getInt32Ty(F.getContext()), 0)}; - return ConstantExpr::getGetElementPtr(GV, GEPIndices); + return ConstantExpr::getGetElementPtr(FrameMap->getType(), GV, GEPIndices); } Type *ShadowStackGCLowering::GetConcreteStackEntryType(Function &F) { @@ -249,7 +249,7 @@ Type *ShadowStackGCLowering::GetConcreteStackEntryType(Function &F) { for (size_t I = 0; I != Roots.size(); I++) EltTys.push_back(Roots[I].second->getAllocatedType()); - return StructType::create(EltTys, "gc_stackentry." + F.getName().str()); + return StructType::create(EltTys, ("gc_stackentry." + F.getName()).str()); } /// doInitialization - If this module uses the GC intrinsics, find them now. If diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp index 2335a88..0635173 100644 --- a/lib/CodeGen/SjLjEHPrepare.cpp +++ b/lib/CodeGen/SjLjEHPrepare.cpp @@ -46,6 +46,8 @@ STATISTIC(NumSpilled, "Number of registers live across unwind edges"); namespace { class SjLjEHPrepare : public FunctionPass { const TargetMachine *TM; + Type *doubleUnderDataTy; + Type *doubleUnderJBufTy; Type *FunctionContextTy; Constant *RegisterFn; Constant *UnregisterFn; @@ -93,12 +95,14 @@ bool SjLjEHPrepare::doInitialization(Module &M) { // builtin_setjmp uses a five word jbuf Type *VoidPtrTy = Type::getInt8PtrTy(M.getContext()); Type *Int32Ty = Type::getInt32Ty(M.getContext()); - FunctionContextTy = StructType::get(VoidPtrTy, // __prev - Int32Ty, // call_site - ArrayType::get(Int32Ty, 4), // __data - VoidPtrTy, // __personality - VoidPtrTy, // __lsda - ArrayType::get(VoidPtrTy, 5), // __jbuf + doubleUnderDataTy = ArrayType::get(Int32Ty, 4); + doubleUnderJBufTy = ArrayType::get(VoidPtrTy, 5); + FunctionContextTy = StructType::get(VoidPtrTy, // __prev + Int32Ty, // call_site + doubleUnderDataTy, // __data + VoidPtrTy, // __personality + VoidPtrTy, // __lsda + doubleUnderJBufTy, // __jbuf nullptr); RegisterFn = M.getOrInsertFunction( "_Unwind_SjLj_Register", Type::getVoidTy(M.getContext()), @@ -204,16 +208,17 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F, IRBuilder<> Builder(LPI->getParent()->getFirstInsertionPt()); // Reference the __data field. - Value *FCData = Builder.CreateConstGEP2_32(FuncCtx, 0, 2, "__data"); + Value *FCData = + Builder.CreateConstGEP2_32(FunctionContextTy, FuncCtx, 0, 2, "__data"); // The exception values come back in context->__data[0]. - Value *ExceptionAddr = - Builder.CreateConstGEP2_32(FCData, 0, 0, "exception_gep"); + Value *ExceptionAddr = Builder.CreateConstGEP2_32(doubleUnderDataTy, FCData, + 0, 0, "exception_gep"); Value *ExnVal = Builder.CreateLoad(ExceptionAddr, true, "exn_val"); ExnVal = Builder.CreateIntToPtr(ExnVal, Builder.getInt8PtrTy()); - Value *SelectorAddr = - Builder.CreateConstGEP2_32(FCData, 0, 1, "exn_selector_gep"); + Value *SelectorAddr = Builder.CreateConstGEP2_32(doubleUnderDataTy, FCData, + 0, 1, "exn_selector_gep"); Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val"); substituteLPadValues(LPI, ExnVal, SelVal); @@ -223,15 +228,16 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F, IRBuilder<> Builder(EntryBB->getTerminator()); if (!PersonalityFn) PersonalityFn = LPads[0]->getPersonalityFn(); - Value *PersonalityFieldPtr = - Builder.CreateConstGEP2_32(FuncCtx, 0, 3, "pers_fn_gep"); + Value *PersonalityFieldPtr = Builder.CreateConstGEP2_32( + FunctionContextTy, FuncCtx, 0, 3, "pers_fn_gep"); Builder.CreateStore( Builder.CreateBitCast(PersonalityFn, Builder.getInt8PtrTy()), PersonalityFieldPtr, /*isVolatile=*/true); // LSDA address Value *LSDA = Builder.CreateCall(LSDAAddrFn, "lsda_addr"); - Value *LSDAFieldPtr = Builder.CreateConstGEP2_32(FuncCtx, 0, 4, "lsda_gep"); + Value *LSDAFieldPtr = + Builder.CreateConstGEP2_32(FunctionContextTy, FuncCtx, 0, 4, "lsda_gep"); Builder.CreateStore(LSDA, LSDAFieldPtr, /*isVolatile=*/true); return FuncCtx; @@ -400,16 +406,19 @@ bool SjLjEHPrepare::setupEntryBlockAndCallSites(Function &F) { IRBuilder<> Builder(EntryBB->getTerminator()); // Get a reference to the jump buffer. - Value *JBufPtr = Builder.CreateConstGEP2_32(FuncCtx, 0, 5, "jbuf_gep"); + Value *JBufPtr = + Builder.CreateConstGEP2_32(FunctionContextTy, FuncCtx, 0, 5, "jbuf_gep"); // Save the frame pointer. - Value *FramePtr = Builder.CreateConstGEP2_32(JBufPtr, 0, 0, "jbuf_fp_gep"); + Value *FramePtr = Builder.CreateConstGEP2_32(doubleUnderJBufTy, JBufPtr, 0, 0, + "jbuf_fp_gep"); Value *Val = Builder.CreateCall(FrameAddrFn, Builder.getInt32(0), "fp"); Builder.CreateStore(Val, FramePtr, /*isVolatile=*/true); // Save the stack pointer. - Value *StackPtr = Builder.CreateConstGEP2_32(JBufPtr, 0, 2, "jbuf_sp_gep"); + Value *StackPtr = Builder.CreateConstGEP2_32(doubleUnderJBufTy, JBufPtr, 0, 2, + "jbuf_sp_gep"); Val = Builder.CreateCall(StackAddrFn, "sp"); Builder.CreateStore(Val, StackPtr, /*isVolatile=*/true); diff --git a/lib/CodeGen/SpillPlacement.h b/lib/CodeGen/SpillPlacement.h index 622361e..03dd58d 100644 --- a/lib/CodeGen/SpillPlacement.h +++ b/lib/CodeGen/SpillPlacement.h @@ -70,7 +70,7 @@ public: static char ID; // Pass identification, replacement for typeid. SpillPlacement() : MachineFunctionPass(ID), nodes(nullptr) {} - ~SpillPlacement() { releaseMemory(); } + ~SpillPlacement() override { releaseMemory(); } /// BorderConstraint - A basic block has separate constraints for entry and /// exit. diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp index 7572803..2bf2d64 100644 --- a/lib/CodeGen/StackColoring.cpp +++ b/lib/CodeGen/StackColoring.cpp @@ -464,7 +464,7 @@ void StackColoring::remapInstructions(DenseMap &SlotRemap) { continue; if (SlotRemap.count(VI.Slot)) { DEBUG(dbgs() << "Remapping debug info for [" - << DIVariable(VI.Var).getName() << "].\n"); + << cast(VI.Var)->getName() << "].\n"); VI.Slot = SlotRemap[VI.Slot]; FixedDbg++; } diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index 58a6d52..2162a51 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -1246,20 +1246,13 @@ void TargetLoweringBase::computeRegisterProperties( ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat); } - // Decide how to handle f32. If the target does not have native support for - // f32, promote it to f64 if it is legal. Otherwise, expand it to i32. + // Decide how to handle f32. If the target does not have native f32 support, + // expand it to i32 and we will be generating soft float library calls. if (!isTypeLegal(MVT::f32)) { - if (isTypeLegal(MVT::f64)) { - NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::f64]; - RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::f64]; - TransformToType[MVT::f32] = MVT::f64; - ValueTypeActions.setTypeAction(MVT::f32, TypePromoteInteger); - } else { - NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; - RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; - TransformToType[MVT::f32] = MVT::i32; - ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); - } + NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; + RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; + TransformToType[MVT::f32] = MVT::i32; + ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); } if (!isTypeLegal(MVT::f16)) { diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index bcf2aa7..5b795e4 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -245,9 +245,11 @@ static StringRef getSectionPrefixForGlobal(SectionKind Kind) { return ".data.rel.ro"; } -static const MCSectionELF *selectELFSectionForGlobal( - MCContext &Ctx, const GlobalValue *GV, SectionKind Kind, Mangler &Mang, - const TargetMachine &TM, bool EmitUniqueSection, unsigned Flags) { +static const MCSectionELF * +selectELFSectionForGlobal(MCContext &Ctx, const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM, bool EmitUniqueSection, + unsigned Flags, unsigned *NextUniqueID) { unsigned EntrySize = 0; if (Kind.isMergeableCString()) { if (Kind.isMergeable2ByteCString()) { @@ -297,9 +299,13 @@ static const MCSectionELF *selectELFSectionForGlobal( Name.push_back('.'); TM.getNameWithPrefix(Name, GV, Mang, true); } + unsigned UniqueID = ~0; + if (EmitUniqueSection && !UniqueSectionNames) { + UniqueID = *NextUniqueID; + (*NextUniqueID)++; + } return Ctx.getELFSection(Name, getELFSectionType(Name, Kind), Flags, - EntrySize, Group, - EmitUniqueSection && !UniqueSectionNames); + EntrySize, Group, UniqueID); } const MCSection *TargetLoweringObjectFileELF::SelectSectionForGlobal( @@ -319,7 +325,7 @@ const MCSection *TargetLoweringObjectFileELF::SelectSectionForGlobal( EmitUniqueSection |= GV->hasComdat(); return selectELFSectionForGlobal(getContext(), GV, Kind, Mang, TM, - EmitUniqueSection, Flags); + EmitUniqueSection, Flags, &NextUniqueID); } const MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable( @@ -332,7 +338,8 @@ const MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable( return ReadOnlySection; return selectELFSectionForGlobal(getContext(), &F, SectionKind::getReadOnly(), - Mang, TM, EmitUniqueSection, ELF::SHF_ALLOC); + Mang, TM, EmitUniqueSection, ELF::SHF_ALLOC, + &NextUniqueID); } bool TargetLoweringObjectFileELF::shouldPutJumpTableInFunctionSection( diff --git a/lib/CodeGen/WinEHPrepare.cpp b/lib/CodeGen/WinEHPrepare.cpp index ab0f96e..35b944e 100644 --- a/lib/CodeGen/WinEHPrepare.cpp +++ b/lib/CodeGen/WinEHPrepare.cpp @@ -20,6 +20,8 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/LibCallSemantics.h" +#include "llvm/CodeGen/WinEHFuncInfo.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" @@ -33,6 +35,7 @@ #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/PromoteMemToReg.h" #include using namespace llvm; @@ -49,14 +52,15 @@ namespace { // frame allocation structure. typedef MapVector> FrameVarInfoMap; -typedef SmallSet VisitedBlockSet; +// TinyPtrVector cannot hold nullptr, so we need our own sentinel that isn't +// quite null. +AllocaInst *getCatchObjectSentinel() { + return static_cast(nullptr) + 1; +} -enum ActionType { Catch, Cleanup }; +typedef SmallSet VisitedBlockSet; class LandingPadActions; -class ActionHandler; -class CatchHandler; -class CleanupHandler; class LandingPadMap; typedef DenseMap CatchHandlerMapTy; @@ -66,7 +70,7 @@ class WinEHPrepare : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid. WinEHPrepare(const TargetMachine *TM = nullptr) - : FunctionPass(ID) {} + : FunctionPass(ID), DT(nullptr) {} bool runOnFunction(Function &Fn) override; @@ -81,31 +85,62 @@ public: private: bool prepareExceptionHandlers(Function &F, SmallVectorImpl &LPads); + void promoteLandingPadValues(LandingPadInst *LPad); + void completeNestedLandingPad(Function *ParentFn, + LandingPadInst *OutlinedLPad, + const LandingPadInst *OriginalLPad, + FrameVarInfoMap &VarInfo); bool outlineHandler(ActionHandler *Action, Function *SrcFn, LandingPadInst *LPad, BasicBlock *StartBB, FrameVarInfoMap &VarInfo); + void addStubInvokeToHandlerIfNeeded(Function *Handler, Value *PersonalityFn); void mapLandingPadBlocks(LandingPadInst *LPad, LandingPadActions &Actions); CatchHandler *findCatchHandler(BasicBlock *BB, BasicBlock *&NextBB, VisitedBlockSet &VisitedBlocks); - CleanupHandler *findCleanupHandler(BasicBlock *StartBB, BasicBlock *EndBB); + void findCleanupHandlers(LandingPadActions &Actions, BasicBlock *StartBB, + BasicBlock *EndBB); void processSEHCatchHandler(CatchHandler *Handler, BasicBlock *StartBB); // All fields are reset by runOnFunction. + DominatorTree *DT; EHPersonality Personality; CatchHandlerMapTy CatchHandlerMap; CleanupHandlerMapTy CleanupHandlerMap; - DenseMap LPadMaps; + DenseMap LPadMaps; + + // This maps landing pad instructions found in outlined handlers to + // the landing pad instruction in the parent function from which they + // were cloned. The cloned/nested landing pad is used as the key + // because the landing pad may be cloned into multiple handlers. + // This map will be used to add the llvm.eh.actions call to the nested + // landing pads after all handlers have been outlined. + DenseMap NestedLPtoOriginalLP; + + // This maps blocks in the parent function which are destinations of + // catch handlers to cloned blocks in (other) outlined handlers. This + // handles the case where a nested landing pads has a catch handler that + // returns to a handler function rather than the parent function. + // The original block is used as the key here because there should only + // ever be one handler function from which the cloned block is not pruned. + // The original block will be pruned from the parent function after all + // handlers have been outlined. This map will be used to adjust the + // return instructions of handlers which return to the block that was + // outlined into a handler. This is done after all handlers have been + // outlined but before the outlined code is pruned from the parent function. + DenseMap LPadTargetBlocks; }; class WinEHFrameVariableMaterializer : public ValueMaterializer { public: WinEHFrameVariableMaterializer(Function *OutlinedFn, FrameVarInfoMap &FrameVarInfo); - ~WinEHFrameVariableMaterializer() {} + ~WinEHFrameVariableMaterializer() override {} + + Value *materializeValueFor(Value *V) override; - virtual Value *materializeValueFor(Value *V) override; + void escapeCatchObject(Value *V); private: FrameVarInfoMap &FrameVarInfo; @@ -119,42 +154,23 @@ public: bool isInitialized() { return OriginLPad != nullptr; } - bool mapIfEHPtrLoad(const LoadInst *Load) { - return mapIfEHLoad(Load, EHPtrStores, EHPtrStoreAddrs); - } - bool mapIfSelectorLoad(const LoadInst *Load) { - return mapIfEHLoad(Load, SelectorStores, SelectorStoreAddrs); - } - + bool isOriginLandingPadBlock(const BasicBlock *BB) const; bool isLandingPadSpecificInst(const Instruction *Inst) const; - void remapSelector(ValueToValueMapTy &VMap, Value *MappedValue) const; + void remapEHValues(ValueToValueMapTy &VMap, Value *EHPtrValue, + Value *SelectorValue) const; private: - bool mapIfEHLoad(const LoadInst *Load, - SmallVectorImpl &Stores, - SmallVectorImpl &StoreAddrs); - const LandingPadInst *OriginLPad; // We will normally only see one of each of these instructions, but // if more than one occurs for some reason we can handle that. TinyPtrVector ExtractedEHPtrs; TinyPtrVector ExtractedSelectors; - - // In optimized code, there will typically be at most one instance of - // each of the following, but in unoptimized IR it is not uncommon - // for the values to be stored, loaded and then stored again. In that - // case we will create a second entry for each store and store address. - SmallVector EHPtrStores; - SmallVector SelectorStores; - SmallVector EHPtrStoreAddrs; - SmallVector SelectorStoreAddrs; }; class WinEHCloningDirectorBase : public CloningDirector { public: - WinEHCloningDirectorBase(Function *HandlerFn, - FrameVarInfoMap &VarInfo, + WinEHCloningDirectorBase(Function *HandlerFn, FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap) : Materializer(HandlerFn, VarInfo), SelectorIDType(Type::getInt32Ty(HandlerFn->getContext())), @@ -180,6 +196,9 @@ public: virtual CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume, BasicBlock *NewBB) = 0; + virtual CloningAction handleLandingPad(ValueToValueMapTy &VMap, + const LandingPadInst *LPad, + BasicBlock *NewBB) = 0; ValueMaterializer *getValueMaterializer() override { return &Materializer; } @@ -192,11 +211,13 @@ protected: class WinEHCatchDirector : public WinEHCloningDirectorBase { public: - WinEHCatchDirector(Function *CatchFn, Value *Selector, - FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap) + WinEHCatchDirector( + Function *CatchFn, Value *Selector, FrameVarInfoMap &VarInfo, + LandingPadMap &LPadMap, + DenseMap &NestedLPads) : WinEHCloningDirectorBase(CatchFn, VarInfo, LPadMap), CurrentSelector(Selector->stripPointerCasts()), - ExceptionObjectVar(nullptr) {} + ExceptionObjectVar(nullptr), NestedLPtoOriginalLP(NestedLPads) {} CloningAction handleBeginCatch(ValueToValueMapTy &VMap, const Instruction *Inst, @@ -210,21 +231,28 @@ public: BasicBlock *NewBB) override; CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume, BasicBlock *NewBB) override; + CloningAction handleLandingPad(ValueToValueMapTy &VMap, + const LandingPadInst *LPad, + BasicBlock *NewBB) override; - const Value *getExceptionVar() { return ExceptionObjectVar; } + Value *getExceptionVar() { return ExceptionObjectVar; } TinyPtrVector &getReturnTargets() { return ReturnTargets; } private: Value *CurrentSelector; - const Value *ExceptionObjectVar; + Value *ExceptionObjectVar; TinyPtrVector ReturnTargets; + + // This will be a reference to the field of the same name in the WinEHPrepare + // object which instantiates this WinEHCatchDirector object. + DenseMap &NestedLPtoOriginalLP; }; class WinEHCleanupDirector : public WinEHCloningDirectorBase { public: - WinEHCleanupDirector(Function *CleanupFn, - FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap) + WinEHCleanupDirector(Function *CleanupFn, FrameVarInfoMap &VarInfo, + LandingPadMap &LPadMap) : WinEHCloningDirectorBase(CleanupFn, VarInfo, LPadMap) {} CloningAction handleBeginCatch(ValueToValueMapTy &VMap, @@ -239,66 +267,9 @@ public: BasicBlock *NewBB) override; CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume, BasicBlock *NewBB) override; -}; - -class ActionHandler { -public: - ActionHandler(BasicBlock *BB, ActionType Type) - : StartBB(BB), Type(Type), HandlerBlockOrFunc(nullptr) {} - - ActionType getType() const { return Type; } - BasicBlock *getStartBlock() const { return StartBB; } - - bool hasBeenProcessed() { return HandlerBlockOrFunc != nullptr; } - - void setHandlerBlockOrFunc(Constant *F) { HandlerBlockOrFunc = F; } - Constant *getHandlerBlockOrFunc() { return HandlerBlockOrFunc; } - -private: - BasicBlock *StartBB; - ActionType Type; - - // Can be either a BlockAddress or a Function depending on the EH personality. - Constant *HandlerBlockOrFunc; -}; - -class CatchHandler : public ActionHandler { -public: - CatchHandler(BasicBlock *BB, Constant *Selector, BasicBlock *NextBB) - : ActionHandler(BB, ActionType::Catch), Selector(Selector), - NextBB(NextBB), ExceptionObjectVar(nullptr) {} - - // Method for support type inquiry through isa, cast, and dyn_cast: - static inline bool classof(const ActionHandler *H) { - return H->getType() == ActionType::Catch; - } - - Constant *getSelector() const { return Selector; } - BasicBlock *getNextBB() const { return NextBB; } - - const Value *getExceptionVar() { return ExceptionObjectVar; } - TinyPtrVector &getReturnTargets() { return ReturnTargets; } - - void setExceptionVar(const Value *Val) { ExceptionObjectVar = Val; } - void setReturnTargets(TinyPtrVector &Targets) { - ReturnTargets = Targets; - } - -private: - Constant *Selector; - BasicBlock *NextBB; - const Value *ExceptionObjectVar; - TinyPtrVector ReturnTargets; -}; - -class CleanupHandler : public ActionHandler { -public: - CleanupHandler(BasicBlock *BB) : ActionHandler(BB, ActionType::Cleanup) {} - - // Method for support type inquiry through isa, cast, and dyn_cast: - static inline bool classof(const ActionHandler *H) { - return H->getType() == ActionType::Cleanup; - } + CloningAction handleLandingPad(ValueToValueMapTy &VMap, + const LandingPadInst *LPad, + BasicBlock *NewBB) override; }; class LandingPadActions { @@ -313,6 +284,7 @@ public: bool includesCleanup() const { return HasCleanupHandlers; } + SmallVectorImpl &actions() { return Actions; } SmallVectorImpl::iterator begin() { return Actions.begin(); } SmallVectorImpl::iterator end() { return Actions.end(); } @@ -336,8 +308,8 @@ FunctionPass *llvm::createWinEHPass(const TargetMachine *TM) { // FIXME: Remove this once the backend can handle the prepared IR. static cl::opt -SEHPrepare("sehprepare", cl::Hidden, - cl::desc("Prepare functions with SEH personalities")); + SEHPrepare("sehprepare", cl::Hidden, + cl::desc("Prepare functions with SEH personalities")); bool WinEHPrepare::runOnFunction(Function &Fn) { SmallVector LPads; @@ -360,6 +332,8 @@ bool WinEHPrepare::runOnFunction(Function &Fn) { if (!isMSVCEHPersonality(Personality)) return false; + DT = &getAnalysis().getDomTree(); + if (isAsynchronousEHPersonality(Personality) && !SEHPrepare) { // Replace all resume instructions with unreachable. // FIXME: Remove this once the backend can handle the prepared IR. @@ -375,11 +349,11 @@ bool WinEHPrepare::runOnFunction(Function &Fn) { return true; } -bool WinEHPrepare::doFinalization(Module &M) { - return false; -} +bool WinEHPrepare::doFinalization(Module &M) { return false; } -void WinEHPrepare::getAnalysisUsage(AnalysisUsage &AU) const {} +void WinEHPrepare::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); +} bool WinEHPrepare::prepareExceptionHandlers( Function &F, SmallVectorImpl &LPads) { @@ -422,9 +396,14 @@ bool WinEHPrepare::prepareExceptionHandlers( if (LPadHasActionList) continue; + // If either of the values in the aggregate returned by the landing pad is + // extracted and stored to memory, promote the stored value to a register. + promoteLandingPadValues(LPad); + LandingPadActions Actions; mapLandingPadBlocks(LPad, Actions); + HandlersOutlined |= !Actions.actions().empty(); for (ActionHandler *Action : Actions) { if (Action->hasBeenProcessed()) continue; @@ -436,24 +415,17 @@ bool WinEHPrepare::prepareExceptionHandlers( if (isAsynchronousEHPersonality(Personality)) { if (auto *CatchAction = dyn_cast(Action)) { processSEHCatchHandler(CatchAction, StartBB); - HandlersOutlined = true; continue; } } - if (outlineHandler(Action, &F, LPad, StartBB, FrameVarInfo)) { - HandlersOutlined = true; - } - } // End for each Action - - // FIXME: We need a guard against partially outlined functions. - if (!HandlersOutlined) - continue; + outlineHandler(Action, &F, LPad, StartBB, FrameVarInfo); + } // Replace the landing pad with a new llvm.eh.action based landing pad. BasicBlock *NewLPadBB = BasicBlock::Create(Context, "lpad", &F, LPadBB); assert(!isa(LPadBB->begin())); - Instruction *NewLPad = LPad->clone(); + auto *NewLPad = cast(LPad->clone()); NewLPadBB->getInstList().push_back(NewLPad); while (!pred_empty(LPadBB)) { auto *pred = *pred_begin(LPadBB); @@ -461,6 +433,19 @@ bool WinEHPrepare::prepareExceptionHandlers( Invoke->setUnwindDest(NewLPadBB); } + // If anyone is still using the old landingpad value, just give them undef + // instead. The eh pointer and selector values are not real. + LPad->replaceAllUsesWith(UndefValue::get(LPad->getType())); + + // Replace the mapping of any nested landing pad that previously mapped + // to this landing pad with a referenced to the cloned version. + for (auto &LPadPair : NestedLPtoOriginalLP) { + const LandingPadInst *OriginalLPad = LPadPair.second; + if (OriginalLPad == LPad) { + LPadPair.second = NewLPad; + } + } + // Replace uses of the old lpad in phis with this block and delete the old // block. LPadBB->replaceSuccessorsPhiUsesWith(NewLPadBB); @@ -474,11 +459,17 @@ bool WinEHPrepare::prepareExceptionHandlers( if (auto *CatchAction = dyn_cast(Action)) { ActionArgs.push_back(ConstantInt::get(Int32Type, 1)); ActionArgs.push_back(CatchAction->getSelector()); + // Find the frame escape index of the exception object alloca in the + // parent. + int FrameEscapeIdx = -1; Value *EHObj = const_cast(CatchAction->getExceptionVar()); - if (EHObj) - ActionArgs.push_back(EHObj); - else - ActionArgs.push_back(ConstantPointerNull::get(Int8PtrType)); + if (EHObj && !isa(EHObj)) { + auto I = FrameVarInfo.find(EHObj); + assert(I != FrameVarInfo.end() && + "failed to map llvm.eh.begincatch var"); + FrameEscapeIdx = std::distance(FrameVarInfo.begin(), I); + } + ActionArgs.push_back(ConstantInt::get(Int32Type, FrameEscapeIdx)); } else { ActionArgs.push_back(ConstantInt::get(Int32Type, 0)); } @@ -502,6 +493,15 @@ bool WinEHPrepare::prepareExceptionHandlers( if (!HandlersOutlined) return false; + // Replace any nested landing pad stubs with the correct action handler. + // This must be done before we remove unreachable blocks because it + // cleans up references to outlined blocks that will be deleted. + for (auto &LPadPair : NestedLPtoOriginalLP) + completeNestedLandingPad(&F, LPadPair.first, LPadPair.second, FrameVarInfo); + NestedLPtoOriginalLP.clear(); + + F.addFnAttr("wineh-parent", F.getName()); + // Delete any blocks that were only used by handlers that were outlined above. removeUnreachableBlocks(F); @@ -554,26 +554,26 @@ bool WinEHPrepare::prepareExceptionHandlers( ++InsertPt; ParentAlloca = new AllocaInst(ParentInst->getType(), nullptr, - ParentInst->getName() + ".reg2mem", InsertPt); + ParentInst->getName() + ".reg2mem", + AllocaInsertPt); new StoreInst(ParentInst, ParentAlloca, InsertPt); } else { - ParentAlloca = DemoteRegToStack(*ParentInst, true, ParentInst); + ParentAlloca = DemoteRegToStack(*ParentInst, true, AllocaInsertPt); } } } - // If the parent alloca is no longer used and only one of the handlers used - // it, erase the parent and leave the copy in the outlined handler. - if (ParentAlloca->getNumUses() == 0 && Allocas.size() == 1) { - ParentAlloca->eraseFromParent(); - continue; - } + // FIXME: We should try to sink unescaped allocas from the parent frame into + // the child frame. If the alloca is escaped, we have to use the lifetime + // markers to ensure that the alloca is only live within the child frame. // Add this alloca to the list of things to escape. AllocasToEscape.push_back(ParentAlloca); // Next replace all outlined allocas that are mapped to it. for (AllocaInst *TempAlloca : Allocas) { + if (TempAlloca == getCatchObjectSentinel()) + continue; // Skip catch parameter sentinels. Function *HandlerFn = TempAlloca->getParent()->getParent(); // FIXME: Sink this GEP into the blocks where it is used. Builder.SetInsertPoint(TempAlloca); @@ -601,19 +601,6 @@ bool WinEHPrepare::prepareExceptionHandlers( Builder.SetInsertPoint(&F.getEntryBlock().back()); Builder.CreateCall(FrameEscapeFn, AllocasToEscape); - // Insert an alloca for the EH state in the entry block. On x86, we will also - // insert stores to update the EH state, but on other ISAs, the runtime does - // it for us. - // FIXME: This record is different on x86. - Type *UnwindHelpTy = Type::getInt64Ty(Context); - AllocaInst *UnwindHelp = - new AllocaInst(UnwindHelpTy, "unwindhelp", &F.getEntryBlock().front()); - Builder.CreateStore(llvm::ConstantInt::get(UnwindHelpTy, -2), UnwindHelp); - Function *UnwindHelpFn = - Intrinsic::getDeclaration(M, Intrinsic::eh_unwindhelp); - Builder.CreateCall(UnwindHelpFn, - Builder.CreateBitCast(UnwindHelp, Int8PtrType)); - // Clean up the handler action maps we created for this function DeleteContainerSeconds(CatchHandlerMap); CatchHandlerMap.clear(); @@ -623,6 +610,125 @@ bool WinEHPrepare::prepareExceptionHandlers( return HandlersOutlined; } +void WinEHPrepare::promoteLandingPadValues(LandingPadInst *LPad) { + // If the return values of the landing pad instruction are extracted and + // stored to memory, we want to promote the store locations to reg values. + SmallVector EHAllocas; + + // The landingpad instruction returns an aggregate value. Typically, its + // value will be passed to a pair of extract value instructions and the + // results of those extracts are often passed to store instructions. + // In unoptimized code the stored value will often be loaded and then stored + // again. + for (auto *U : LPad->users()) { + ExtractValueInst *Extract = dyn_cast(U); + if (!Extract) + continue; + + for (auto *EU : Extract->users()) { + if (auto *Store = dyn_cast(EU)) { + auto *AV = cast(Store->getPointerOperand()); + EHAllocas.push_back(AV); + } + } + } + + // We can't do this without a dominator tree. + assert(DT); + + if (!EHAllocas.empty()) { + PromoteMemToReg(EHAllocas, *DT); + EHAllocas.clear(); + } + + // After promotion, some extracts may be trivially dead. Remove them. + SmallVector Users(LPad->user_begin(), LPad->user_end()); + for (auto *U : Users) + RecursivelyDeleteTriviallyDeadInstructions(U); +} + +void WinEHPrepare::completeNestedLandingPad(Function *ParentFn, + LandingPadInst *OutlinedLPad, + const LandingPadInst *OriginalLPad, + FrameVarInfoMap &FrameVarInfo) { + // Get the nested block and erase the unreachable instruction that was + // temporarily inserted as its terminator. + LLVMContext &Context = ParentFn->getContext(); + BasicBlock *OutlinedBB = OutlinedLPad->getParent(); + assert(isa(OutlinedBB->getTerminator())); + OutlinedBB->getTerminator()->eraseFromParent(); + // That should leave OutlinedLPad as the last instruction in its block. + assert(&OutlinedBB->back() == OutlinedLPad); + + // The original landing pad will have already had its action intrinsic + // built by the outlining loop. We need to clone that into the outlined + // location. It may also be necessary to add references to the exception + // variables to the outlined handler in which this landing pad is nested + // and remap return instructions in the nested handlers that should return + // to an address in the outlined handler. + Function *OutlinedHandlerFn = OutlinedBB->getParent(); + BasicBlock::const_iterator II = OriginalLPad; + ++II; + // The instruction after the landing pad should now be a call to eh.actions. + const Instruction *Recover = II; + assert(match(Recover, m_Intrinsic())); + IntrinsicInst *EHActions = cast(Recover->clone()); + + // Remap the exception variables into the outlined function. + WinEHFrameVariableMaterializer Materializer(OutlinedHandlerFn, FrameVarInfo); + SmallVector ActionTargets; + SmallVector ActionList; + parseEHActions(EHActions, ActionList); + for (auto *Action : ActionList) { + auto *Catch = dyn_cast(Action); + if (!Catch) + continue; + // The dyn_cast to function here selects C++ catch handlers and skips + // SEH catch handlers. + auto *Handler = dyn_cast(Catch->getHandlerBlockOrFunc()); + if (!Handler) + continue; + // Visit all the return instructions, looking for places that return + // to a location within OutlinedHandlerFn. + for (BasicBlock &NestedHandlerBB : *Handler) { + auto *Ret = dyn_cast(NestedHandlerBB.getTerminator()); + if (!Ret) + continue; + + // Handler functions must always return a block address. + BlockAddress *BA = cast(Ret->getReturnValue()); + // The original target will have been in the main parent function, + // but if it is the address of a block that has been outlined, it + // should be a block that was outlined into OutlinedHandlerFn. + assert(BA->getFunction() == ParentFn); + + // Ignore targets that aren't part of OutlinedHandlerFn. + if (!LPadTargetBlocks.count(BA->getBasicBlock())) + continue; + + // If the return value is the address ofF a block that we + // previously outlined into the parent handler function, replace + // the return instruction and add the mapped target to the list + // of possible return addresses. + BasicBlock *MappedBB = LPadTargetBlocks[BA->getBasicBlock()]; + assert(MappedBB->getParent() == OutlinedHandlerFn); + BlockAddress *NewBA = BlockAddress::get(OutlinedHandlerFn, MappedBB); + Ret->eraseFromParent(); + ReturnInst::Create(Context, NewBA, &NestedHandlerBB); + ActionTargets.push_back(NewBA); + } + } + DeleteContainerPointers(ActionList); + ActionList.clear(); + OutlinedBB->getInstList().push_back(EHActions); + + // Insert an indirect branch into the outlined landing pad BB. + IndirectBrInst *IBr = IndirectBrInst::Create(EHActions, 0, OutlinedBB); + // Add the previously collected action targets. + for (auto *Target : ActionTargets) + IBr->addDestination(Target->getBasicBlock()); +} + // This function examines a block to determine whether the block ends with a // conditional branch to a catch handler based on a selector comparison. // This function is used both by the WinEHPrepare::findSelectorComparison() and @@ -657,6 +763,59 @@ static bool isSelectorDispatch(BasicBlock *BB, BasicBlock *&CatchHandler, return false; } +static BasicBlock *createStubLandingPad(Function *Handler, + Value *PersonalityFn) { + // FIXME: Finish this! + LLVMContext &Context = Handler->getContext(); + BasicBlock *StubBB = BasicBlock::Create(Context, "stub"); + Handler->getBasicBlockList().push_back(StubBB); + IRBuilder<> Builder(StubBB); + LandingPadInst *LPad = Builder.CreateLandingPad( + llvm::StructType::get(Type::getInt8PtrTy(Context), + Type::getInt32Ty(Context), nullptr), + PersonalityFn, 0); + LPad->setCleanup(true); + Builder.CreateUnreachable(); + return StubBB; +} + +// Cycles through the blocks in an outlined handler function looking for an +// invoke instruction and inserts an invoke of llvm.donothing with an empty +// landing pad if none is found. The code that generates the .xdata tables for +// the handler needs at least one landing pad to identify the parent function's +// personality. +void WinEHPrepare::addStubInvokeToHandlerIfNeeded(Function *Handler, + Value *PersonalityFn) { + ReturnInst *Ret = nullptr; + for (BasicBlock &BB : *Handler) { + TerminatorInst *Terminator = BB.getTerminator(); + // If we find an invoke, there is nothing to be done. + auto *II = dyn_cast(Terminator); + if (II) + return; + // If we've already recorded a return instruction, keep looking for invokes. + if (Ret) + continue; + // If we haven't recorded a return instruction yet, try this terminator. + Ret = dyn_cast(Terminator); + } + + // If we got this far, the handler contains no invokes. We should have seen + // at least one return. We'll insert an invoke of llvm.donothing ahead of + // that return. + assert(Ret); + BasicBlock *OldRetBB = Ret->getParent(); + BasicBlock *NewRetBB = SplitBlock(OldRetBB, Ret); + // SplitBlock adds an unconditional branch instruction at the end of the + // parent block. We want to replace that with an invoke call, so we can + // erase it now. + OldRetBB->getTerminator()->eraseFromParent(); + BasicBlock *StubLandingPad = createStubLandingPad(Handler, PersonalityFn); + Function *F = + Intrinsic::getDeclaration(Handler->getParent(), Intrinsic::donothing); + InvokeInst::Create(F, NewRetBB, StubLandingPad, None, "", OldRetBB); +} + bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn, LandingPadInst *LPad, BasicBlock *StartBB, FrameVarInfoMap &VarInfo) { @@ -680,6 +839,8 @@ bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn, SrcFn->getName() + ".cleanup", M); } + Handler->addFnAttr("wineh-parent", SrcFn->getName()); + // Generate a standard prolog to setup the frame recovery structure. IRBuilder<> Builder(Context); BasicBlock *Entry = BasicBlock::Create(Context, "entry"); @@ -696,10 +857,14 @@ bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn, LPadMap.mapLandingPad(LPad); if (auto *CatchAction = dyn_cast(Action)) { Constant *Sel = CatchAction->getSelector(); - Director.reset(new WinEHCatchDirector(Handler, Sel, VarInfo, LPadMap)); - LPadMap.remapSelector(VMap, ConstantInt::get(Type::getInt32Ty(Context), 1)); + Director.reset(new WinEHCatchDirector(Handler, Sel, VarInfo, LPadMap, + NestedLPtoOriginalLP)); + LPadMap.remapEHValues(VMap, UndefValue::get(Int8PtrType), + ConstantInt::get(Type::getInt32Ty(Context), 1)); } else { Director.reset(new WinEHCleanupDirector(Handler, VarInfo, LPadMap)); + LPadMap.remapEHValues(VMap, UndefValue::get(Int8PtrType), + UndefValue::get(Type::getInt32Ty(Context))); } SmallVector Returns; @@ -735,12 +900,45 @@ bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn, Entry->getInstList().splice(Entry->end(), FirstClonedBB->getInstList()); FirstClonedBB->eraseFromParent(); + // Make sure we can identify the handler's personality later. + addStubInvokeToHandlerIfNeeded(Handler, LPad->getPersonalityFn()); + if (auto *CatchAction = dyn_cast(Action)) { WinEHCatchDirector *CatchDirector = reinterpret_cast(Director.get()); CatchAction->setExceptionVar(CatchDirector->getExceptionVar()); CatchAction->setReturnTargets(CatchDirector->getReturnTargets()); - } + + // Look for blocks that are not part of the landing pad that we just + // outlined but terminate with a call to llvm.eh.endcatch and a + // branch to a block that is in the handler we just outlined. + // These blocks will be part of a nested landing pad that intends to + // return to an address in this handler. This case is best handled + // after both landing pads have been outlined, so for now we'll just + // save the association of the blocks in LPadTargetBlocks. The + // return instructions which are created from these branches will be + // replaced after all landing pads have been outlined. + for (const auto MapEntry : VMap) { + // VMap maps all values and blocks that were just cloned, but dead + // blocks which were pruned will map to nullptr. + if (!isa(MapEntry.first) || MapEntry.second == nullptr) + continue; + const BasicBlock *MappedBB = cast(MapEntry.first); + for (auto *Pred : predecessors(const_cast(MappedBB))) { + auto *Branch = dyn_cast(Pred->getTerminator()); + if (!Branch || !Branch->isUnconditional() || Pred->size() <= 1) + continue; + BasicBlock::iterator II = const_cast(Branch); + --II; + if (match(cast(II), m_Intrinsic())) { + // This would indicate that a nested landing pad wants to return + // to a block that is outlined into two different handlers. + assert(!LPadTargetBlocks.count(MappedBB)); + LPadTargetBlocks[MappedBB] = cast(MapEntry.second); + } + } + } + } // End if (CatchAction) Action->setHandlerBlockOrFunc(Handler); @@ -787,9 +985,8 @@ void LandingPadMap::mapLandingPad(const LandingPadInst *LPad) { // The landingpad instruction returns an aggregate value. Typically, its // value will be passed to a pair of extract value instructions and the - // results of those extracts are often passed to store instructions. - // In unoptimized code the stored value will often be loaded and then stored - // again. + // results of those extracts will have been promoted to reg values before + // this routine is called. for (auto *U : LPad->users()) { const ExtractValueInst *Extract = dyn_cast(U); if (!Extract) @@ -800,36 +997,17 @@ void LandingPadMap::mapLandingPad(const LandingPadInst *LPad) { assert((Idx == 0 || Idx == 1) && "Unexpected operation: extracting an unknown landing pad element"); if (Idx == 0) { - // Element 0 doesn't directly corresponds to anything in the WinEH - // scheme. - // It will be stored to a memory location, then later loaded and finally - // the loaded value will be used as the argument to an - // llvm.eh.begincatch - // call. We're tracking it here so that we can skip the store and load. ExtractedEHPtrs.push_back(Extract); } else if (Idx == 1) { - // Element 1 corresponds to the filter selector. We'll map it to 1 for - // matching purposes, but it will also probably be stored to memory and - // reloaded, so we need to track the instuction so that we can map the - // loaded value too. ExtractedSelectors.push_back(Extract); } - - // Look for stores of the extracted values. - for (auto *EU : Extract->users()) { - if (auto *Store = dyn_cast(EU)) { - if (Idx == 1) { - SelectorStores.push_back(Store); - SelectorStoreAddrs.push_back(Store->getPointerOperand()); - } else { - EHPtrStores.push_back(Store); - EHPtrStoreAddrs.push_back(Store->getPointerOperand()); - } - } - } } } +bool LandingPadMap::isOriginLandingPadBlock(const BasicBlock *BB) const { + return BB->getLandingPadInst() == OriginLPad; +} + bool LandingPadMap::isLandingPadSpecificInst(const Instruction *Inst) const { if (Inst == OriginLPad) return true; @@ -841,47 +1019,16 @@ bool LandingPadMap::isLandingPadSpecificInst(const Instruction *Inst) const { if (Inst == Extract) return true; } - for (auto *Store : EHPtrStores) { - if (Inst == Store) - return true; - } - for (auto *Store : SelectorStores) { - if (Inst == Store) - return true; - } - return false; } -void LandingPadMap::remapSelector(ValueToValueMapTy &VMap, - Value *MappedValue) const { - // Remap all selector extract instructions to the specified value. +void LandingPadMap::remapEHValues(ValueToValueMapTy &VMap, Value *EHPtrValue, + Value *SelectorValue) const { + // Remap all landing pad extract instructions to the specified values. + for (auto *Extract : ExtractedEHPtrs) + VMap[Extract] = EHPtrValue; for (auto *Extract : ExtractedSelectors) - VMap[Extract] = MappedValue; -} - -bool LandingPadMap::mapIfEHLoad(const LoadInst *Load, - SmallVectorImpl &Stores, - SmallVectorImpl &StoreAddrs) { - // This makes the assumption that a store we've previously seen dominates - // this load instruction. That might seem like a rather huge assumption, - // but given the way that landingpads are constructed its fairly safe. - // FIXME: Add debug/assert code that verifies this. - const Value *LoadAddr = Load->getPointerOperand(); - for (auto *StoreAddr : StoreAddrs) { - if (LoadAddr == StoreAddr) { - // Handle the common debug scenario where this loaded value is stored - // to a different location. - for (auto *U : Load->users()) { - if (auto *Store = dyn_cast(U)) { - Stores.push_back(Store); - StoreAddrs.push_back(Store->getPointerOperand()); - } - } - return true; - } - } - return false; + VMap[Extract] = SelectorValue; } CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction( @@ -891,40 +1038,13 @@ CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction( if (LPadMap.isLandingPadSpecificInst(Inst)) return CloningDirector::SkipInstruction; - if (auto *Load = dyn_cast(Inst)) { - // Look for loads of (previously suppressed) landingpad values. - // The EHPtr load can be mapped to an undef value as it should only be used - // as an argument to llvm.eh.begincatch, but the selector value needs to be - // mapped to a constant value of 1. This value will be used to simplify the - // branching to always flow to the current handler. - if (LPadMap.mapIfSelectorLoad(Load)) { - VMap[Inst] = ConstantInt::get(SelectorIDType, 1); - return CloningDirector::SkipInstruction; - } - if (LPadMap.mapIfEHPtrLoad(Load)) { - VMap[Inst] = UndefValue::get(Int8PtrType); - return CloningDirector::SkipInstruction; - } - - // Any other loads just get cloned. - return CloningDirector::CloneInstruction; - } - // Nested landing pads will be cloned as stubs, with just the // landingpad instruction and an unreachable instruction. When // all landingpads have been outlined, we'll replace this with the // llvm.eh.actions call and indirect branch created when the // landing pad was outlined. - if (auto *NestedLPad = dyn_cast(Inst)) { - Instruction *NewInst = NestedLPad->clone(); - if (NestedLPad->hasName()) - NewInst->setName(NestedLPad->getName()); - // FIXME: Store this mapping somewhere else also. - VMap[NestedLPad] = NewInst; - BasicBlock::InstListType &InstList = NewBB->getInstList(); - InstList.push_back(NewInst); - InstList.push_back(new UnreachableInst(NewBB->getContext())); - return CloningDirector::StopCloningBB; + if (auto *LPad = dyn_cast(Inst)) { + return handleLandingPad(VMap, LPad, NewBB); } if (auto *Invoke = dyn_cast(Inst)) @@ -944,6 +1064,20 @@ CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction( return CloningDirector::CloneInstruction; } +CloningDirector::CloningAction WinEHCatchDirector::handleLandingPad( + ValueToValueMapTy &VMap, const LandingPadInst *LPad, BasicBlock *NewBB) { + Instruction *NewInst = LPad->clone(); + if (LPad->hasName()) + NewInst->setName(LPad->getName()); + // Save this correlation for later processing. + NestedLPtoOriginalLP[cast(NewInst)] = LPad; + VMap[LPad] = NewInst; + BasicBlock::InstListType &InstList = NewBB->getInstList(); + InstList.push_back(NewInst); + InstList.push_back(new UnreachableInst(NewBB->getContext())); + return CloningDirector::StopCloningBB; +} + CloningDirector::CloningAction WinEHCatchDirector::handleBeginCatch( ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { // The argument to the call is some form of the first element of the @@ -958,6 +1092,11 @@ CloningDirector::CloningAction WinEHCatchDirector::handleBeginCatch( "llvm.eh.begincatch found while " "outlining catch handler."); ExceptionObjectVar = Inst->getOperand(1)->stripPointerCasts(); + if (isa(ExceptionObjectVar)) + return CloningDirector::SkipInstruction; + assert(cast(ExceptionObjectVar)->isStaticAlloca() && + "catch parameter is not static alloca"); + Materializer.escapeCatchObject(ExceptionObjectVar); return CloningDirector::SkipInstruction; } @@ -971,27 +1110,32 @@ WinEHCatchDirector::handleEndCatch(ValueToValueMapTy &VMap, // The end catch call can occur in one of two places: either in a // landingpad block that is part of the catch handlers exception mechanism, - // or at the end of the catch block. If it occurs in a landing pad, we must - // skip it and continue so that the landing pad gets cloned. - // FIXME: This case isn't fully supported yet and shouldn't turn up in any - // of the test cases until it is. - if (IntrinCall->getParent()->isLandingPad()) + // or at the end of the catch block. However, a catch-all handler may call + // end catch from the original landing pad. If the call occurs in a nested + // landing pad block, we must skip it and continue so that the landing pad + // gets cloned. + auto *ParentBB = IntrinCall->getParent(); + if (ParentBB->isLandingPad() && !LPadMap.isOriginLandingPadBlock(ParentBB)) return CloningDirector::SkipInstruction; - // If an end catch occurs anywhere else the next instruction should be an - // unconditional branch instruction that we want to replace with a return - // to the the address of the branch target. - const BasicBlock *EndCatchBB = IntrinCall->getParent(); - const TerminatorInst *Terminator = EndCatchBB->getTerminator(); - const BranchInst *Branch = dyn_cast(Terminator); - assert(Branch && Branch->isUnconditional()); - assert(std::next(BasicBlock::const_iterator(IntrinCall)) == - BasicBlock::const_iterator(Branch)); - - BasicBlock *ContinueLabel = Branch->getSuccessor(0); - ReturnInst::Create(NewBB->getContext(), BlockAddress::get(ContinueLabel), - NewBB); - ReturnTargets.push_back(ContinueLabel); + // If an end catch occurs anywhere else we want to terminate the handler + // with a return to the code that follows the endcatch call. If the + // next instruction is not an unconditional branch, we need to split the + // block to provide a clear target for the return instruction. + BasicBlock *ContinueBB; + auto Next = std::next(BasicBlock::const_iterator(IntrinCall)); + const BranchInst *Branch = dyn_cast(Next); + if (!Branch || !Branch->isUnconditional()) { + // We're interrupting the cloning process at this location, so the + // const_cast we're doing here will not cause a problem. + ContinueBB = SplitBlock(const_cast(ParentBB), + const_cast(cast(Next))); + } else { + ContinueBB = Branch->getSuccessor(0); + } + + ReturnInst::Create(NewBB->getContext(), BlockAddress::get(ContinueBB), NewBB); + ReturnTargets.push_back(ContinueBB); // We just added a terminator to the cloned block. // Tell the caller to stop processing the current basic block so that @@ -1029,6 +1173,20 @@ WinEHCatchDirector::handleResume(ValueToValueMapTy &VMap, return CloningDirector::StopCloningBB; } +CloningDirector::CloningAction WinEHCleanupDirector::handleLandingPad( + ValueToValueMapTy &VMap, const LandingPadInst *LPad, BasicBlock *NewBB) { + // The MS runtime will terminate the process if an exception occurs in a + // cleanup handler, so we shouldn't encounter landing pads in the actual + // cleanup code, but they may appear in catch blocks. Depending on where + // we started cloning we may see one, but it will get dropped during dead + // block pruning. + Instruction *NewInst = new UnreachableInst(NewBB->getContext()); + VMap[LPad] = NewInst; + BasicBlock::InstListType &InstList = NewBB->getInstList(); + InstList.push_back(NewInst); + return CloningDirector::StopCloningBB; +} + CloningDirector::CloningAction WinEHCleanupDirector::handleBeginCatch( ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { // Catch blocks within cleanup handlers will always be unreachable. @@ -1041,12 +1199,9 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleBeginCatch( CloningDirector::CloningAction WinEHCleanupDirector::handleEndCatch( ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) { - // Catch blocks within cleanup handlers will always be unreachable. - // We'll insert an unreachable instruction now, but it will be pruned - // before the cloning process is complete. - BasicBlock::InstListType &InstList = NewBB->getInstList(); - InstList.push_back(new UnreachableInst(NewBB->getContext())); - return CloningDirector::StopCloningBB; + // Cleanup handlers nested within catch handlers may begin with a call to + // eh.endcatch. We can just ignore that instruction. + return CloningDirector::SkipInstruction; } CloningDirector::CloningAction WinEHCleanupDirector::handleTypeIdFor( @@ -1080,6 +1235,9 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleInvoke( NewCall->setDebugLoc(Invoke->getDebugLoc()); VMap[Invoke] = NewCall; + // Remap the operands. + llvm::RemapInstruction(NewCall, VMap, RF_None, nullptr, &Materializer); + // Insert an unconditional branch to the normal destination. BranchInst::Create(Invoke->getNormalDest(), NewBB); @@ -1088,7 +1246,7 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleInvoke( // We just added a terminator to the cloned block. // Tell the caller to stop processing the current basic block. - return CloningDirector::StopCloningBB; + return CloningDirector::CloneSuccessors; } CloningDirector::CloningAction WinEHCleanupDirector::handleResume( @@ -1104,7 +1262,8 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleResume( WinEHFrameVariableMaterializer::WinEHFrameVariableMaterializer( Function *OutlinedFn, FrameVarInfoMap &FrameVarInfo) : FrameVarInfo(FrameVarInfo), Builder(OutlinedFn->getContext()) { - Builder.SetInsertPoint(&OutlinedFn->getEntryBlock()); + BasicBlock *EntryBB = &OutlinedFn->getEntryBlock(); + Builder.SetInsertPoint(EntryBB, EntryBB->getFirstInsertionPt()); } Value *WinEHFrameVariableMaterializer::materializeValueFor(Value *V) { @@ -1139,6 +1298,15 @@ Value *WinEHFrameVariableMaterializer::materializeValueFor(Value *V) { return nullptr; } +void WinEHFrameVariableMaterializer::escapeCatchObject(Value *V) { + // Catch parameter objects have to live in the parent frame. When we see a use + // of a catch parameter, add a sentinel to the multimap to indicate that it's + // used from another handler. This will prevent us from trying to sink the + // alloca into the handler and ensure that the catch parameter is present in + // the call to llvm.frameescape. + FrameVarInfo[V].push_back(getCatchObjectSentinel()); +} + // This function maps the catch and cleanup handlers that are reachable from the // specified landing pad. The landing pad sequence will have this basic shape: // @@ -1176,13 +1344,7 @@ void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad, DEBUG(dbgs() << "Mapping landing pad: " << BB->getName() << "\n"); if (NumClauses == 0) { - // This landing pad contains only cleanup code. - CleanupHandler *Action = new CleanupHandler(BB); - CleanupHandlerMap[BB] = Action; - Actions.insertCleanupHandler(Action); - DEBUG(dbgs() << " Assuming cleanup code in block " << BB->getName() - << "\n"); - assert(LPad->isCleanup()); + findCleanupHandlers(Actions, BB, nullptr); return; } @@ -1202,14 +1364,8 @@ void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad, // exceptions but code called from catches can. For SEH, it isn't // important if some finally code before a catch-all is executed out of // line or after recovering from the exception. - if (Personality == EHPersonality::MSVC_CXX) { - if (auto *CleanupAction = findCleanupHandler(BB, BB)) { - // Add a cleanup entry to the list - Actions.insertCleanupHandler(CleanupAction); - DEBUG(dbgs() << " Found cleanup code in block " - << CleanupAction->getStartBlock()->getName() << "\n"); - } - } + if (Personality == EHPersonality::MSVC_CXX) + findCleanupHandlers(Actions, BB, BB); // Add the catch handler to the action list. CatchHandler *Action = @@ -1226,13 +1382,7 @@ void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad, CatchHandler *CatchAction = findCatchHandler(BB, NextBB, VisitedBlocks); // See if there is any interesting code executed before the dispatch. - if (auto *CleanupAction = - findCleanupHandler(BB, CatchAction->getStartBlock())) { - // Add a cleanup entry to the list - Actions.insertCleanupHandler(CleanupAction); - DEBUG(dbgs() << " Found cleanup code in block " - << CleanupAction->getStartBlock()->getName() << "\n"); - } + findCleanupHandlers(Actions, BB, CatchAction->getStartBlock()); assert(CatchAction); ++HandlersFound; @@ -1248,12 +1398,7 @@ void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad, // If we didn't wind up in a catch-all, see if there is any interesting code // executed before the resume. - if (auto *CleanupAction = findCleanupHandler(BB, BB)) { - // Add a cleanup entry to the list - Actions.insertCleanupHandler(CleanupAction); - DEBUG(dbgs() << " Found cleanup code in block " - << CleanupAction->getStartBlock()->getName() << "\n"); - } + findCleanupHandlers(Actions, BB, BB); // It's possible that some optimization moved code into a landingpad that // wasn't @@ -1313,20 +1458,56 @@ CatchHandler *WinEHPrepare::findCatchHandler(BasicBlock *BB, return nullptr; } -// These are helper functions to combine repeated code from findCleanupHandler. -static CleanupHandler *createCleanupHandler(CleanupHandlerMapTy &CleanupHandlerMap, - BasicBlock *BB) { +// These are helper functions to combine repeated code from findCleanupHandlers. +static void createCleanupHandler(LandingPadActions &Actions, + CleanupHandlerMapTy &CleanupHandlerMap, + BasicBlock *BB) { CleanupHandler *Action = new CleanupHandler(BB); CleanupHandlerMap[BB] = Action; - return Action; + Actions.insertCleanupHandler(Action); + DEBUG(dbgs() << " Found cleanup code in block " + << Action->getStartBlock()->getName() << "\n"); +} + +static bool isFrameAddressCall(Value *V) { + return match(V, m_Intrinsic(m_SpecificInt(0))); +} + +static CallSite matchOutlinedFinallyCall(BasicBlock *BB, + Instruction *MaybeCall) { + // Look for finally blocks that Clang has already outlined for us. + // %fp = call i8* @llvm.frameaddress(i32 0) + // call void @"fin$parent"(iN 1, i8* %fp) + if (isFrameAddressCall(MaybeCall) && MaybeCall != BB->getTerminator()) + MaybeCall = MaybeCall->getNextNode(); + CallSite FinallyCall(MaybeCall); + if (!FinallyCall || FinallyCall.arg_size() != 2) + return CallSite(); + if (!match(FinallyCall.getArgument(0), m_SpecificInt(1))) + return CallSite(); + if (!isFrameAddressCall(FinallyCall.getArgument(1))) + return CallSite(); + return FinallyCall; +} + +static BasicBlock *followSingleUnconditionalBranches(BasicBlock *BB) { + // Skip single ubr blocks. + while (BB->getFirstNonPHIOrDbg() == BB->getTerminator()) { + auto *Br = dyn_cast(BB->getTerminator()); + if (Br && Br->isUnconditional()) + BB = Br->getSuccessor(0); + else + return BB; + } + return BB; } // This function searches starting with the input block for the next block that // contains code that is not part of a catch handler and would not be eliminated // during handler outlining. // -CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, - BasicBlock *EndBB) { +void WinEHPrepare::findCleanupHandlers(LandingPadActions &Actions, + BasicBlock *StartBB, BasicBlock *EndBB) { // Here we will skip over the following: // // landing pad prolog: @@ -1343,6 +1524,7 @@ CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, // Anything other than an unconditional branch will kick us out of this loop // one way or another. while (BB) { + BB = followSingleUnconditionalBranches(BB); // If we've already scanned this block, don't scan it again. If it is // a cleanup block, there will be an action in the CleanupHandlerMap. // If we've scanned it and it is not a cleanup block, there will be a @@ -1351,7 +1533,12 @@ CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, // avoid creating a null entry for blocks we haven't scanned. if (CleanupHandlerMap.count(BB)) { if (auto *Action = CleanupHandlerMap[BB]) { - return cast(Action); + Actions.insertCleanupHandler(Action); + DEBUG(dbgs() << " Found cleanup code in block " + << Action->getStartBlock()->getName() << "\n"); + // FIXME: This cleanup might chain into another, and we need to discover + // that. + return; } else { // Here we handle the case where the cleanup handler map contains a // value for this block but the value is a nullptr. This means that @@ -1363,11 +1550,9 @@ CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, // would terminate the search for cleanup code, so the unconditional // branch is the only case for which we might need to continue // searching. - if (BB == EndBB) - return nullptr; - BasicBlock *SuccBB; - if (!match(BB->getTerminator(), m_UnconditionalBr(SuccBB))) - return nullptr; + BasicBlock *SuccBB = followSingleUnconditionalBranches(BB); + if (SuccBB == BB || SuccBB == EndBB) + return; BB = SuccBB; continue; } @@ -1390,26 +1575,23 @@ CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, } // Look for the bare resume pattern: - // %exn2 = load i8** %exn.slot - // %sel2 = load i32* %ehselector.slot - // %lpad.val1 = insertvalue { i8*, i32 } undef, i8* %exn2, 0 - // %lpad.val2 = insertvalue { i8*, i32 } %lpad.val1, i32 %sel2, 1 + // %lpad.val1 = insertvalue { i8*, i32 } undef, i8* %exn, 0 + // %lpad.val2 = insertvalue { i8*, i32 } %lpad.val1, i32 %sel, 1 // resume { i8*, i32 } %lpad.val2 if (auto *Resume = dyn_cast(Terminator)) { InsertValueInst *Insert1 = nullptr; InsertValueInst *Insert2 = nullptr; Value *ResumeVal = Resume->getOperand(0); - // If there is only one landingpad, we may use the lpad directly with no - // insertions. - if (isa(ResumeVal)) - return nullptr; - if (!isa(ResumeVal)) { + // If the resume value isn't a phi or landingpad value, it should be a + // series of insertions. Identify them so we can avoid them when scanning + // for cleanups. + if (!isa(ResumeVal) && !isa(ResumeVal)) { Insert2 = dyn_cast(ResumeVal); if (!Insert2) - return createCleanupHandler(CleanupHandlerMap, BB); + return createCleanupHandler(Actions, CleanupHandlerMap, BB); Insert1 = dyn_cast(Insert2->getAggregateOperand()); if (!Insert1) - return createCleanupHandler(CleanupHandlerMap, BB); + return createCleanupHandler(Actions, CleanupHandlerMap, BB); } for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), IE = BB->end(); II != IE; ++II) { @@ -1420,66 +1602,133 @@ CleanupHandler *WinEHPrepare::findCleanupHandler(BasicBlock *StartBB, continue; if (!Inst->hasOneUse() || (Inst->user_back() != Insert1 && Inst->user_back() != Insert2)) { - return createCleanupHandler(CleanupHandlerMap, BB); + return createCleanupHandler(Actions, CleanupHandlerMap, BB); } } - return nullptr; + return; } BranchInst *Branch = dyn_cast(Terminator); - if (Branch) { - if (Branch->isConditional()) { - // Look for the selector dispatch. - // %sel = load i32* %ehselector.slot - // %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIf to i8*)) - // %matches = icmp eq i32 %sel12, %2 - // br i1 %matches, label %catch14, label %eh.resume - CmpInst *Compare = dyn_cast(Branch->getCondition()); - if (!Compare || !Compare->isEquality()) - return createCleanupHandler(CleanupHandlerMap, BB); - for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), - IE = BB->end(); - II != IE; ++II) { - Instruction *Inst = II; - if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) - continue; - if (Inst == Compare || Inst == Branch) - continue; - if (!Inst->hasOneUse() || (Inst->user_back() != Compare)) - return createCleanupHandler(CleanupHandlerMap, BB); - if (match(Inst, m_Intrinsic())) - continue; - if (!isa(Inst)) - return createCleanupHandler(CleanupHandlerMap, BB); - } - // The selector dispatch block should always terminate our search. - assert(BB == EndBB); - return nullptr; - } else { - // Look for empty blocks with unconditional branches. - for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), - IE = BB->end(); - II != IE; ++II) { - Instruction *Inst = II; - if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) - continue; - if (Inst == Branch) - continue; - if (match(Inst, m_Intrinsic())) - continue; - // Anything else makes this interesting cleanup code. - return createCleanupHandler(CleanupHandlerMap, BB); + if (Branch && Branch->isConditional()) { + // Look for the selector dispatch. + // %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIf to i8*)) + // %matches = icmp eq i32 %sel, %2 + // br i1 %matches, label %catch14, label %eh.resume + CmpInst *Compare = dyn_cast(Branch->getCondition()); + if (!Compare || !Compare->isEquality()) + return createCleanupHandler(Actions, CleanupHandlerMap, BB); + for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), IE = BB->end(); + II != IE; ++II) { + Instruction *Inst = II; + if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) + continue; + if (Inst == Compare || Inst == Branch) + continue; + if (match(Inst, m_Intrinsic())) + continue; + return createCleanupHandler(Actions, CleanupHandlerMap, BB); + } + // The selector dispatch block should always terminate our search. + assert(BB == EndBB); + return; + } + + if (isAsynchronousEHPersonality(Personality)) { + // If this is a landingpad block, split the block at the first non-landing + // pad instruction. + Instruction *MaybeCall = BB->getFirstNonPHIOrDbg(); + if (LPadMap) { + while (MaybeCall != BB->getTerminator() && + LPadMap->isLandingPadSpecificInst(MaybeCall)) + MaybeCall = MaybeCall->getNextNode(); + } + + // Look for outlined finally calls. + if (CallSite FinallyCall = matchOutlinedFinallyCall(BB, MaybeCall)) { + Function *Fin = FinallyCall.getCalledFunction(); + assert(Fin && "outlined finally call should be direct"); + auto *Action = new CleanupHandler(BB); + Action->setHandlerBlockOrFunc(Fin); + Actions.insertCleanupHandler(Action); + CleanupHandlerMap[BB] = Action; + DEBUG(dbgs() << " Found frontend-outlined finally call to " + << Fin->getName() << " in block " + << Action->getStartBlock()->getName() << "\n"); + + // Split the block if there were more interesting instructions and look + // for finally calls in the normal successor block. + BasicBlock *SuccBB = BB; + if (FinallyCall.getInstruction() != BB->getTerminator() && + FinallyCall.getInstruction()->getNextNode() != BB->getTerminator()) { + SuccBB = BB->splitBasicBlock(FinallyCall.getInstruction()->getNextNode()); + } else { + if (FinallyCall.isInvoke()) { + SuccBB = cast(FinallyCall.getInstruction())->getNormalDest(); + } else { + SuccBB = BB->getUniqueSuccessor(); + assert(SuccBB && "splitOutlinedFinallyCalls didn't insert a branch"); + } } + BB = SuccBB; if (BB == EndBB) - return nullptr; - // The branch was unconditional. - BB = Branch->getSuccessor(0); + return; + continue; + } + } + + // Anything else is either a catch block or interesting cleanup code. + for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), IE = BB->end(); + II != IE; ++II) { + Instruction *Inst = II; + if (LPadMap && LPadMap->isLandingPadSpecificInst(Inst)) + continue; + // Unconditional branches fall through to this loop. + if (Inst == Branch) continue; - } // End else of if branch was conditional - } // End if Branch + // If this is a catch block, there is no cleanup code to be found. + if (match(Inst, m_Intrinsic())) + return; + // If this a nested landing pad, it may contain an endcatch call. + if (match(Inst, m_Intrinsic())) + return; + // Anything else makes this interesting cleanup code. + return createCleanupHandler(Actions, CleanupHandlerMap, BB); + } - // Anything else makes this interesting cleanup code. - return createCleanupHandler(CleanupHandlerMap, BB); + // Only unconditional branches in empty blocks should get this far. + assert(Branch && Branch->isUnconditional()); + if (BB == EndBB) + return; + BB = Branch->getSuccessor(0); } - return nullptr; +} + +// This is a public function, declared in WinEHFuncInfo.h and is also +// referenced by WinEHNumbering in FunctionLoweringInfo.cpp. +void llvm::parseEHActions(const IntrinsicInst *II, + SmallVectorImpl &Actions) { + for (unsigned I = 0, E = II->getNumArgOperands(); I != E;) { + uint64_t ActionKind = + cast(II->getArgOperand(I))->getZExtValue(); + if (ActionKind == /*catch=*/1) { + auto *Selector = cast(II->getArgOperand(I + 1)); + ConstantInt *EHObjIndex = cast(II->getArgOperand(I + 2)); + int64_t EHObjIndexVal = EHObjIndex->getSExtValue(); + Constant *Handler = cast(II->getArgOperand(I + 3)); + I += 4; + auto *CH = new CatchHandler(/*BB=*/nullptr, Selector, /*NextBB=*/nullptr); + CH->setHandlerBlockOrFunc(Handler); + CH->setExceptionVarIndex(EHObjIndexVal); + Actions.push_back(CH); + } else if (ActionKind == 0) { + Constant *Handler = cast(II->getArgOperand(I + 1)); + I += 2; + auto *CH = new CleanupHandler(/*BB=*/nullptr); + CH->setHandlerBlockOrFunc(Handler); + Actions.push_back(CH); + } else { + llvm_unreachable("Expected either a catch or cleanup handler!"); + } + } + std::reverse(Actions.begin(), Actions.end()); } diff --git a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp index 9f56214..7846529 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp @@ -197,8 +197,7 @@ public: DataAlignmentFactor(DataAlignmentFactor), ReturnAddressRegister(ReturnAddressRegister) {} - ~CIE() { - } + ~CIE() override {} uint64_t getCodeAlignmentFactor() const { return CodeAlignmentFactor; } int64_t getDataAlignmentFactor() const { return DataAlignmentFactor; } @@ -245,8 +244,7 @@ public: InitialLocation(InitialLocation), AddressRange(AddressRange), LinkedCIE(Cie) {} - ~FDE() { - } + ~FDE() override {} CIE *getLinkedCIE() const { return LinkedCIE; } diff --git a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp index b14af07..0aff327 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp @@ -48,9 +48,10 @@ public: reset(); } - uint32_t getChildCount() const { return Args.size(); } + uint32_t getChildCount() const override { return Args.size(); } - std::unique_ptr getChildAtIndex(uint32_t Index) const { + std::unique_ptr + getChildAtIndex(uint32_t Index) const override { if (Index >= Args.size()) return nullptr; @@ -58,7 +59,7 @@ public: Args[Index]->getSymIndexId()); } - std::unique_ptr getNext() { + std::unique_ptr getNext() override { if (CurIter == Args.end()) return nullptr; const auto &Result = **CurIter; @@ -66,9 +67,9 @@ public: return Session.getConcreteSymbolById(Result.getSymIndexId()); } - void reset() { CurIter = Args.empty() ? Args.end() : Args.begin(); } + void reset() override { CurIter = Args.empty() ? Args.end() : Args.begin(); } - FunctionArgEnumerator *clone() const { + FunctionArgEnumerator *clone() const override { return new FunctionArgEnumerator(Session, Func); } diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp index 8018206..af3563f 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp @@ -34,25 +34,27 @@ public: std::unique_ptr ArgEnumerator) : Session(PDBSession), Enumerator(std::move(ArgEnumerator)) {} - uint32_t getChildCount() const { return Enumerator->getChildCount(); } + uint32_t getChildCount() const override { + return Enumerator->getChildCount(); + } - std::unique_ptr getChildAtIndex(uint32_t Index) const { + std::unique_ptr getChildAtIndex(uint32_t Index) const override { auto FunctionArgSymbol = Enumerator->getChildAtIndex(Index); if (!FunctionArgSymbol) return nullptr; return Session.getSymbolById(FunctionArgSymbol->getTypeId()); } - std::unique_ptr getNext() { + std::unique_ptr getNext() override { auto FunctionArgSymbol = Enumerator->getNext(); if (!FunctionArgSymbol) return nullptr; return Session.getSymbolById(FunctionArgSymbol->getTypeId()); } - void reset() { Enumerator->reset(); } + void reset() override { Enumerator->reset(); } - MyType *clone() const { + MyType *clone() const override { std::unique_ptr Clone(Enumerator->clone()); return new FunctionArgEnumerator(Session, std::move(Clone)); } diff --git a/lib/ExecutionEngine/EventListenerCommon.h b/lib/ExecutionEngine/EventListenerCommon.h deleted file mode 100644 index 6453099..0000000 --- a/lib/ExecutionEngine/EventListenerCommon.h +++ /dev/null @@ -1,68 +0,0 @@ -//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Common functionality for JITEventListener implementations -// -//===----------------------------------------------------------------------===// - -#ifndef EVENT_LISTENER_COMMON_H -#define EVENT_LISTENER_COMMON_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/Support/Path.h" - -namespace llvm { - -namespace jitprofiling { - -class FilenameCache { - // Holds the filename of each Scope, so that we can pass a null-terminated - // string into oprofile. - DenseMap Filenames; - DenseMap Paths; - - public: - const char *getFilename(MDNode *Scope) { - assert(Scope->isResolved() && "Expected Scope to be resolved"); - std::string &Filename = Filenames[Scope]; - if (Filename.empty()) { - DIScope DIScope(Scope); - Filename = DIScope.getFilename(); - } - return Filename.c_str(); - } - - const char *getFullPath(MDNode *Scope) { - assert(Scope->isResolved() && "Expected Scope to be resolved"); - std::string &P = Paths[Scope]; - if (P.empty()) { - DIScope DIScope(Scope); - StringRef DirName = DIScope.getDirectory(); - StringRef FileName = DIScope.getFilename(); - SmallString<256> FullPath; - if (DirName != "." && DirName != "") { - FullPath = DirName; - } - if (FileName != "") { - sys::path::append(FullPath, FileName); - } - P = FullPath.str(); - } - return P.c_str(); - } -}; - -} // namespace jitprofiling - -} // namespace llvm - -#endif //EVENT_LISTENER_COMMON_H diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp index c586ba7..d7038fd 100644 --- a/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/lib/ExecutionEngine/ExecutionEngine.cpp @@ -18,9 +18,11 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/ValueHandle.h" @@ -45,11 +47,13 @@ STATISTIC(NumGlobals , "Number of global vars initialized"); ExecutionEngine *(*ExecutionEngine::MCJITCtor)( std::unique_ptr M, std::string *ErrorStr, - std::unique_ptr MCJMM, + std::shared_ptr MemMgr, + std::shared_ptr Resolver, std::unique_ptr TM) = nullptr; ExecutionEngine *(*ExecutionEngine::OrcMCJITReplacementCtor)( - std::string *ErrorStr, std::unique_ptr OrcJMM, + std::string *ErrorStr, std::shared_ptr MemMgr, + std::shared_ptr Resolver, std::unique_ptr TM) = nullptr; ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr M, @@ -58,8 +62,7 @@ ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr M, void JITEventListener::anchor() {} ExecutionEngine::ExecutionEngine(std::unique_ptr M) - : EEState(*this), - LazyFunctionCreator(nullptr) { + : LazyFunctionCreator(nullptr) { CompilingLazily = false; GVCompilationDisabled = false; SymbolSearchingDisabled = false; @@ -151,38 +154,52 @@ Function *ExecutionEngine::FindFunctionNamed(const char *FnName) { } -void *ExecutionEngineState::RemoveMapping(const GlobalValue *ToUnmap) { - GlobalAddressMapTy::iterator I = GlobalAddressMap.find(ToUnmap); - void *OldVal; +uint64_t ExecutionEngineState::RemoveMapping(StringRef Name) { + GlobalAddressMapTy::iterator I = GlobalAddressMap.find(Name); + uint64_t OldVal; // FIXME: This is silly, we shouldn't end up with a mapping -> 0 in the // GlobalAddressMap. if (I == GlobalAddressMap.end()) - OldVal = nullptr; + OldVal = 0; else { + GlobalAddressReverseMap.erase(I->second); OldVal = I->second; GlobalAddressMap.erase(I); } - GlobalAddressReverseMap.erase(OldVal); return OldVal; } +std::string ExecutionEngine::getMangledName(const GlobalValue *GV) { + MutexGuard locked(lock); + Mangler Mang(DL); + SmallString<128> FullName; + Mang.getNameWithPrefix(FullName, GV->getName()); + return FullName.str(); +} + void ExecutionEngine::addGlobalMapping(const GlobalValue *GV, void *Addr) { MutexGuard locked(lock); + addGlobalMapping(getMangledName(GV), (uint64_t) Addr); +} + +void ExecutionEngine::addGlobalMapping(StringRef Name, uint64_t Addr) { + MutexGuard locked(lock); + + assert(!Name.empty() && "Empty GlobalMapping symbol name!"); - DEBUG(dbgs() << "JIT: Map \'" << GV->getName() - << "\' to [" << Addr << "]\n";); - void *&CurVal = EEState.getGlobalAddressMap()[GV]; + DEBUG(dbgs() << "JIT: Map \'" << Name << "\' to [" << Addr << "]\n";); + uint64_t &CurVal = EEState.getGlobalAddressMap()[Name]; assert((!CurVal || !Addr) && "GlobalMapping already established!"); CurVal = Addr; // If we are using the reverse mapping, add it too. if (!EEState.getGlobalAddressReverseMap().empty()) { - AssertingVH &V = - EEState.getGlobalAddressReverseMap()[Addr]; - assert((!V || !GV) && "GlobalMapping already established!"); - V = GV; + std::string &V = EEState.getGlobalAddressReverseMap()[CurVal]; + assert((!V.empty() || !Name.empty()) && + "GlobalMapping already established!"); + V = Name; } } @@ -197,13 +214,19 @@ void ExecutionEngine::clearGlobalMappingsFromModule(Module *M) { MutexGuard locked(lock); for (Module::iterator FI = M->begin(), FE = M->end(); FI != FE; ++FI) - EEState.RemoveMapping(FI); + EEState.RemoveMapping(getMangledName(FI)); for (Module::global_iterator GI = M->global_begin(), GE = M->global_end(); GI != GE; ++GI) - EEState.RemoveMapping(GI); + EEState.RemoveMapping(getMangledName(GI)); +} + +uint64_t ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, + void *Addr) { + MutexGuard locked(lock); + return updateGlobalMapping(getMangledName(GV), (uint64_t) Addr); } -void *ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, void *Addr) { +uint64_t ExecutionEngine::updateGlobalMapping(StringRef Name, uint64_t Addr) { MutexGuard locked(lock); ExecutionEngineState::GlobalAddressMapTy &Map = @@ -211,10 +234,10 @@ void *ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, void *Addr) { // Deleting from the mapping? if (!Addr) - return EEState.RemoveMapping(GV); + return EEState.RemoveMapping(Name); - void *&CurVal = Map[GV]; - void *OldVal = CurVal; + uint64_t &CurVal = Map[Name]; + uint64_t OldVal = CurVal; if (CurVal && !EEState.getGlobalAddressReverseMap().empty()) EEState.getGlobalAddressReverseMap().erase(CurVal); @@ -222,20 +245,35 @@ void *ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, void *Addr) { // If we are using the reverse mapping, add it too. if (!EEState.getGlobalAddressReverseMap().empty()) { - AssertingVH &V = - EEState.getGlobalAddressReverseMap()[Addr]; - assert((!V || !GV) && "GlobalMapping already established!"); - V = GV; + std::string &V = EEState.getGlobalAddressReverseMap()[CurVal]; + assert((!V.empty() || !Name.empty()) && + "GlobalMapping already established!"); + V = Name; } return OldVal; } -void *ExecutionEngine::getPointerToGlobalIfAvailable(const GlobalValue *GV) { +uint64_t ExecutionEngine::getAddressToGlobalIfAvailable(StringRef S) { MutexGuard locked(lock); - + uint64_t Address = 0; ExecutionEngineState::GlobalAddressMapTy::iterator I = - EEState.getGlobalAddressMap().find(GV); - return I != EEState.getGlobalAddressMap().end() ? I->second : nullptr; + EEState.getGlobalAddressMap().find(S); + if (I != EEState.getGlobalAddressMap().end()) + Address = I->second; + return Address; +} + + +void *ExecutionEngine::getPointerToGlobalIfAvailable(StringRef S) { + MutexGuard locked(lock); + if (void* Address = (void *) getAddressToGlobalIfAvailable(S)) + return Address; + return nullptr; +} + +void *ExecutionEngine::getPointerToGlobalIfAvailable(const GlobalValue *GV) { + MutexGuard locked(lock); + return getPointerToGlobalIfAvailable(getMangledName(GV)); } const GlobalValue *ExecutionEngine::getGlobalValueAtAddress(void *Addr) { @@ -244,15 +282,25 @@ const GlobalValue *ExecutionEngine::getGlobalValueAtAddress(void *Addr) { // If we haven't computed the reverse mapping yet, do so first. if (EEState.getGlobalAddressReverseMap().empty()) { for (ExecutionEngineState::GlobalAddressMapTy::iterator - I = EEState.getGlobalAddressMap().begin(), - E = EEState.getGlobalAddressMap().end(); I != E; ++I) + I = EEState.getGlobalAddressMap().begin(), + E = EEState.getGlobalAddressMap().end(); I != E; ++I) { + StringRef Name = I->first(); + uint64_t Addr = I->second; EEState.getGlobalAddressReverseMap().insert(std::make_pair( - I->second, I->first)); + Addr, Name)); + } } - std::map >::iterator I = - EEState.getGlobalAddressReverseMap().find(Addr); - return I != EEState.getGlobalAddressReverseMap().end() ? I->second : nullptr; + std::map::iterator I = + EEState.getGlobalAddressReverseMap().find((uint64_t) Addr); + + if (I != EEState.getGlobalAddressReverseMap().end()) { + StringRef Name = I->second; + for (unsigned i = 0, e = Modules.size(); i != e; ++i) + if (GlobalValue *GV = Modules[i]->getNamedValue(Name)) + return GV; + } + return nullptr; } namespace { @@ -404,8 +452,9 @@ EngineBuilder::EngineBuilder() : EngineBuilder(nullptr) {} EngineBuilder::EngineBuilder(std::unique_ptr M) : M(std::move(M)), WhichEngine(EngineKind::Either), ErrorStr(nullptr), - OptLevel(CodeGenOpt::Default), MCJMM(nullptr), RelocModel(Reloc::Default), - CMModel(CodeModel::JITDefault), UseOrcMCJITReplacement(false) { + OptLevel(CodeGenOpt::Default), MemMgr(nullptr), Resolver(nullptr), + RelocModel(Reloc::Default), CMModel(CodeModel::JITDefault), + UseOrcMCJITReplacement(false) { // IR module verification is enabled by default in debug builds, and disabled // by default in release builds. #ifndef NDEBUG @@ -419,7 +468,21 @@ EngineBuilder::~EngineBuilder() = default; EngineBuilder &EngineBuilder::setMCJITMemoryManager( std::unique_ptr mcjmm) { - MCJMM = std::move(mcjmm); + auto SharedMM = std::shared_ptr(std::move(mcjmm)); + MemMgr = SharedMM; + Resolver = SharedMM; + return *this; +} + +EngineBuilder& +EngineBuilder::setMemoryManager(std::unique_ptr MM) { + MemMgr = std::shared_ptr(std::move(MM)); + return *this; +} + +EngineBuilder& +EngineBuilder::setSymbolResolver(std::unique_ptr SR) { + Resolver = std::shared_ptr(std::move(SR)); return *this; } @@ -434,7 +497,7 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { // If the user specified a memory manager but didn't specify which engine to // create, we assume they only want the JIT, and we fail if they only want // the interpreter. - if (MCJMM) { + if (MemMgr) { if (WhichEngine & EngineKind::JIT) WhichEngine = EngineKind::JIT; else { @@ -456,12 +519,13 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { ExecutionEngine *EE = nullptr; if (ExecutionEngine::OrcMCJITReplacementCtor && UseOrcMCJITReplacement) { - EE = ExecutionEngine::OrcMCJITReplacementCtor(ErrorStr, std::move(MCJMM), + EE = ExecutionEngine::OrcMCJITReplacementCtor(ErrorStr, std::move(MemMgr), + std::move(Resolver), std::move(TheTM)); EE->addModule(std::move(M)); } else if (ExecutionEngine::MCJITCtor) - EE = ExecutionEngine::MCJITCtor(std::move(M), ErrorStr, std::move(MCJMM), - std::move(TheTM)); + EE = ExecutionEngine::MCJITCtor(std::move(M), ErrorStr, std::move(MemMgr), + std::move(Resolver), std::move(TheTM)); if (EE) { EE->setVerifyModules(VerifyModules); @@ -492,7 +556,7 @@ void *ExecutionEngine::getPointerToGlobal(const GlobalValue *GV) { return getPointerToFunction(F); MutexGuard locked(lock); - if (void *P = EEState.getGlobalAddressMap()[GV]) + if (void* P = getPointerToGlobalIfAvailable(GV)) return P; // Global variable might have been added since interpreter started. @@ -502,7 +566,7 @@ void *ExecutionEngine::getPointerToGlobal(const GlobalValue *GV) { else llvm_unreachable("Global hasn't had an address allocated yet!"); - return EEState.getGlobalAddressMap()[GV]; + return getPointerToGlobalIfAvailable(GV); } /// \brief Converts a Constant* into a GenericValue, including handling of @@ -1274,25 +1338,3 @@ void ExecutionEngine::EmitGlobalVariable(const GlobalVariable *GV) { NumInitBytes += (unsigned)GVSize; ++NumGlobals; } - -ExecutionEngineState::ExecutionEngineState(ExecutionEngine &EE) - : EE(EE), GlobalAddressMap(this) { -} - -sys::Mutex * -ExecutionEngineState::AddressMapConfig::getMutex(ExecutionEngineState *EES) { - return &EES->EE.lock; -} - -void ExecutionEngineState::AddressMapConfig::onDelete(ExecutionEngineState *EES, - const GlobalValue *Old) { - void *OldVal = EES->GlobalAddressMap.lookup(Old); - EES->GlobalAddressReverseMap.erase(OldVal); -} - -void ExecutionEngineState::AddressMapConfig::onRAUW(ExecutionEngineState *, - const GlobalValue *, - const GlobalValue *) { - llvm_unreachable("The ExecutionEngine doesn't know how to handle a" - " RAUW on a value it has a global mapping for."); -} diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp index aaa53f0..22ff311 100644 --- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -351,7 +351,7 @@ class SimpleBindingMemoryManager : public RTDyldMemoryManager { public: SimpleBindingMemoryManager(const SimpleBindingMMFunctions& Functions, void *Opaque); - virtual ~SimpleBindingMemoryManager(); + ~SimpleBindingMemoryManager() override; uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, diff --git a/lib/ExecutionEngine/GDBRegistrationListener.cpp b/lib/ExecutionEngine/GDBRegistrationListener.cpp index 8ef878c..1ab6203 100644 --- a/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -103,7 +103,7 @@ public: /// Unregisters each object that was previously registered and releases all /// internal resources. - virtual ~GDBJITRegistrationListener(); + ~GDBJITRegistrationListener() override; /// Creates an entry in the JIT registry for the buffer @p Object, /// which must contain an object file in executable memory with any diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index aa32452..4135900 100644 --- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Config/config.h" -#include "EventListenerCommon.h" #include "IntelJITEventsWrapper.h" #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineFunction.h" @@ -29,7 +28,6 @@ #include "llvm/Support/raw_ostream.h" using namespace llvm; -using namespace llvm::jitprofiling; using namespace llvm::object; #define DEBUG_TYPE "amplifier-jit-event-listener" @@ -41,7 +39,6 @@ class IntelJITEventListener : public JITEventListener { std::unique_ptr Wrapper; MethodIDMap MethodIDs; - FilenameCache Filenames; typedef SmallVector MethodAddressVector; typedef DenseMap ObjectMap; diff --git a/lib/ExecutionEngine/Interpreter/Execution.cpp b/lib/ExecutionEngine/Interpreter/Execution.cpp index 2e8eb16..a26740b 100644 --- a/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -316,7 +316,7 @@ void Interpreter::visitICmpInst(ICmpInst &I) { #define IMPLEMENT_VECTOR_FCMP(OP) \ case Type::VectorTyID: \ - if(dyn_cast(Ty)->getElementType()->isFloatTy()) { \ + if (cast(Ty)->getElementType()->isFloatTy()) { \ IMPLEMENT_VECTOR_FCMP_T(OP, Float); \ } else { \ IMPLEMENT_VECTOR_FCMP_T(OP, Double); \ @@ -363,7 +363,7 @@ static GenericValue executeFCMP_OEQ(GenericValue Src1, GenericValue Src2, #define MASK_VECTOR_NANS(TY, X,Y, FLAG) \ if (TY->isVectorTy()) { \ - if (dyn_cast(TY)->getElementType()->isFloatTy()) { \ + if (cast(TY)->getElementType()->isFloatTy()) { \ MASK_VECTOR_NANS_T(X, Y, Float, FLAG) \ } else { \ MASK_VECTOR_NANS_T(X, Y, Double, FLAG) \ @@ -536,7 +536,7 @@ static GenericValue executeFCMP_ORD(GenericValue Src1, GenericValue Src2, if(Ty->isVectorTy()) { assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); Dest.AggregateVal.resize( Src1.AggregateVal.size() ); - if(dyn_cast(Ty)->getElementType()->isFloatTy()) { + if (cast(Ty)->getElementType()->isFloatTy()) { for( size_t _i=0;_iisVectorTy()) { assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); Dest.AggregateVal.resize( Src1.AggregateVal.size() ); - if(dyn_cast(Ty)->getElementType()->isFloatTy()) { + if (cast(Ty)->getElementType()->isFloatTy()) { for( size_t _i=0;_i(Ty)->getElementType()->isFloatTy()) \ + if (cast(Ty)->getElementType()->isFloatTy()) \ FLOAT_VECTOR_FUNCTION(OP, FloatVal) \ else { \ - if (dyn_cast(Ty)->getElementType()->isDoubleTy()) \ + if (cast(Ty)->getElementType()->isDoubleTy()) \ FLOAT_VECTOR_FUNCTION(OP, DoubleVal) \ else { \ dbgs() << "Unhandled type for OP instruction: " << *Ty << "\n"; \ @@ -745,12 +745,12 @@ void Interpreter::visitBinaryOperator(BinaryOperator &I) { case Instruction::FMul: FLOAT_VECTOR_OP(*) break; case Instruction::FDiv: FLOAT_VECTOR_OP(/) break; case Instruction::FRem: - if (dyn_cast(Ty)->getElementType()->isFloatTy()) + if (cast(Ty)->getElementType()->isFloatTy()) for (unsigned i = 0; i < R.AggregateVal.size(); ++i) R.AggregateVal[i].FloatVal = fmod(Src1.AggregateVal[i].FloatVal, Src2.AggregateVal[i].FloatVal); else { - if (dyn_cast(Ty)->getElementType()->isDoubleTy()) + if (cast(Ty)->getElementType()->isDoubleTy()) for (unsigned i = 0; i < R.AggregateVal.size(); ++i) R.AggregateVal[i].DoubleVal = fmod(Src1.AggregateVal[i].DoubleVal, Src2.AggregateVal[i].DoubleVal); diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index b022101..e2fe065 100644 --- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -95,16 +95,15 @@ static ExFunc lookupFunction(const Function *F) { FunctionType *FT = F->getFunctionType(); for (unsigned i = 0, e = FT->getNumContainedTypes(); i != e; ++i) ExtName += getTypeID(FT->getContainedType(i)); - ExtName += "_" + F->getName().str(); + ExtName += ("_" + F->getName()).str(); sys::ScopedLock Writer(*FunctionsLock); ExFunc FnPtr = (*FuncNames)[ExtName]; if (!FnPtr) - FnPtr = (*FuncNames)["lle_X_" + F->getName().str()]; + FnPtr = (*FuncNames)[("lle_X_" + F->getName()).str()]; if (!FnPtr) // Try calling a generic function... if it exists... - FnPtr = (ExFunc)(intptr_t) - sys::DynamicLibrary::SearchForAddressOfSymbol("lle_X_" + - F->getName().str()); + FnPtr = (ExFunc)(intptr_t)sys::DynamicLibrary::SearchForAddressOfSymbol( + ("lle_X_" + F->getName()).str()); if (FnPtr) ExportedFunctions->insert(std::make_pair(F, FnPtr)); // Cache for later return FnPtr; diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h index 2be9c59..0dc0463 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -108,7 +108,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor { public: explicit Interpreter(std::unique_ptr M); - ~Interpreter(); + ~Interpreter() override; /// runAtExitHandlers - Run any functions registered by the program's calls to /// atexit(3), which we intercept and store in AtExitHandlers. diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp index 20b8553..7e37afe 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -8,6 +8,7 @@ //===----------------------------------------------------------------------===// #include "MCJIT.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/MCJIT.h" @@ -41,26 +42,35 @@ static struct RegisterJIT { extern "C" void LLVMLinkInMCJIT() { } -ExecutionEngine *MCJIT::createJIT(std::unique_ptr M, - std::string *ErrorStr, - std::unique_ptr MemMgr, - std::unique_ptr TM) { +ExecutionEngine* +MCJIT::createJIT(std::unique_ptr M, + std::string *ErrorStr, + std::shared_ptr MemMgr, + std::shared_ptr Resolver, + std::unique_ptr TM) { // Try to register the program as a source of symbols to resolve against. // // FIXME: Don't do this here. sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr); - std::unique_ptr MM = std::move(MemMgr); - if (!MM) - MM = std::unique_ptr(new SectionMemoryManager()); + if (!MemMgr || !Resolver) { + auto RTDyldMM = std::make_shared(); + if (!MemMgr) + MemMgr = RTDyldMM; + if (!Resolver) + Resolver = RTDyldMM; + } - return new MCJIT(std::move(M), std::move(TM), std::move(MM)); + return new MCJIT(std::move(M), std::move(TM), std::move(MemMgr), + std::move(Resolver)); } MCJIT::MCJIT(std::unique_ptr M, std::unique_ptr tm, - std::unique_ptr MM) + std::shared_ptr MemMgr, + std::shared_ptr Resolver) : ExecutionEngine(std::move(M)), TM(std::move(tm)), Ctx(nullptr), - MemMgr(this, std::move(MM)), Dyld(&MemMgr), ObjCache(nullptr) { + MemMgr(std::move(MemMgr)), Resolver(*this, std::move(Resolver)), + Dyld(*this->MemMgr, this->Resolver), ObjCache(nullptr) { // FIXME: We are managing our modules, so we do not want the base class // ExecutionEngine to manage them as well. To avoid double destruction // of the first (and only) module added in ExecutionEngine constructor @@ -221,7 +231,7 @@ void MCJIT::finalizeLoadedModules() { Dyld.registerEHFrames(); // Set page permissions. - MemMgr.finalizeMemory(); + MemMgr->finalizeMemory(); } // FIXME: Rename this. @@ -253,11 +263,11 @@ void MCJIT::finalizeModule(Module *M) { finalizeLoadedModules(); } -uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) { +RuntimeDyld::SymbolInfo MCJIT::findExistingSymbol(const std::string &Name) { Mangler Mang(TM->getDataLayout()); SmallString<128> FullName; Mang.getNameWithPrefix(FullName, Name); - return Dyld.getSymbol(FullName).getAddress(); + return Dyld.getSymbol(FullName); } Module *MCJIT::findModuleForSymbol(const std::string &Name, @@ -284,14 +294,17 @@ Module *MCJIT::findModuleForSymbol(const std::string &Name, } uint64_t MCJIT::getSymbolAddress(const std::string &Name, - bool CheckFunctionsOnly) -{ + bool CheckFunctionsOnly) { + return findSymbol(Name, CheckFunctionsOnly).getAddress(); +} + +RuntimeDyld::SymbolInfo MCJIT::findSymbol(const std::string &Name, + bool CheckFunctionsOnly) { MutexGuard locked(lock); // First, check to see if we already have this symbol. - uint64_t Addr = getExistingSymbolAddress(Name); - if (Addr) - return Addr; + if (auto Sym = findExistingSymbol(Name)) + return Sym; for (object::OwningBinary &OB : Archives) { object::Archive *A = OB.getBinary(); @@ -310,9 +323,8 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, // This causes the object file to be loaded. addObjectFile(std::move(OF)); // The address should be here now. - Addr = getExistingSymbolAddress(Name); - if (Addr) - return Addr; + if (auto Sym = findExistingSymbol(Name)) + return Sym; } } } @@ -323,15 +335,18 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, generateCodeForModule(M); // Check the RuntimeDyld table again, it should be there now. - return getExistingSymbolAddress(Name); + return findExistingSymbol(Name); } // If a LazyFunctionCreator is installed, use it to get/create the function. // FIXME: Should we instead have a LazySymbolCreator callback? - if (LazyFunctionCreator) - Addr = (uint64_t)LazyFunctionCreator(Name); + if (LazyFunctionCreator) { + auto Addr = static_cast( + reinterpret_cast(LazyFunctionCreator(Name))); + return RuntimeDyld::SymbolInfo(Addr, JITSymbolFlags::Exported); + } - return Addr; + return nullptr; } uint64_t MCJIT::getGlobalValueAddress(const std::string &Name) { @@ -528,7 +543,9 @@ GenericValue MCJIT::runFunction(Function *F, void *MCJIT::getPointerToNamedFunction(StringRef Name, bool AbortOnFailure) { if (!isSymbolSearchingDisabled()) { - void *ptr = MemMgr.getPointerToNamedFunction(Name, false); + void *ptr = + reinterpret_cast( + static_cast(Resolver.findSymbol(Name).getAddress())); if (ptr) return ptr; } @@ -566,7 +583,7 @@ void MCJIT::UnregisterJITEventListener(JITEventListener *L) { void MCJIT::NotifyObjectEmitted(const object::ObjectFile& Obj, const RuntimeDyld::LoadedObjectInfo &L) { MutexGuard locked(lock); - MemMgr.notifyObjectLoaded(this, Obj); + MemMgr->notifyObjectLoaded(this, Obj); for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { EventListeners[I]->NotifyObjectEmitted(Obj, L); } @@ -578,15 +595,16 @@ void MCJIT::NotifyFreeingObject(const object::ObjectFile& Obj) { L->NotifyFreeingObject(Obj); } -uint64_t LinkingMemoryManager::getSymbolAddress(const std::string &Name) { - uint64_t Result = ParentEngine->getSymbolAddress(Name, false); +RuntimeDyld::SymbolInfo +LinkingSymbolResolver::findSymbol(const std::string &Name) { + auto Result = ParentEngine.findSymbol(Name, false); // If the symbols wasn't found and it begins with an underscore, try again // without the underscore. if (!Result && Name[0] == '_') - Result = ParentEngine->getSymbolAddress(Name.substr(1), false); + Result = ParentEngine.findSymbol(Name.substr(1), false); if (Result) return Result; - if (ParentEngine->isSymbolSearchingDisabled()) - return 0; - return ClientMM->getSymbolAddress(Name); + if (ParentEngine.isSymbolSearchingDisabled()) + return nullptr; + return ClientResolver->findSymbol(Name); } diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h index de4a8f6..59e9949 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -16,6 +16,7 @@ #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/ExecutionEngine/ObjectCache.h" #include "llvm/ExecutionEngine/ObjectMemoryBuffer.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Module.h" @@ -26,59 +27,23 @@ class MCJIT; // functions across modules that it owns. It aggregates the memory manager // that is passed in to the MCJIT constructor and defers most functionality // to that object. -class LinkingMemoryManager : public RTDyldMemoryManager { +class LinkingSymbolResolver : public RuntimeDyld::SymbolResolver { public: - LinkingMemoryManager(MCJIT *Parent, - std::unique_ptr MM) - : ParentEngine(Parent), ClientMM(std::move(MM)) {} + LinkingSymbolResolver(MCJIT &Parent, + std::shared_ptr Resolver) + : ParentEngine(Parent), ClientResolver(std::move(Resolver)) {} - uint64_t getSymbolAddress(const std::string &Name) override; + RuntimeDyld::SymbolInfo findSymbol(const std::string &Name) override; - // Functions deferred to client memory manager - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - return ClientMM->allocateCodeSection(Size, Alignment, SectionID, SectionName); - } - - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsReadOnly) override { - return ClientMM->allocateDataSection(Size, Alignment, - SectionID, SectionName, IsReadOnly); - } - - void reserveAllocationSpace(uintptr_t CodeSize, uintptr_t DataSizeRO, - uintptr_t DataSizeRW) override { - return ClientMM->reserveAllocationSpace(CodeSize, DataSizeRO, DataSizeRW); - } - - bool needsToReserveAllocationSpace() override { - return ClientMM->needsToReserveAllocationSpace(); - } - - void notifyObjectLoaded(ExecutionEngine *EE, - const object::ObjectFile &Obj) override { - ClientMM->notifyObjectLoaded(EE, Obj); - } - - void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, - size_t Size) override { - ClientMM->registerEHFrames(Addr, LoadAddr, Size); - } - - void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, - size_t Size) override { - ClientMM->deregisterEHFrames(Addr, LoadAddr, Size); - } - - bool finalizeMemory(std::string *ErrMsg = nullptr) override { - return ClientMM->finalizeMemory(ErrMsg); + // MCJIT doesn't support logical dylibs. + RuntimeDyld::SymbolInfo + findSymbolInLogicalDylib(const std::string &Name) override { + return nullptr; } private: - MCJIT *ParentEngine; - std::unique_ptr ClientMM; + MCJIT &ParentEngine; + std::shared_ptr ClientResolver; }; // About Module states: added->loaded->finalized. @@ -103,7 +68,8 @@ private: class MCJIT : public ExecutionEngine { MCJIT(std::unique_ptr M, std::unique_ptr tm, - std::unique_ptr MemMgr); + std::shared_ptr MemMgr, + std::shared_ptr Resolver); typedef llvm::SmallPtrSet ModulePtrSet; @@ -214,7 +180,8 @@ class MCJIT : public ExecutionEngine { std::unique_ptr TM; MCContext *Ctx; - LinkingMemoryManager MemMgr; + std::shared_ptr MemMgr; + LinkingSymbolResolver Resolver; RuntimeDyld Dyld; std::vector EventListeners; @@ -238,7 +205,7 @@ class MCJIT : public ExecutionEngine { ModulePtrSet::iterator E); public: - ~MCJIT(); + ~MCJIT() override; /// @name ExecutionEngine interface implementation /// @{ @@ -324,17 +291,22 @@ public: MCJITCtor = createJIT; } - static ExecutionEngine *createJIT(std::unique_ptr M, - std::string *ErrorStr, - std::unique_ptr MemMgr, - std::unique_ptr TM); + static ExecutionEngine* + createJIT(std::unique_ptr M, + std::string *ErrorStr, + std::shared_ptr MemMgr, + std::shared_ptr Resolver, + std::unique_ptr TM); // @} + RuntimeDyld::SymbolInfo findSymbol(const std::string &Name, + bool CheckFunctionsOnly); + // DEPRECATED - Please use findSymbol instead. // This is not directly exposed via the ExecutionEngine API, but it is // used by the LinkingMemoryManager. uint64_t getSymbolAddress(const std::string &Name, - bool CheckFunctionsOnly); + bool CheckFunctionsOnly); protected: /// emitObject -- Generate a JITed object in memory from the specified module @@ -348,7 +320,7 @@ protected: const RuntimeDyld::LoadedObjectInfo &L); void NotifyFreeingObject(const object::ObjectFile& Obj); - uint64_t getExistingSymbolAddress(const std::string &Name); + RuntimeDyld::SymbolInfo findExistingSymbol(const std::string &Name); Module *findModuleForSymbol(const std::string &Name, bool CheckFunctionsOnly); }; diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index 9ab4003..23e7662 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Config/config.h" -#include "EventListenerCommon.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/OProfileWrapper.h" @@ -28,7 +27,6 @@ #include using namespace llvm; -using namespace llvm::jitprofiling; using namespace llvm::object; #define DEBUG_TYPE "oprofile-jit-event-listener" diff --git a/lib/ExecutionEngine/Orc/Android.mk b/lib/ExecutionEngine/Orc/Android.mk index 61c1daf..f28f359 100644 --- a/lib/ExecutionEngine/Orc/Android.mk +++ b/lib/ExecutionEngine/Orc/Android.mk @@ -6,6 +6,7 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES := \ CloneSubModule.cpp \ + ExecutionUtils.cpp \ IndirectionUtils.cpp \ OrcMCJITReplacement.cpp \ OrcTargetSupport.cpp diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt index b0a8445..b38b459 100644 --- a/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -1,5 +1,6 @@ add_llvm_library(LLVMOrcJIT CloneSubModule.cpp + ExecutionUtils.cpp IndirectionUtils.cpp OrcMCJITReplacement.cpp OrcTargetSupport.cpp diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp new file mode 100644 index 0000000..b7220db --- /dev/null +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -0,0 +1,102 @@ +//===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" + +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Module.h" + +namespace llvm { +namespace orc { + +CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) + : InitList( + GV ? dyn_cast_or_null(GV->getInitializer()) : nullptr), + I((InitList && End) ? InitList->getNumOperands() : 0) { +} + +bool CtorDtorIterator::operator==(const CtorDtorIterator &Other) const { + assert(InitList == Other.InitList && "Incomparable iterators."); + return I == Other.I; +} + +bool CtorDtorIterator::operator!=(const CtorDtorIterator &Other) const { + return !(*this == Other); +} + +CtorDtorIterator& CtorDtorIterator::operator++() { + ++I; + return *this; +} + +CtorDtorIterator CtorDtorIterator::operator++(int) { + CtorDtorIterator Temp = *this; + ++I; + return Temp; +} + +CtorDtorIterator::Element CtorDtorIterator::operator*() const { + ConstantStruct *CS = dyn_cast(InitList->getOperand(I)); + assert(CS && "Unrecognized type in llvm.global_ctors/llvm.global_dtors"); + + Constant *FuncC = CS->getOperand(1); + Function *Func = nullptr; + + // Extract function pointer, pulling off any casts. + while (FuncC) { + if (Function *F = dyn_cast_or_null(FuncC)) { + Func = F; + break; + } else if (ConstantExpr *CE = dyn_cast_or_null(FuncC)) { + if (CE->isCast()) + FuncC = dyn_cast_or_null(CE->getOperand(0)); + else + break; + } else { + // This isn't anything we recognize. Bail out with Func left set to null. + break; + } + } + + ConstantInt *Priority = dyn_cast(CS->getOperand(0)); + Value *Data = CS->getOperand(2); + return Element(Priority->getZExtValue(), Func, Data); +} + +iterator_range getConstructors(const Module &M) { + const GlobalVariable *CtorsList = M.getNamedGlobal("llvm.global_ctors"); + return make_range(CtorDtorIterator(CtorsList, false), + CtorDtorIterator(CtorsList, true)); +} + +iterator_range getDestructors(const Module &M) { + const GlobalVariable *DtorsList = M.getNamedGlobal("llvm.global_dtors"); + return make_range(CtorDtorIterator(DtorsList, false), + CtorDtorIterator(DtorsList, true)); +} + +void LocalCXXRuntimeOverrides::runDestructors() { + auto& CXXDestructorDataPairs = DSOHandleOverride; + for (auto &P : CXXDestructorDataPairs) + P.first(P.second); + CXXDestructorDataPairs.clear(); +} + +int LocalCXXRuntimeOverrides::CXAAtExitOverride(DestructorPtr Destructor, + void *Arg, void *DSOHandle) { + auto& CXXDestructorDataPairs = + *reinterpret_cast(DSOHandle); + CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg)); + return 0; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 8cf490f..ebeedef 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -14,17 +14,25 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" #include +#include namespace llvm { namespace orc { -GlobalVariable* createImplPointer(Function &F, const Twine &Name, - Constant *Initializer) { - assert(F.getParent() && "Function isn't in a module."); +Constant* createIRTypedAddress(FunctionType &FT, TargetAddress Addr) { + Constant *AddrIntVal = + ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); + Constant *AddrPtrVal = + ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, + PointerType::get(&FT, 0)); + return AddrPtrVal; +} + +GlobalVariable* createImplPointer(PointerType &PT, Module &M, + const Twine &Name, Constant *Initializer) { if (!Initializer) - Initializer = Constant::getNullValue(F.getType()); - Module &M = *F.getParent(); - return new GlobalVariable(M, F.getType(), false, GlobalValue::ExternalLinkage, + Initializer = Constant::getNullValue(&PT); + return new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, Initializer, Name, nullptr, GlobalValue::NotThreadLocal, 0, true); } @@ -44,8 +52,41 @@ void makeStub(Function &F, GlobalVariable &ImplPointer) { Builder.CreateRet(Call); } +// Utility class for renaming global values and functions during partitioning. +class GlobalRenamer { +public: + + static bool needsRenaming(const Value &New) { + if (!New.hasName() || New.getName().startswith("\01L")) + return true; + return false; + } + + const std::string& getRename(const Value &Orig) { + // See if we have a name for this global. + { + auto I = Names.find(&Orig); + if (I != Names.end()) + return I->second; + } + + // Nope. Create a new one. + // FIXME: Use a more robust uniquing scheme. (This may blow up if the user + // writes a "__orc_anon[[:digit:]]* method). + unsigned ID = Names.size(); + std::ostringstream NameStream; + NameStream << "__orc_anon" << ID++; + auto I = Names.insert(std::make_pair(&Orig, NameStream.str())); + return I.first->second; + } +private: + DenseMap Names; +}; + void partition(Module &M, const ModulePartitionMap &PMap) { + GlobalRenamer Renamer; + for (auto &KVPair : PMap) { auto ExtractGlobalVars = @@ -54,20 +95,26 @@ void partition(Module &M, const ModulePartitionMap &PMap) { if (KVPair.second.count(&Orig)) { copyGVInitializer(New, Orig, VMap); } - if (New.getLinkage() == GlobalValue::PrivateLinkage) { + if (New.hasLocalLinkage()) { + if (Renamer.needsRenaming(New)) + New.setName(Renamer.getRename(Orig)); New.setLinkage(GlobalValue::ExternalLinkage); New.setVisibility(GlobalValue::HiddenVisibility); } + assert(!Renamer.needsRenaming(New) && "Invalid global name."); }; auto ExtractFunctions = [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) { if (KVPair.second.count(&Orig)) copyFunctionBody(New, Orig, VMap); - if (New.getLinkage() == GlobalValue::InternalLinkage) { + if (New.hasLocalLinkage()) { + if (Renamer.needsRenaming(New)) + New.setName(Renamer.getRename(Orig)); New.setLinkage(GlobalValue::ExternalLinkage); New.setVisibility(GlobalValue::HiddenVisibility); } + assert(!Renamer.needsRenaming(New) && "Invalid function name."); }; CloneSubModule(*KVPair.first, M, ExtractGlobalVars, ExtractFunctions, diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 00e39bb..4023344 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -26,15 +26,21 @@ namespace orc { class OrcMCJITReplacement : public ExecutionEngine { - class ForwardingRTDyldMM : public RTDyldMemoryManager { + // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that + // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are + // expecting - see finalizeMemory. + class MCJITReplacementMemMgr : public MCJITMemoryManager { public: - ForwardingRTDyldMM(OrcMCJITReplacement &M) : M(M) {} + MCJITReplacementMemMgr(OrcMCJITReplacement &M, + std::shared_ptr ClientMM) + : M(M), ClientMM(std::move(ClientMM)) {} uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, StringRef SectionName) override { uint8_t *Addr = - M.MM->allocateCodeSection(Size, Alignment, SectionID, SectionName); + ClientMM->allocateCodeSection(Size, Alignment, SectionID, + SectionName); M.SectionsAllocatedSinceLastLoad.insert(Addr); return Addr; } @@ -42,43 +48,35 @@ class OrcMCJITReplacement : public ExecutionEngine { uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, StringRef SectionName, bool IsReadOnly) override { - uint8_t *Addr = M.MM->allocateDataSection(Size, Alignment, SectionID, - SectionName, IsReadOnly); + uint8_t *Addr = ClientMM->allocateDataSection(Size, Alignment, SectionID, + SectionName, IsReadOnly); M.SectionsAllocatedSinceLastLoad.insert(Addr); return Addr; } void reserveAllocationSpace(uintptr_t CodeSize, uintptr_t DataSizeRO, uintptr_t DataSizeRW) override { - return M.MM->reserveAllocationSpace(CodeSize, DataSizeRO, DataSizeRW); + return ClientMM->reserveAllocationSpace(CodeSize, DataSizeRO, + DataSizeRW); } bool needsToReserveAllocationSpace() override { - return M.MM->needsToReserveAllocationSpace(); + return ClientMM->needsToReserveAllocationSpace(); } void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) override { - return M.MM->registerEHFrames(Addr, LoadAddr, Size); + return ClientMM->registerEHFrames(Addr, LoadAddr, Size); } void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) override { - return M.MM->deregisterEHFrames(Addr, LoadAddr, Size); - } - - uint64_t getSymbolAddress(const std::string &Name) override { - return M.getSymbolAddressWithoutMangling(Name); - } - - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override { - return M.MM->getPointerToNamedFunction(Name, AbortOnFailure); + return ClientMM->deregisterEHFrames(Addr, LoadAddr, Size); } void notifyObjectLoaded(ExecutionEngine *EE, const object::ObjectFile &O) override { - return M.MM->notifyObjectLoaded(EE, O); + return ClientMM->notifyObjectLoaded(EE, O); } bool finalizeMemory(std::string *ErrMsg = nullptr) override { @@ -96,21 +94,41 @@ class OrcMCJITReplacement : public ExecutionEngine { // get more than one set of objects loaded but not yet finalized is if // they were loaded during relocation of another set. if (M.UnfinalizedSections.size() == 1) - return M.MM->finalizeMemory(ErrMsg); + return ClientMM->finalizeMemory(ErrMsg); return false; } private: OrcMCJITReplacement &M; + std::shared_ptr ClientMM; + }; + + class LinkingResolver : public RuntimeDyld::SymbolResolver { + public: + LinkingResolver(OrcMCJITReplacement &M) : M(M) {} + + RuntimeDyld::SymbolInfo findSymbol(const std::string &Name) override { + return M.findMangledSymbol(Name); + } + + RuntimeDyld::SymbolInfo + findSymbolInLogicalDylib(const std::string &Name) override { + return M.ClientResolver->findSymbolInLogicalDylib(Name); + } + + private: + OrcMCJITReplacement &M; }; private: static ExecutionEngine * createOrcMCJITReplacement(std::string *ErrorMsg, - std::unique_ptr OrcJMM, + std::shared_ptr MemMgr, + std::shared_ptr Resolver, std::unique_ptr TM) { - return new OrcMCJITReplacement(std::move(OrcJMM), std::move(TM)); + return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver), + std::move(TM)); } public: @@ -118,12 +136,15 @@ public: OrcMCJITReplacementCtor = createOrcMCJITReplacement; } - OrcMCJITReplacement(std::unique_ptr MM, - std::unique_ptr TM) - : TM(std::move(TM)), MM(std::move(MM)), Mang(this->TM->getDataLayout()), + OrcMCJITReplacement( + std::shared_ptr MemMgr, + std::shared_ptr ClientResolver, + std::unique_ptr TM) + : TM(std::move(TM)), MemMgr(*this, std::move(MemMgr)), + Resolver(*this), ClientResolver(std::move(ClientResolver)), + Mang(this->TM->getDataLayout()), NotifyObjectLoaded(*this), NotifyFinalized(*this), - ObjectLayer(ObjectLayerT::CreateRTDyldMMFtor(), NotifyObjectLoaded, - NotifyFinalized), + ObjectLayer(NotifyObjectLoaded, NotifyFinalized), CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)), LazyEmitLayer(CompileLayer) { setDataLayout(this->TM->getDataLayout()); @@ -139,15 +160,13 @@ public: Modules.push_back(std::move(M)); std::vector Ms; Ms.push_back(&*Modules.back()); - LazyEmitLayer.addModuleSet(std::move(Ms), - llvm::make_unique(*this)); + LazyEmitLayer.addModuleSet(std::move(Ms), &MemMgr, &Resolver); } void addObjectFile(std::unique_ptr O) override { std::vector> Objs; Objs.push_back(std::move(O)); - ObjectLayer.addObjectSet(std::move(Objs), - llvm::make_unique(*this)); + ObjectLayer.addObjectSet(std::move(Objs), &MemMgr, &Resolver); } void addObjectFile(object::OwningBinary O) override { @@ -157,8 +176,7 @@ public: std::vector> Objs; Objs.push_back(std::move(Obj)); auto H = - ObjectLayer.addObjectSet(std::move(Objs), - llvm::make_unique(*this)); + ObjectLayer.addObjectSet(std::move(Objs), &MemMgr, &Resolver); std::vector> Bufs; Bufs.push_back(std::move(Buf)); @@ -170,7 +188,11 @@ public: } uint64_t getSymbolAddress(StringRef Name) { - return getSymbolAddressWithoutMangling(Mangle(Name)); + return findSymbol(Name).getAddress(); + } + + RuntimeDyld::SymbolInfo findSymbol(StringRef Name) { + return findMangledSymbol(Mangle(Name)); } void finalizeObject() override { @@ -214,18 +236,19 @@ public: } private: - uint64_t getSymbolAddressWithoutMangling(StringRef Name) { - if (uint64_t Addr = LazyEmitLayer.findSymbol(Name, false).getAddress()) - return Addr; - if (uint64_t Addr = MM->getSymbolAddress(Name)) - return Addr; - if (uint64_t Addr = scanArchives(Name)) - return Addr; - return 0; + RuntimeDyld::SymbolInfo findMangledSymbol(StringRef Name) { + if (auto Sym = LazyEmitLayer.findSymbol(Name, false)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), Sym.getFlags()); + if (auto Sym = ClientResolver->findSymbol(Name)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), Sym.getFlags()); + if (auto Sym = scanArchives(Name)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), Sym.getFlags()); + + return nullptr; } - uint64_t scanArchives(StringRef Name) { + JITSymbol scanArchives(StringRef Name) { for (object::OwningBinary &OB : Archives) { object::Archive *A = OB.getBinary(); // Look for our symbols in each Archive @@ -241,14 +264,13 @@ private: std::vector> ObjSet; ObjSet.push_back(std::unique_ptr( static_cast(ChildBin.release()))); - ObjectLayer.addObjectSet( - std::move(ObjSet), llvm::make_unique(*this)); - if (uint64_t Addr = ObjectLayer.findSymbol(Name, true).getAddress()) - return Addr; + ObjectLayer.addObjectSet(std::move(ObjSet), &MemMgr, &Resolver); + if (auto Sym = ObjectLayer.findSymbol(Name, true)) + return Sym; } } } - return 0; + return nullptr; } class NotifyObjectLoadedT { @@ -267,7 +289,7 @@ private: assert(Objects.size() == Infos.size() && "Incorrect number of Infos for Objects."); for (unsigned I = 0; I < Objects.size(); ++I) - M.MM->notifyObjectLoaded(&M, *Objects[I]); + M.MemMgr.notifyObjectLoaded(&M, *Objects[I]); }; private: @@ -299,7 +321,9 @@ private: typedef LazyEmittingLayer LazyEmitLayerT; std::unique_ptr TM; - std::unique_ptr MM; + MCJITReplacementMemMgr MemMgr; + LinkingResolver Resolver; + std::shared_ptr ClientResolver; Mangler Mang; NotifyObjectLoadedT NotifyObjectLoaded; diff --git a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp index 6fe5301..fc56e67 100644 --- a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp @@ -6,39 +6,6 @@ using namespace llvm::orc; namespace { -std::array X86GPRsToSave = {{ - "rbp", "rbx", "r12", "r13", "r14", "r15", // Callee saved. - "rdi", "rsi", "rdx", "rcx", "r8", "r9", // Int args. -}}; - -std::array X86XMMsToSave = {{ - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" // FP args -}}; - -template unsigned saveX86Regs(OStream &OS) { - for (const auto &GPR : X86GPRsToSave) - OS << " pushq %" << GPR << "\n"; - - OS << " subq $" << (16 * X86XMMsToSave.size()) << ", %rsp\n"; - - for (unsigned i = 0; i < X86XMMsToSave.size(); ++i) - OS << " movdqu %" << X86XMMsToSave[i] << ", " - << (16 * (X86XMMsToSave.size() - i - 1)) << "(%rsp)\n"; - - return (8 * X86GPRsToSave.size()) + (16 * X86XMMsToSave.size()); -} - -template void restoreX86Regs(OStream &OS) { - for (unsigned i = 0; i < X86XMMsToSave.size(); ++i) - OS << " movdqu " << (16 * i) << "(%rsp), %" - << X86XMMsToSave[(X86XMMsToSave.size() - i - 1)] << "\n"; - OS << " addq $" << (16 * X86XMMsToSave.size()) << ", %rsp\n"; - - for (unsigned i = 0; i < X86GPRsToSave.size(); ++i) - OS << " popq %" << X86GPRsToSave[X86GPRsToSave.size() - i - 1] << "\n"; -} - -template uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM, TargetAddress CallbackID) { return JCBM->executeCompileCallback(CallbackID); @@ -53,14 +20,28 @@ const char* OrcX86_64::ResolverBlockName = "orc_resolver_block"; void OrcX86_64::insertResolverBlock( Module &M, JITCompileCallbackManagerBase &JCBM) { + + // Trampoline code-sequence length, used to get trampoline address from return + // address. const unsigned X86_64_TrampolineLength = 6; - auto CallbackPtr = executeCompileCallback; + + // List of x86-64 GPRs to save. Note - RBP saved separately below. + std::array GPRs = {{ + "rax", "rbx", "rcx", "rdx", + "rsi", "rdi", "r8", "r9", + "r10", "r11", "r12", "r13", + "r14", "r15" + }}; + + // Address of the executeCompileCallback function. uint64_t CallbackAddr = - static_cast(reinterpret_cast(CallbackPtr)); + static_cast( + reinterpret_cast(executeCompileCallback)); std::ostringstream AsmStream; Triple TT(M.getTargetTriple()); + // Switch to text section. if (TT.getOS() == Triple::Darwin) AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" << ".align 4, 0x90\n"; @@ -68,24 +49,51 @@ void OrcX86_64::insertResolverBlock( AsmStream << ".text\n" << ".align 16, 0x90\n"; + // Bake in a pointer to the callback manager immediately before the + // start of the resolver function. AsmStream << "jit_callback_manager_addr:\n" - << " .quad " << &JCBM << "\n" - << ResolverBlockName << ":\n"; - - uint64_t ReturnAddrOffset = saveX86Regs(AsmStream); - - // Compute index, load object address, and call JIT. - AsmStream << " leaq jit_callback_manager_addr(%rip), %rdi\n" + << " .quad " << &JCBM << "\n"; + + // Start the resolver function. + AsmStream << ResolverBlockName << ":\n" + << " pushq %rbp\n" + << " movq %rsp, %rbp\n"; + + // Store the GPRs. + for (const auto &GPR : GPRs) + AsmStream << " pushq %" << GPR << "\n"; + + // Store floating-point state with FXSAVE. + // Note: We need to keep the stack 16-byte aligned, so if we've emitted an odd + // number of 64-bit pushes so far (GPRs.size() plus 1 for RBP) then add + // an extra 64 bits of padding to the FXSave area. + unsigned Padding = (GPRs.size() + 1) % 2 ? 8 : 0; + unsigned FXSaveSize = 512 + Padding; + AsmStream << " subq $" << FXSaveSize << ", %rsp\n" + << " fxsave (%rsp)\n" + + // Load callback manager address, compute trampoline address, call JIT. + << " lea jit_callback_manager_addr(%rip), %rdi\n" << " movq (%rdi), %rdi\n" - << " movq " << ReturnAddrOffset << "(%rsp), %rsi\n" + << " movq 0x8(%rbp), %rsi\n" << " subq $" << X86_64_TrampolineLength << ", %rsi\n" << " movabsq $" << CallbackAddr << ", %rax\n" << " callq *%rax\n" - << " movq %rax, " << ReturnAddrOffset << "(%rsp)\n"; - restoreX86Regs(AsmStream); + // Replace the return to the trampoline with the return address of the + // compiled function body. + << " movq %rax, 0x8(%rbp)\n" + + // Restore the floating point state. + << " fxrstor (%rsp)\n" + << " addq $" << FXSaveSize << ", %rsp\n"; + + for (const auto &GPR : make_range(GPRs.rbegin(), GPRs.rend())) + AsmStream << " popq %" << GPR << "\n"; - AsmStream << " retq\n"; + // Restore original RBP and return to compiled function body. + AsmStream << " popq %rbp\n" + << " retq\n"; M.appendModuleInlineAsm(AsmStream.str()); } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index a0ed7cf..a13ecb7 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -57,7 +57,8 @@ static void dumpSectionMemory(const SectionEntry &S, StringRef State) { unsigned BytesRemaining = S.Size; if (StartPadding) { - dbgs() << "\n" << format("0x%016" PRIx64, LoadAddr & ~(ColsPerRow - 1)) << ":"; + dbgs() << "\n" << format("0x%016" PRIx64, + LoadAddr & ~(uint64_t)(ColsPerRow - 1)) << ":"; while (StartPadding--) dbgs() << " "; } @@ -92,7 +93,7 @@ void RuntimeDyldImpl::resolveRelocations() { // entry provides the section to which the relocation will be applied. uint64_t Addr = Sections[i].LoadAddress; DEBUG(dbgs() << "Resolving relocations Section #" << i << "\t" - << format("0x%x", Addr) << "\n"); + << format("%p", (uintptr_t)Addr) << "\n"); DEBUG(dumpSectionMemory(Sections[i], "before relocations")); resolveRelocationList(Relocations[i], Addr); DEBUG(dumpSectionMemory(Sections[i], "after relocations")); @@ -151,10 +152,10 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { // Compute the memory size required to load all sections to be loaded // and pass this information to the memory manager - if (MemMgr->needsToReserveAllocationSpace()) { + if (MemMgr.needsToReserveAllocationSpace()) { uint64_t CodeSize = 0, DataSizeRO = 0, DataSizeRW = 0; computeTotalAllocSize(Obj, CodeSize, DataSizeRO, DataSizeRW); - MemMgr->reserveAllocationSpace(CodeSize, DataSizeRO, DataSizeRW); + MemMgr.reserveAllocationSpace(CodeSize, DataSizeRO, DataSizeRW); } // Used sections from the object file @@ -360,19 +361,20 @@ void RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, if (Name == ".eh_frame") SectionSize += 4; - if (SectionSize > 0) { - // save the total size of the section - if (IsCode) { - CodeSectionSizes.push_back(SectionSize); - } else if (IsReadOnly) { - ROSectionSizes.push_back(SectionSize); - } else { - RWSectionSizes.push_back(SectionSize); - } - // update the max alignment - if (Alignment > MaxAlignment) { - MaxAlignment = Alignment; - } + if (!SectionSize) + SectionSize = 1; + + if (IsCode) { + CodeSectionSizes.push_back(SectionSize); + } else if (IsReadOnly) { + ROSectionSizes.push_back(SectionSize); + } else { + RWSectionSizes.push_back(SectionSize); + } + + // update the max alignment + if (Alignment > MaxAlignment) { + MaxAlignment = Alignment; } } } @@ -485,7 +487,7 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, // Skip common symbols already elsewhere. if (GlobalSymbolTable.count(Name) || - MemMgr->getSymbolAddressInLogicalDylib(Name)) { + Resolver.findSymbolInLogicalDylib(Name)) { DEBUG(dbgs() << "\tSkipping already emitted common symbol '" << Name << "'\n"); continue; @@ -502,8 +504,8 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, // Allocate memory for the section unsigned SectionID = Sections.size(); - uint8_t *Addr = MemMgr->allocateDataSection(CommonSize, sizeof(void *), - SectionID, StringRef(), false); + uint8_t *Addr = MemMgr.allocateDataSection(CommonSize, sizeof(void *), + SectionID, StringRef(), false); if (!Addr) report_fatal_error("Unable to allocate memory for common symbols!"); uint64_t Offset = 0; @@ -577,10 +579,12 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, if (IsRequired) { Check(Section.getContents(data)); Allocate = DataSize + PaddingSize + StubBufSize; - Addr = IsCode ? MemMgr->allocateCodeSection(Allocate, Alignment, SectionID, - Name) - : MemMgr->allocateDataSection(Allocate, Alignment, SectionID, - Name, IsReadOnly); + if (!Allocate) + Allocate = 1; + Addr = IsCode ? MemMgr.allocateCodeSection(Allocate, Alignment, SectionID, + Name) + : MemMgr.allocateDataSection(Allocate, Alignment, SectionID, + Name, IsReadOnly); if (!Addr) report_fatal_error("Unable to allocate section memory!"); @@ -787,9 +791,9 @@ void RuntimeDyldImpl::resolveExternalSymbols() { uint64_t Addr = 0; RTDyldSymbolTable::const_iterator Loc = GlobalSymbolTable.find(Name); if (Loc == GlobalSymbolTable.end()) { - // This is an external symbol, try to get its address from - // MemoryManager. - Addr = MemMgr->getSymbolAddress(Name.data()); + // This is an external symbol, try to get its address from the symbol + // resolver. + Addr = Resolver.findSymbol(Name.data()).getAddress(); // The call to getSymbolAddress may have caused additional modules to // be loaded, which may have added new entries to the // ExternalSymbolRelocations map. Consquently, we need to update our @@ -810,7 +814,6 @@ void RuntimeDyldImpl::resolveExternalSymbols() { report_fatal_error("Program used external function '" + Name + "' which could not be resolved!"); - updateGOTEntries(Name, Addr); DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t" << format("0x%lx", Addr) << "\n"); // This list may have been updated when we called getSymbolAddress, so @@ -835,7 +838,12 @@ uint64_t RuntimeDyld::LoadedObjectInfo::getSectionLoadAddress( return 0; } -RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { +void RuntimeDyld::MemoryManager::anchor() {} +void RuntimeDyld::SymbolResolver::anchor() {} + +RuntimeDyld::RuntimeDyld(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) + : MemMgr(MemMgr), Resolver(Resolver) { // FIXME: There's a potential issue lurking here if a single instance of // RuntimeDyld is used to load multiple objects. The current implementation // associates a single memory manager with a RuntimeDyld instance. Even @@ -843,7 +851,6 @@ RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { // they share a single memory manager. This can become a problem when page // permissions are applied. Dyld = nullptr; - MM = mm; ProcessAllSections = false; Checker = nullptr; } @@ -851,27 +858,33 @@ RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { RuntimeDyld::~RuntimeDyld() {} static std::unique_ptr -createRuntimeDyldCOFF(Triple::ArchType Arch, RTDyldMemoryManager *MM, +createRuntimeDyldCOFF(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver, bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { - std::unique_ptr Dyld(RuntimeDyldCOFF::create(Arch, MM)); + std::unique_ptr Dyld = + RuntimeDyldCOFF::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); Dyld->setRuntimeDyldChecker(Checker); return Dyld; } static std::unique_ptr -createRuntimeDyldELF(RTDyldMemoryManager *MM, bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { - std::unique_ptr Dyld(new RuntimeDyldELF(MM)); +createRuntimeDyldELF(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver, + bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { + std::unique_ptr Dyld(new RuntimeDyldELF(MM, Resolver)); Dyld->setProcessAllSections(ProcessAllSections); Dyld->setRuntimeDyldChecker(Checker); return Dyld; } static std::unique_ptr -createRuntimeDyldMachO(Triple::ArchType Arch, RTDyldMemoryManager *MM, - bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { - std::unique_ptr Dyld(RuntimeDyldMachO::create(Arch, MM)); +createRuntimeDyldMachO(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver, + bool ProcessAllSections, + RuntimeDyldCheckerImpl *Checker) { + std::unique_ptr Dyld = + RuntimeDyldMachO::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); Dyld->setRuntimeDyldChecker(Checker); return Dyld; @@ -881,14 +894,14 @@ std::unique_ptr RuntimeDyld::loadObject(const ObjectFile &Obj) { if (!Dyld) { if (Obj.isELF()) - Dyld = createRuntimeDyldELF(MM, ProcessAllSections, Checker); + Dyld = createRuntimeDyldELF(MemMgr, Resolver, ProcessAllSections, Checker); else if (Obj.isMachO()) Dyld = createRuntimeDyldMachO( - static_cast(Obj.getArch()), MM, + static_cast(Obj.getArch()), MemMgr, Resolver, ProcessAllSections, Checker); else if (Obj.isCOFF()) Dyld = createRuntimeDyldCOFF( - static_cast(Obj.getArch()), MM, + static_cast(Obj.getArch()), MemMgr, Resolver, ProcessAllSections, Checker); else report_fatal_error("Incompatible object format!"); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 56bcb8e..8055d55 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -40,13 +40,15 @@ public: namespace llvm { std::unique_ptr -llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, RTDyldMemoryManager *MM) { +llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, + RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) { switch (Arch) { default: llvm_unreachable("Unsupported target for RuntimeDyldCOFF."); break; case Triple::x86_64: - return make_unique(MM); + return make_unique(MemMgr, Resolver); } } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h index 681a3e5..32b8fa2 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h @@ -31,11 +31,15 @@ public: std::unique_ptr loadObject(const object::ObjectFile &Obj) override; bool isCompatibleFile(const object::ObjectFile &Obj) const override; - static std::unique_ptr create(Triple::ArchType Arch, - RTDyldMemoryManager *MM); + + static std::unique_ptr + create(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver); protected: - RuntimeDyldCOFF(RTDyldMemoryManager *MM) : RuntimeDyldImpl(MM) {} + RuntimeDyldCOFF(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldImpl(MemMgr, Resolver) {} uint64_t getSymbolOffset(const SymbolRef &Sym); }; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index c991408..957571b 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -738,7 +738,7 @@ uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { if (auto InternalSymbol = getRTDyld().getSymbol(Symbol)) return InternalSymbol.getAddress(); - return getRTDyld().MemMgr->getSymbolAddress(Symbol); + return getRTDyld().Resolver.findSymbol(Symbol).getAddress(); } uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h index e8d299a..69d2a7d 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -19,6 +19,7 @@ class RuntimeDyldCheckerImpl { friend class RuntimeDyldChecker; friend class RuntimeDyldImpl; friend class RuntimeDyldCheckerExprEval; + friend class RuntimeDyldELF; public: RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 6278170..bbffdfb 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "RuntimeDyldELF.h" +#include "RuntimeDyldCheckerImpl.h" #include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" @@ -183,32 +184,30 @@ LoadedELFObjectInfo::getObjectForDebug(const ObjectFile &Obj) const { namespace llvm { -RuntimeDyldELF::RuntimeDyldELF(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {} +RuntimeDyldELF::RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldImpl(MemMgr, Resolver), GOTSectionID(0), CurrentGOTIndex(0) {} RuntimeDyldELF::~RuntimeDyldELF() {} void RuntimeDyldELF::registerEHFrames() { - if (!MemMgr) - return; for (int i = 0, e = UnregisteredEHFrameSections.size(); i != e; ++i) { SID EHFrameSID = UnregisteredEHFrameSections[i]; uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; size_t EHFrameSize = Sections[EHFrameSID].Size; - MemMgr->registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); + MemMgr.registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); RegisteredEHFrameSections.push_back(EHFrameSID); } UnregisteredEHFrameSections.clear(); } void RuntimeDyldELF::deregisterEHFrames() { - if (!MemMgr) - return; for (int i = 0, e = RegisteredEHFrameSections.size(); i != e; ++i) { SID EHFrameSID = RegisteredEHFrameSections[i]; uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; size_t EHFrameSize = Sections[EHFrameSID].Size; - MemMgr->deregisterEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); + MemMgr.deregisterEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); } RegisteredEHFrameSections.clear(); } @@ -247,27 +246,16 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, << format("%p\n", Section.Address + Offset)); break; } - case ELF::R_X86_64_GOTPCREL: { - // findGOTEntry returns the 'G + GOT' part of the relocation calculation - // based on the load/target address of the GOT (not the current/local addr). - uint64_t GOTAddr = findGOTEntry(Value, SymOffset); - uint64_t FinalAddress = Section.LoadAddress + Offset; - // The processRelocationRef method combines the symbol offset and the addend - // and in most cases that's what we want. For this relocation type, we need - // the raw addend, so we subtract the symbol offset to get it. - int64_t RealOffset = GOTAddr + Addend - SymOffset - FinalAddress; - assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN); - int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); - support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; - break; - } case ELF::R_X86_64_PC32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version support::ulittle32_t::ref Placeholder( (void *)(Section.ObjAddress + Offset)); uint64_t FinalAddress = Section.LoadAddress + Offset; - int64_t RealOffset = Placeholder + Value + Addend - FinalAddress; + int64_t RealOffset = Value + Addend - FinalAddress; + // Don't add the placeholder if this is a stub + if (Offset < Section.Size) + RealOffset += Placeholder; assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN); int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; @@ -279,8 +267,10 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, support::ulittle64_t::ref Placeholder( (void *)(Section.ObjAddress + Offset)); uint64_t FinalAddress = Section.LoadAddress + Offset; - support::ulittle64_t::ref(Section.Address + Offset) = - Placeholder + Value + Addend - FinalAddress; + int64_t RealOffset = Value + Addend - FinalAddress; + if (Offset < Section.Size) + RealOffset += Placeholder; + support::ulittle64_t::ref(Section.Address + Offset) = RealOffset; break; } } @@ -1325,16 +1315,18 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( Stubs[Value] = StubOffset; createStubFunction((uint8_t *)StubAddress); - // Create a GOT entry for the external function. - GOTEntries.push_back(Value); - - // Make our stub function a relative call to the GOT entry. - RelocationEntry RE(SectionID, StubOffset + 2, ELF::R_X86_64_GOTPCREL, - -4); - addRelocationForSymbol(RE, Value.SymbolName); - // Bump our stub offset counter Section.StubOffset = StubOffset + getMaxStubSize(); + + // Allocate a GOT Entry + uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); + + // The load of the GOT address has an addend of -4 + resolveGOTOffsetRelocation(SectionID, StubOffset + 2, GOTOffset - 4); + + // Fill in the value of the symbol we're targeting into the GOT + addRelocationForSymbol(computeGOTOffsetRE(SectionID,GOTOffset,0,ELF::R_X86_64_64), + Value.SymbolName); } // Make the target call a call into the stub table. @@ -1345,10 +1337,17 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( Value.Offset); addRelocationForSection(RE, Value.SectionID); } + } else if (Arch == Triple::x86_64 && RelType == ELF::R_X86_64_GOTPCREL) { + uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); + resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend); + + // Fill in the value of the symbol we're targeting into the GOT + RelocationEntry RE = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_64); + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); } else { - if (Arch == Triple::x86_64 && RelType == ELF::R_X86_64_GOTPCREL) { - GOTEntries.push_back(Value); - } RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset); if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); @@ -1358,22 +1357,6 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( return ++RelI; } -void RuntimeDyldELF::updateGOTEntries(StringRef Name, uint64_t Addr) { - - SmallVectorImpl>::iterator it; - SmallVectorImpl>::iterator end = GOTs.end(); - - for (it = GOTs.begin(); it != end; ++it) { - GOTRelocations &GOTEntries = it->second; - for (int i = 0, e = GOTEntries.size(); i != e; ++i) { - if (GOTEntries[i].SymbolName != nullptr && - GOTEntries[i].SymbolName == Name) { - GOTEntries[i].Offset = Addr; - } - } - } -} - size_t RuntimeDyldELF::getGOTEntrySize() { // We don't use the GOT in all of these cases, but it's essentially free // to put them all here. @@ -1400,83 +1383,53 @@ size_t RuntimeDyldELF::getGOTEntrySize() { return Result; } -uint64_t RuntimeDyldELF::findGOTEntry(uint64_t LoadAddress, uint64_t Offset) { - - const size_t GOTEntrySize = getGOTEntrySize(); - - SmallVectorImpl>::const_iterator it; - SmallVectorImpl>::const_iterator end = - GOTs.end(); - - int GOTIndex = -1; - for (it = GOTs.begin(); it != end; ++it) { - SID GOTSectionID = it->first; - const GOTRelocations &GOTEntries = it->second; - - // Find the matching entry in our vector. - uint64_t SymbolOffset = 0; - for (int i = 0, e = GOTEntries.size(); i != e; ++i) { - if (!GOTEntries[i].SymbolName) { - if (getSectionLoadAddress(GOTEntries[i].SectionID) == LoadAddress && - GOTEntries[i].Offset == Offset) { - GOTIndex = i; - SymbolOffset = GOTEntries[i].Offset; - break; - } - } else { - // GOT entries for external symbols use the addend as the address when - // the external symbol has been resolved. - if (GOTEntries[i].Offset == LoadAddress) { - GOTIndex = i; - // Don't use the Addend here. The relocation handler will use it. - break; - } - } - } - - if (GOTIndex != -1) { - if (GOTEntrySize == sizeof(uint64_t)) { - uint64_t *LocalGOTAddr = (uint64_t *)getSectionAddress(GOTSectionID); - // Fill in this entry with the address of the symbol being referenced. - LocalGOTAddr[GOTIndex] = LoadAddress + SymbolOffset; - } else { - uint32_t *LocalGOTAddr = (uint32_t *)getSectionAddress(GOTSectionID); - // Fill in this entry with the address of the symbol being referenced. - LocalGOTAddr[GOTIndex] = (uint32_t)(LoadAddress + SymbolOffset); - } - - // Calculate the load address of this entry - return getSectionLoadAddress(GOTSectionID) + (GOTIndex * GOTEntrySize); - } +uint64_t RuntimeDyldELF::allocateGOTEntries(unsigned SectionID, unsigned no) +{ + (void)SectionID; // The GOT Section is the same for all section in the object file + if (GOTSectionID == 0) { + GOTSectionID = Sections.size(); + // Reserve a section id. We'll allocate the section later + // once we know the total size + Sections.push_back(SectionEntry(".got", 0, 0, 0)); } + uint64_t StartOffset = CurrentGOTIndex * getGOTEntrySize(); + CurrentGOTIndex += no; + return StartOffset; +} - assert(GOTIndex != -1 && "Unable to find requested GOT entry."); - return 0; +void RuntimeDyldELF::resolveGOTOffsetRelocation(unsigned SectionID, uint64_t Offset, uint64_t GOTOffset) +{ + // Fill in the relative address of the GOT Entry into the stub + RelocationEntry GOTRE(SectionID, Offset, ELF::R_X86_64_PC32, GOTOffset); + addRelocationForSection(GOTRE, GOTSectionID); +} + +RelocationEntry RuntimeDyldELF::computeGOTOffsetRE(unsigned SectionID, uint64_t GOTOffset, uint64_t SymbolOffset, + uint32_t Type) +{ + (void)SectionID; // The GOT Section is the same for all section in the object file + return RelocationEntry(GOTSectionID, GOTOffset, Type, SymbolOffset); } void RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) { // If necessary, allocate the global offset table - if (MemMgr) { - // Allocate the GOT if necessary - size_t numGOTEntries = GOTEntries.size(); - if (numGOTEntries != 0) { - // Allocate memory for the section - unsigned SectionID = Sections.size(); - size_t TotalSize = numGOTEntries * getGOTEntrySize(); - uint8_t *Addr = MemMgr->allocateDataSection(TotalSize, getGOTEntrySize(), - SectionID, ".got", false); - if (!Addr) - report_fatal_error("Unable to allocate memory for GOT!"); - - GOTs.push_back(std::make_pair(SectionID, GOTEntries)); - Sections.push_back(SectionEntry(".got", Addr, TotalSize, 0)); - // For now, initialize all GOT entries to zero. We'll fill them in as - // needed when GOT-based relocations are applied. - memset(Addr, 0, TotalSize); - } - } else { - report_fatal_error("Unable to allocate memory for GOT!"); + if (GOTSectionID != 0) { + // Allocate memory for the section + size_t TotalSize = CurrentGOTIndex * getGOTEntrySize(); + uint8_t *Addr = MemMgr.allocateDataSection(TotalSize, getGOTEntrySize(), + GOTSectionID, ".got", false); + if (!Addr) + report_fatal_error("Unable to allocate memory for GOT!"); + + Sections[GOTSectionID] = SectionEntry(".got", Addr, TotalSize, 0); + + if (Checker) + Checker->registerSection(Obj.getFileName(), GOTSectionID); + + // For now, initialize all GOT entries to zero. We'll fill them in as + // needed when GOT-based relocations are applied. + memset(Addr, 0, TotalSize); } // Look for and record the EH frame section. @@ -1490,6 +1443,9 @@ void RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, break; } } + + GOTSectionID = 0; + CurrentGOTIndex = 0; } bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const { diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index 71260d0..590d26a 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -80,16 +80,32 @@ class RuntimeDyldELF : public RuntimeDyldImpl { ObjSectionToIDMap &LocalSections, RelocationValueRef &Rel); - uint64_t findGOTEntry(uint64_t LoadAddr, uint64_t Offset); size_t getGOTEntrySize(); - void updateGOTEntries(StringRef Name, uint64_t Addr) override; + SectionEntry &getSection(unsigned SectionID) { return Sections[SectionID]; } - // Relocation entries for symbols whose position-independent offset is - // updated in a global offset table. - typedef SmallVector GOTRelocations; - GOTRelocations GOTEntries; // List of entries requiring finalization. - SmallVector, 8> GOTs; // Allocated tables. + // Allocate no GOT entries for use in the given section. + uint64_t allocateGOTEntries(unsigned SectionID, unsigned no); + + // Resolve the relvative address of GOTOffset in Section ID and place + // it at the given Offset + void resolveGOTOffsetRelocation(unsigned SectionID, uint64_t Offset, + uint64_t GOTOffset); + + // For a GOT entry referenced from SectionID, compute a relocation entry + // that will place the final resolved value in the GOT slot + RelocationEntry computeGOTOffsetRE(unsigned SectionID, + uint64_t GOTOffset, + uint64_t SymbolOffset, + unsigned Type); + + // The tentative ID for the GOT section + unsigned GOTSectionID; + + // Records the current number of allocated slots in the GOT + // (This would be equivalent to GOTEntries.size() were it not for relocations + // that consume more than one slot) + unsigned CurrentGOTIndex; // When a module is loaded we save the SectionID of the EH frame section // in a table until we receive a request to register all unregistered @@ -98,8 +114,9 @@ class RuntimeDyldELF : public RuntimeDyldImpl { SmallVector RegisteredEHFrameSections; public: - RuntimeDyldELF(RTDyldMemoryManager *mm); - virtual ~RuntimeDyldELF(); + RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver); + ~RuntimeDyldELF() override; std::unique_ptr loadObject(const object::ObjectFile &O) override; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 05060dd..ee51a75 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -18,6 +18,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" #include "llvm/Object/ObjectFile.h" @@ -51,7 +52,7 @@ class Twine; class SectionEntry { public: /// Name - section name. - StringRef Name; + std::string Name; /// Address - address in the linker's memory where the section resides. uint8_t *Address; @@ -188,7 +189,10 @@ class RuntimeDyldImpl { friend class RuntimeDyldCheckerImpl; protected: // The MemoryManager to load objects into. - RTDyldMemoryManager *MemMgr; + RuntimeDyld::MemoryManager &MemMgr; + + // The symbol resolver to use for external symbols. + RuntimeDyld::SymbolResolver &Resolver; // Attached RuntimeDyldChecker instance. Null if no instance attached. RuntimeDyldCheckerImpl *Checker; @@ -357,10 +361,6 @@ protected: /// \brief Resolve relocations to external symbols. void resolveExternalSymbols(); - /// \brief Update GOT entries for external symbols. - // The base class does nothing. ELF overrides this. - virtual void updateGOTEntries(StringRef Name, uint64_t Addr) {} - // \brief Compute an upper bound of the memory that is required to load all // sections void computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, @@ -374,8 +374,10 @@ protected: std::pair loadObjectImpl(const object::ObjectFile &Obj); public: - RuntimeDyldImpl(RTDyldMemoryManager *mm) - : MemMgr(mm), Checker(nullptr), ProcessAllSections(false), HasError(false) { + RuntimeDyldImpl(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) + : MemMgr(MemMgr), Resolver(Resolver), Checker(nullptr), + ProcessAllSections(false), HasError(false) { } virtual ~RuntimeDyldImpl(); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index 2d39662..675063c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -178,25 +178,30 @@ bool RuntimeDyldMachO::isCompatibleFile(const object::ObjectFile &Obj) const { } template -void RuntimeDyldMachOCRTPBase::finalizeLoad(const ObjectFile &ObjImg, +void RuntimeDyldMachOCRTPBase::finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) { unsigned EHFrameSID = RTDYLD_INVALID_SECTION_ID; unsigned TextSID = RTDYLD_INVALID_SECTION_ID; unsigned ExceptTabSID = RTDYLD_INVALID_SECTION_ID; - ObjSectionToIDMap::iterator i, e; - for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) { - const SectionRef &Section = i->first; + for (const auto &Section : Obj.sections()) { StringRef Name; Section.getName(Name); - if (Name == "__eh_frame") - EHFrameSID = i->second; - else if (Name == "__text") - TextSID = i->second; + + // Force emission of the __text, __eh_frame, and __gcc_except_tab sections + // if they're present. Otherwise call down to the impl to handle other + // sections that have already been emitted. + if (Name == "__text") + TextSID = findOrEmitSection(Obj, Section, true, SectionMap); + else if (Name == "__eh_frame") + EHFrameSID = findOrEmitSection(Obj, Section, false, SectionMap); else if (Name == "__gcc_except_tab") - ExceptTabSID = i->second; - else - impl().finalizeSection(ObjImg, i->second, Section); + ExceptTabSID = findOrEmitSection(Obj, Section, true, SectionMap); + else { + auto I = SectionMap.find(Section); + if (I != SectionMap.end()) + impl().finalizeSection(Obj, I->second, Section); + } } UnregisteredEHFrameSections.push_back( EHFrameRelatedSections(EHFrameSID, TextSID, ExceptTabSID)); @@ -239,7 +244,8 @@ unsigned char *RuntimeDyldMachOCRTPBase::processFDE(unsigned char *P, } static int64_t computeDelta(SectionEntry *A, SectionEntry *B) { - int64_t ObjDistance = A->ObjAddress - B->ObjAddress; + int64_t ObjDistance = + static_cast(A->ObjAddress) - static_cast(B->ObjAddress); int64_t MemDistance = A->LoadAddress - B->LoadAddress; return ObjDistance - MemDistance; } @@ -247,8 +253,6 @@ static int64_t computeDelta(SectionEntry *A, SectionEntry *B) { template void RuntimeDyldMachOCRTPBase::registerEHFrames() { - if (!MemMgr) - return; for (int i = 0, e = UnregisteredEHFrameSections.size(); i != e; ++i) { EHFrameRelatedSections &SectionInfo = UnregisteredEHFrameSections[i]; if (SectionInfo.EHFrameSID == RTDYLD_INVALID_SECTION_ID || @@ -271,22 +275,28 @@ void RuntimeDyldMachOCRTPBase::registerEHFrames() { P = processFDE(P, DeltaForText, DeltaForEH); } while (P != End); - MemMgr->registerEHFrames(EHFrame->Address, EHFrame->LoadAddress, - EHFrame->Size); + MemMgr.registerEHFrames(EHFrame->Address, EHFrame->LoadAddress, + EHFrame->Size); } UnregisteredEHFrameSections.clear(); } std::unique_ptr -RuntimeDyldMachO::create(Triple::ArchType Arch, RTDyldMemoryManager *MM) { +RuntimeDyldMachO::create(Triple::ArchType Arch, + RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) { switch (Arch) { default: llvm_unreachable("Unsupported target for RuntimeDyldMachO."); break; - case Triple::arm: return make_unique(MM); - case Triple::aarch64: return make_unique(MM); - case Triple::x86: return make_unique(MM); - case Triple::x86_64: return make_unique(MM); + case Triple::arm: + return make_unique(MemMgr, Resolver); + case Triple::aarch64: + return make_unique(MemMgr, Resolver); + case Triple::x86: + return make_unique(MemMgr, Resolver); + case Triple::x86_64: + return make_unique(MemMgr, Resolver); } } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h index f8bfc03..45a94ba 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -49,7 +49,9 @@ protected: // EH frame sections with the memory manager. SmallVector UnregisteredEHFrameSections; - RuntimeDyldMachO(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {} + RuntimeDyldMachO(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldImpl(MemMgr, Resolver) {} /// This convenience method uses memcpy to extract a contiguous addend (the /// addend size and offset are taken from the corresponding fields of the RE). @@ -114,8 +116,10 @@ protected: public: /// Create a RuntimeDyldMachO instance for the given target architecture. - static std::unique_ptr create(Triple::ArchType Arch, - RTDyldMemoryManager *mm); + static std::unique_ptr + create(Triple::ArchType Arch, + RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver); std::unique_ptr loadObject(const object::ObjectFile &O) override; @@ -142,7 +146,9 @@ private: int64_t DeltaForEH); public: - RuntimeDyldMachOCRTPBase(RTDyldMemoryManager *mm) : RuntimeDyldMachO(mm) {} + RuntimeDyldMachOCRTPBase(RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldMachO(MemMgr, Resolver) {} void finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) override; diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index ce2f4a2..cd534a1 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -32,7 +32,9 @@ private: SmallVector RegisteredEHFrameSections; public: - RuntimeDyldCOFFX86_64(RTDyldMemoryManager *MM) : RuntimeDyldCOFF(MM) {} + RuntimeDyldCOFFX86_64(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldCOFF(MM, Resolver) {} unsigned getMaxStubSize() override { return 6; // 2-byte jmp instruction + 32-bit relative address @@ -177,13 +179,11 @@ public: unsigned getStubAlignment() override { return 1; } void registerEHFrames() override { - if (!MemMgr) - return; for (auto const &EHFrameSID : UnregisteredEHFrameSections) { uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; size_t EHFrameSize = Sections[EHFrameSID].Size; - MemMgr->registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); + MemMgr.registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); RegisteredEHFrameSections.push_back(EHFrameSID); } UnregisteredEHFrameSections.clear(); diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index 196fa62..99fd6e3 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -23,8 +23,9 @@ public: typedef uint64_t TargetPtrT; - RuntimeDyldMachOAArch64(RTDyldMemoryManager *MM) - : RuntimeDyldMachOCRTPBase(MM) {} + RuntimeDyldMachOAArch64(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldMachOCRTPBase(MM, Resolver) {} unsigned getMaxStubSize() override { return 8; } diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index 09e430e..09e51f2 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -25,7 +25,9 @@ public: typedef uint32_t TargetPtrT; - RuntimeDyldMachOARM(RTDyldMemoryManager *MM) : RuntimeDyldMachOCRTPBase(MM) {} + RuntimeDyldMachOARM(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldMachOCRTPBase(MM, Resolver) {} unsigned getMaxStubSize() override { return 8; } diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index 67d7027..053f90c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -22,8 +22,9 @@ public: typedef uint32_t TargetPtrT; - RuntimeDyldMachOI386(RTDyldMemoryManager *MM) - : RuntimeDyldMachOCRTPBase(MM) {} + RuntimeDyldMachOI386(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldMachOCRTPBase(MM, Resolver) {} unsigned getMaxStubSize() override { return 0; } diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index 0734017..4b3b01b 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -22,8 +22,9 @@ public: typedef uint64_t TargetPtrT; - RuntimeDyldMachOX86_64(RTDyldMemoryManager *MM) - : RuntimeDyldMachOCRTPBase(MM) {} + RuntimeDyldMachOX86_64(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldMachOCRTPBase(MM, Resolver) {} unsigned getMaxStubSize() override { return 8; } diff --git a/lib/Fuzzer/CMakeLists.txt b/lib/Fuzzer/CMakeLists.txt index 81e51d1..bfd87ec 100644 --- a/lib/Fuzzer/CMakeLists.txt +++ b/lib/Fuzzer/CMakeLists.txt @@ -1,8 +1,10 @@ -# Disable the coverage instrumentation for the fuzzer itself. -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2 -fsanitize-coverage=0") -if( LLVM_USE_SANITIZE_COVERAGE ) +set(LIBFUZZER_FLAGS_BASE "${CMAKE_CXX_FLAGS_RELEASE}") +# Disable the coverage and sanitizer instrumentation for the fuzzer itself. +set(CMAKE_CXX_FLAGS_RELEASE "${LIBFUZZER_FLAGS_BASE} -O2 -fno-sanitize=all") +if( LLVM_USE_SANITIZE_COVERAGE ) add_library(LLVMFuzzerNoMain OBJECT FuzzerCrossOver.cpp + FuzzerDFSan.cpp FuzzerDriver.cpp FuzzerIO.cpp FuzzerLoop.cpp diff --git a/lib/Fuzzer/FuzzerDFSan.cpp b/lib/Fuzzer/FuzzerDFSan.cpp new file mode 100644 index 0000000..16f8c0f --- /dev/null +++ b/lib/Fuzzer/FuzzerDFSan.cpp @@ -0,0 +1,275 @@ +//===- FuzzerDFSan.cpp - DFSan-based fuzzer mutator -----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// DataFlowSanitizer (DFSan) is a tool for +// generalised dynamic data flow (taint) analysis: +// http://clang.llvm.org/docs/DataFlowSanitizer.html . +// +// This file implements a mutation algorithm based on taint +// analysis feedback from DFSan. +// +// The approach has some similarity to "Taint-based Directed Whitebox Fuzzing" +// by Vijay Ganesh & Tim Leek & Martin Rinard: +// http://dspace.mit.edu/openaccess-disseminate/1721.1/59320, +// but it uses a full blown LLVM IR taint analysis and separate instrumentation +// to analyze all of the "attack points" at once. +// +// Workflow: +// * lib/Fuzzer/Fuzzer*.cpp is compiled w/o any instrumentation. +// * The code under test is compiled with DFSan *and* with special extra hooks +// that are inserted before dfsan. Currently supported hooks: +// - __sanitizer_cov_trace_cmp: inserted before every ICMP instruction, +// receives the type, size and arguments of ICMP. +// * Every call to HOOK(a,b) is replaced by DFSan with +// __dfsw_HOOK(a, b, label(a), label(b)) so that __dfsw_HOOK +// gets all the taint labels for the arguments. +// * At the Fuzzer startup we assign a unique DFSan label +// to every byte of the input string (Fuzzer::CurrentUnit) so that for any +// chunk of data we know which input bytes it has derived from. +// * The __dfsw_* functions (implemented in this file) record the +// parameters (i.e. the application data and the corresponding taint labels) +// in a global state. +// * Fuzzer::MutateWithDFSan() tries to use the data recorded by __dfsw_* +// hooks to guide the fuzzing towards new application states. +// For example if 4 bytes of data that derive from input bytes {4,5,6,7} +// are compared with a constant 12345 and the comparison always yields +// the same result, we try to insert 12345, 12344, 12346 into bytes +// {4,5,6,7} of the next fuzzed inputs. +// +// This code does not function when DFSan is not linked in. +// Instead of using ifdefs and thus requiring a separate build of lib/Fuzzer +// we redeclare the dfsan_* interface functions as weak and check if they +// are nullptr before calling. +// If this approach proves to be useful we may add attribute(weak) to the +// dfsan declarations in dfsan_interface.h +// +// This module is in the "proof of concept" stage. +// It is capable of solving only the simplest puzzles +// like test/dfsan/DFSanSimpleCmpTest.cpp. +//===----------------------------------------------------------------------===// + +/* Example of manual usage: +( + cd $LLVM/lib/Fuzzer/ + clang -fPIC -c -g -O2 -std=c++11 Fuzzer*.cpp + clang++ -O0 -std=c++11 -fsanitize-coverage=3 \ + -mllvm -sanitizer-coverage-experimental-trace-compares=1 \ + -fsanitize=dataflow -fsanitize-blacklist=./dfsan_fuzzer_abi.list \ + test/dfsan/DFSanSimpleCmpTest.cpp Fuzzer*.o + ./a.out +) +*/ + +#include "FuzzerInternal.h" +#include + +#include +#include +#include + +extern "C" { +__attribute__((weak)) +dfsan_label dfsan_create_label(const char *desc, void *userdata); +__attribute__((weak)) +void dfsan_set_label(dfsan_label label, void *addr, size_t size); +__attribute__((weak)) +void dfsan_add_label(dfsan_label label, void *addr, size_t size); +__attribute__((weak)) +const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label); +} // extern "C" + +namespace { + +// These values are copied from include/llvm/IR/InstrTypes.h. +// We do not include the LLVM headers here to remain independent. +// If these values ever change, an assertion in ComputeCmp will fail. +enum Predicate { + ICMP_EQ = 32, ///< equal + ICMP_NE = 33, ///< not equal + ICMP_UGT = 34, ///< unsigned greater than + ICMP_UGE = 35, ///< unsigned greater or equal + ICMP_ULT = 36, ///< unsigned less than + ICMP_ULE = 37, ///< unsigned less or equal + ICMP_SGT = 38, ///< signed greater than + ICMP_SGE = 39, ///< signed greater or equal + ICMP_SLT = 40, ///< signed less than + ICMP_SLE = 41, ///< signed less or equal +}; + +template +bool ComputeCmp(size_t CmpType, U Arg1, U Arg2) { + switch(CmpType) { + case ICMP_EQ : return Arg1 == Arg2; + case ICMP_NE : return Arg1 != Arg2; + case ICMP_UGT: return Arg1 > Arg2; + case ICMP_UGE: return Arg1 >= Arg2; + case ICMP_ULT: return Arg1 < Arg2; + case ICMP_ULE: return Arg1 <= Arg2; + case ICMP_SGT: return (S)Arg1 > (S)Arg2; + case ICMP_SGE: return (S)Arg1 >= (S)Arg2; + case ICMP_SLT: return (S)Arg1 < (S)Arg2; + case ICMP_SLE: return (S)Arg1 <= (S)Arg2; + default: assert(0 && "unsupported CmpType"); + } + return false; +} + +static bool ComputeCmp(size_t CmpSize, size_t CmpType, uint64_t Arg1, + uint64_t Arg2) { + if (CmpSize == 8) return ComputeCmp(CmpType, Arg1, Arg2); + if (CmpSize == 4) return ComputeCmp(CmpType, Arg1, Arg2); + if (CmpSize == 2) return ComputeCmp(CmpType, Arg1, Arg2); + if (CmpSize == 1) return ComputeCmp(CmpType, Arg1, Arg2); + assert(0 && "unsupported type size"); + return true; +} + +// As a simplification we use the range of input bytes instead of a set of input +// bytes. +struct LabelRange { + uint16_t Beg, End; // Range is [Beg, End), thus Beg==End is an empty range. + + LabelRange(uint16_t Beg = 0, uint16_t End = 0) : Beg(Beg), End(End) {} + + static LabelRange Join(LabelRange LR1, LabelRange LR2) { + if (LR1.Beg == LR1.End) return LR2; + if (LR2.Beg == LR2.End) return LR1; + return {std::min(LR1.Beg, LR2.Beg), std::max(LR1.End, LR2.End)}; + } + LabelRange &Join(LabelRange LR) { + return *this = Join(*this, LR); + } + static LabelRange Singleton(const dfsan_label_info *LI) { + uint16_t Idx = (uint16_t)(uintptr_t)LI->userdata; + assert(Idx > 0); + return {(uint16_t)(Idx - 1), Idx}; + } +}; + +std::ostream &operator<<(std::ostream &os, const LabelRange &LR) { + return os << "[" << LR.Beg << "," << LR.End << ")"; +} + +class DFSanState { + public: + DFSanState(const fuzzer::Fuzzer::FuzzingOptions &Options) + : Options(Options) {} + + struct CmpSiteInfo { + size_t ResCounters[2] = {0, 0}; + size_t CmpSize = 0; + LabelRange LR; + std::unordered_map CountedConstants; + }; + + LabelRange GetLabelRange(dfsan_label L); + void DFSanCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType, + uint64_t Arg1, uint64_t Arg2, dfsan_label L1, + dfsan_label L2); + bool Mutate(fuzzer::Unit *U); + + private: + std::unordered_map PcToCmpSiteInfoMap; + LabelRange LabelRanges[1 << (sizeof(dfsan_label) * 8)] = {}; + const fuzzer::Fuzzer::FuzzingOptions &Options; +}; + +LabelRange DFSanState::GetLabelRange(dfsan_label L) { + LabelRange &LR = LabelRanges[L]; + if (LR.Beg < LR.End || L == 0) + return LR; + const dfsan_label_info *LI = dfsan_get_label_info(L); + if (LI->l1 || LI->l2) + return LR = LabelRange::Join(GetLabelRange(LI->l1), GetLabelRange(LI->l2)); + return LR = LabelRange::Singleton(LI); +} + +void DFSanState::DFSanCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType, + uint64_t Arg1, uint64_t Arg2, dfsan_label L1, + dfsan_label L2) { + if (L1 == 0 && L2 == 0) + return; // Not actionable. + if (L1 != 0 && L2 != 0) + return; // Probably still actionable. + bool Res = ComputeCmp(CmpSize, CmpType, Arg1, Arg2); + CmpSiteInfo &CSI = PcToCmpSiteInfoMap[PC]; + CSI.CmpSize = CmpSize; + CSI.LR.Join(GetLabelRange(L1)).Join(GetLabelRange(L2)); + if (!L1) CSI.CountedConstants[Arg1]++; + if (!L2) CSI.CountedConstants[Arg2]++; + size_t Counter = CSI.ResCounters[Res]++; + + if (Options.Verbosity >= 2 && + (Counter & (Counter - 1)) == 0 && + CSI.ResCounters[!Res] == 0) + std::cerr << "DFSAN:" + << " PC " << std::hex << PC << std::dec + << " S " << CmpSize + << " T " << CmpType + << " A1 " << Arg1 << " A2 " << Arg2 << " R " << Res + << " L" << L1 << GetLabelRange(L1) + << " L" << L2 << GetLabelRange(L2) + << " LR " << CSI.LR + << "\n"; +} + +bool DFSanState::Mutate(fuzzer::Unit *U) { + for (auto &PCToCmp : PcToCmpSiteInfoMap) { + auto &CSI = PCToCmp.second; + if (CSI.ResCounters[0] * CSI.ResCounters[1] != 0) continue; + if (CSI.ResCounters[0] + CSI.ResCounters[1] < 1000) continue; + if (CSI.CountedConstants.size() != 1) continue; + uintptr_t C = CSI.CountedConstants.begin()->first; + if (U->size() >= CSI.CmpSize) { + size_t RangeSize = CSI.LR.End - CSI.LR.Beg; + size_t Idx = CSI.LR.Beg + rand() % RangeSize; + if (Idx + CSI.CmpSize > U->size()) continue; + C += rand() % 5 - 2; + memcpy(U->data() + Idx, &C, CSI.CmpSize); + return true; + } + } + return false; +} + +static DFSanState *DFSan; + +} // namespace + +namespace fuzzer { + +bool Fuzzer::MutateWithDFSan(Unit *U) { + if (!&dfsan_create_label || !DFSan) return false; + return DFSan->Mutate(U); +} + +void Fuzzer::InitializeDFSan() { + if (!&dfsan_create_label || !Options.UseDFSan) return; + DFSan = new DFSanState(Options); + CurrentUnit.resize(Options.MaxLen); + for (size_t i = 0; i < static_cast(Options.MaxLen); i++) { + dfsan_label L = dfsan_create_label("input", (void*)(i + 1)); + // We assume that no one else has called dfsan_create_label before. + assert(L == i + 1); + dfsan_set_label(L, &CurrentUnit[i], 1); + } +} + +} // namespace fuzzer + +extern "C" { +void __dfsw___sanitizer_cov_trace_cmp(uint64_t SizeAndType, uint64_t Arg1, + uint64_t Arg2, dfsan_label L0, + dfsan_label L1, dfsan_label L2) { + assert(L0 == 0); + uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uint64_t CmpSize = (SizeAndType >> 32) / 8; + uint64_t Type = (SizeAndType << 32) >> 32; + DFSan->DFSanCmpCallback(PC, CmpSize, Type, Arg1, Arg2, L1, L2); +} +} // extern "C" diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp index 9ccd744..05a699e 100644 --- a/lib/Fuzzer/FuzzerDriver.cpp +++ b/lib/Fuzzer/FuzzerDriver.cpp @@ -18,6 +18,10 @@ #include #include #include +#include +#include +#include +#include namespace fuzzer { @@ -26,19 +30,26 @@ struct FlagDescription { const char *Name; const char *Description; int Default; - int *Flag; + int *IntFlag; + const char **StrFlag; }; struct { -#define FUZZER_FLAG(Type, Name, Default, Description) Type Name; +#define FUZZER_FLAG_INT(Name, Default, Description) int Name; +#define FUZZER_FLAG_STRING(Name, Description) const char *Name; #include "FuzzerFlags.def" -#undef FUZZER_FLAG +#undef FUZZER_FLAG_INT +#undef FUZZER_FLAG_STRING } Flags; static FlagDescription FlagDescriptions [] { -#define FUZZER_FLAG(Type, Name, Default, Description) {#Name, Description, Default, &Flags.Name}, +#define FUZZER_FLAG_INT(Name, Default, Description) \ + { #Name, Description, Default, &Flags.Name, nullptr}, +#define FUZZER_FLAG_STRING(Name, Description) \ + { #Name, Description, 0, nullptr, &Flags.Name }, #include "FuzzerFlags.def" -#undef FUZZER_FLAG +#undef FUZZER_FLAG_INT +#undef FUZZER_FLAG_STRING }; static const size_t kNumFlags = @@ -79,11 +90,18 @@ static bool ParseOneFlag(const char *Param) { const char *Name = FlagDescriptions[F].Name; const char *Str = FlagValue(Param, Name); if (Str) { - int Val = std::stol(Str); - *FlagDescriptions[F].Flag = Val; - if (Flags.verbosity >= 2) - std::cerr << "Flag: " << Name << " " << Val << "\n"; - return true; + if (FlagDescriptions[F].IntFlag) { + int Val = std::stol(Str); + *FlagDescriptions[F].IntFlag = Val; + if (Flags.verbosity >= 2) + std::cerr << "Flag: " << Name << " " << Val << "\n"; + return true; + } else if (FlagDescriptions[F].StrFlag) { + *FlagDescriptions[F].StrFlag = Str; + if (Flags.verbosity >= 2) + std::cerr << "Flag: " << Name << " " << Str << "\n"; + return true; + } } } PrintHelp(); @@ -92,8 +110,12 @@ static bool ParseOneFlag(const char *Param) { // We don't use any library to minimize dependencies. static void ParseFlags(int argc, char **argv) { - for (size_t F = 0; F < kNumFlags; F++) - *FlagDescriptions[F].Flag = FlagDescriptions[F].Default; + for (size_t F = 0; F < kNumFlags; F++) { + if (FlagDescriptions[F].IntFlag) + *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; + if (FlagDescriptions[F].StrFlag) + *FlagDescriptions[F].StrFlag = nullptr; + } for (int A = 1; A < argc; A++) { if (ParseOneFlag(argv[A])) continue; inputs.push_back(argv[A]); @@ -139,6 +161,26 @@ static int RunInMultipleProcesses(int argc, char **argv, int NumWorkers, return HasErrors ? 1 : 0; } +std::vector ReadTokensFile(const char *TokensFilePath) { + if (!TokensFilePath) return {}; + std::string TokensFileContents = FileToString(TokensFilePath); + std::istringstream ISS(TokensFileContents); + std::vector Res = {std::istream_iterator{ISS}, + std::istream_iterator{}}; + Res.push_back(" "); + Res.push_back("\t"); + Res.push_back("\n"); + return Res; +} + +int ApplyTokens(const Fuzzer &F, const char *InputFilePath) { + Unit U = FileToVector(InputFilePath); + auto T = F.SubstituteTokens(U); + T.push_back(0); + std::cout << T.data(); + return 0; +} + int FuzzerDriver(int argc, char **argv, UserCallback Callback) { using namespace fuzzer; @@ -161,8 +203,10 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) { Options.UseCounters = Flags.use_counters; Options.UseFullCoverageSet = Flags.use_full_coverage_set; Options.UseCoveragePairs = Flags.use_coverage_pairs; + Options.UseDFSan = Flags.dfsan; Options.PreferSmallDuringInitialShuffle = Flags.prefer_small_during_initial_shuffle; + Options.Tokens = ReadTokensFile(Flags.tokens); if (Flags.runs >= 0) Options.MaxNumberOfRuns = Flags.runs; if (!inputs.empty()) @@ -181,6 +225,16 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) { if (Flags.timeout > 0) SetTimer(Flags.timeout); + if (Flags.verbosity >= 2) { + std::cerr << "Tokens: {"; + for (auto &T : Options.Tokens) + std::cerr << T << ","; + std::cerr << "}\n"; + } + + if (Flags.apply_tokens) + return ApplyTokens(F, Flags.apply_tokens); + for (auto &inp : inputs) F.ReadDir(inp); diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def index 08176af..dbaf75d 100644 --- a/lib/Fuzzer/FuzzerFlags.def +++ b/lib/Fuzzer/FuzzerFlags.def @@ -6,41 +6,48 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// Flags. FUZZER_FLAG macro should be defined at the point of inclusion. -// We are not using any flag parsing library for better portability and -// independence. +// Flags. FUZZER_FLAG_INT/FUZZER_FLAG_STRING macros should be defined at the +// point of inclusion. We are not using any flag parsing library for better +// portability and independence. //===----------------------------------------------------------------------===// -FUZZER_FLAG(int, verbosity, 1, "Verbosity level.") -FUZZER_FLAG(int, seed, 0, "Random seed. If 0, seed is generated.") -FUZZER_FLAG(int, iterations, -1, +FUZZER_FLAG_INT(verbosity, 1, "Verbosity level.") +FUZZER_FLAG_INT(seed, 0, "Random seed. If 0, seed is generated.") +FUZZER_FLAG_INT(iterations, -1, "Number of iterations of the fuzzer internal loop" " (-1 for infinite iterations).") -FUZZER_FLAG(int, runs, -1, +FUZZER_FLAG_INT(runs, -1, "Number of individual test runs (-1 for infinite runs).") -FUZZER_FLAG(int, max_len, 64, "Maximal length of the test input.") -FUZZER_FLAG(int, cross_over, 1, "If 1, cross over inputs.") -FUZZER_FLAG(int, mutate_depth, 5, +FUZZER_FLAG_INT(max_len, 64, "Maximal length of the test input.") +FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") +FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") -FUZZER_FLAG( - int, prefer_small_during_initial_shuffle, -1, +FUZZER_FLAG_INT( + prefer_small_during_initial_shuffle, -1, "If 1, always prefer smaller inputs during the initial corpus shuffle." " If 0, never do that. If -1, do it sometimes.") -FUZZER_FLAG(int, exit_on_first, 0, +FUZZER_FLAG_INT(exit_on_first, 0, "If 1, exit after the first new interesting input is found.") -FUZZER_FLAG(int, timeout, -1, "Timeout in seconds (if positive).") -FUZZER_FLAG(int, help, 0, "Print help.") -FUZZER_FLAG( - int, save_minimized_corpus, 0, +FUZZER_FLAG_INT(timeout, -1, "Timeout in seconds (if positive).") +FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT( + save_minimized_corpus, 0, "If 1, the minimized corpus is saved into the first input directory") -FUZZER_FLAG(int, use_counters, 0, "Use coverage counters") -FUZZER_FLAG(int, use_full_coverage_set, 0, +FUZZER_FLAG_INT(use_counters, 0, "Use coverage counters") +FUZZER_FLAG_INT(use_full_coverage_set, 0, "Experimental: Maximize the number of different full" " coverage sets as opposed to maximizing the total coverage." " This is potentially MUCH slower, but may discover more paths.") -FUZZER_FLAG(int, use_coverage_pairs, 0, +FUZZER_FLAG_INT(use_coverage_pairs, 0, "Experimental: Maximize the number of different coverage pairs.") -FUZZER_FLAG(int, jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" +FUZZER_FLAG_INT(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" " this number of jobs in separate worker processes" " with stdout/stderr redirected to fuzz-JOB.log.") -FUZZER_FLAG(int, workers, 0, +FUZZER_FLAG_INT(workers, 0, "Number of simultaneous worker processes to run the jobs.") +FUZZER_FLAG_INT(dfsan, 1, "Use DFSan for taint-guided mutations. No-op unless " + "the DFSan instrumentation was compiled in.") + +FUZZER_FLAG_STRING(tokens, "Use the file with tokens (one token per line) to" + " fuzz a token based input language.") +FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes " + " with tokens and write the result to stdout.") diff --git a/lib/Fuzzer/FuzzerIO.cpp b/lib/Fuzzer/FuzzerIO.cpp index 224808c..ef23d42 100644 --- a/lib/Fuzzer/FuzzerIO.cpp +++ b/lib/Fuzzer/FuzzerIO.cpp @@ -33,6 +33,12 @@ Unit FileToVector(const std::string &Path) { std::istreambuf_iterator()); } +std::string FileToString(const std::string &Path) { + std::ifstream T(Path); + return std::string((std::istreambuf_iterator(T)), + std::istreambuf_iterator()); +} + void CopyFileToErr(const std::string &Path) { std::ifstream T(Path); std::copy(std::istreambuf_iterator(T), std::istreambuf_iterator(), diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h index e4e5eb7..7787109 100644 --- a/lib/Fuzzer/FuzzerInternal.h +++ b/lib/Fuzzer/FuzzerInternal.h @@ -23,7 +23,8 @@ namespace fuzzer { typedef std::vector Unit; using namespace std::chrono; -Unit ReadFile(const char *Path); +std::string FileToString(const std::string &Path); +Unit FileToVector(const std::string &Path); void ReadDirToVectorOfUnits(const char *Path, std::vector *V); void WriteToFile(const Unit &U, const std::string &Path); void CopyFileToErr(const std::string &Path); @@ -51,17 +52,17 @@ class Fuzzer { bool UseCounters = false; bool UseFullCoverageSet = false; bool UseCoveragePairs = false; + bool UseDFSan = false; int PreferSmallDuringInitialShuffle = -1; size_t MaxNumberOfRuns = ULONG_MAX; std::string OutputCorpus; + std::vector Tokens; }; - Fuzzer(UserCallback Callback, FuzzingOptions Options) - : Callback(Callback), Options(Options) { - SetDeathCallback(); - } + Fuzzer(UserCallback Callback, FuzzingOptions Options); void AddToCorpus(const Unit &U) { Corpus.push_back(U); } size_t Loop(size_t NumIterations); void ShuffleAndMinimize(); + void InitializeDFSan(); size_t CorpusSize() const { return Corpus.size(); } void ReadDir(const std::string &Path) { ReadDirToVectorOfUnits(Path.c_str(), &Corpus); @@ -76,20 +77,28 @@ class Fuzzer { size_t getTotalNumberOfRuns() { return TotalNumberOfRuns; } - static void AlarmCallback(); + static void StaticAlarmCallback(); + + Unit SubstituteTokens(const Unit &U) const; private: + void AlarmCallback(); + void ExecuteCallback(const Unit &U); size_t MutateAndTestOne(Unit *U); size_t RunOne(const Unit &U); size_t RunOneMaximizeTotalCoverage(const Unit &U); size_t RunOneMaximizeFullCoverageSet(const Unit &U); size_t RunOneMaximizeCoveragePairs(const Unit &U); void WriteToOutputCorpus(const Unit &U); - static void WriteToCrash(const Unit &U, const char *Prefix); + void WriteToCrash(const Unit &U, const char *Prefix); + bool MutateWithDFSan(Unit *U); + void PrintStats(const char *Where, size_t Cov, const char *End = "\n"); + void PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter = ""); void SetDeathCallback(); - static void DeathCallback(); - static Unit CurrentUnit; + static void StaticDeathCallback(); + void DeathCallback(); + Unit CurrentUnit; size_t TotalNumberOfRuns = 0; @@ -108,7 +117,8 @@ class Fuzzer { UserCallback Callback; FuzzingOptions Options; system_clock::time_point ProcessStartTime = system_clock::now(); - static system_clock::time_point UnitStartTime; + system_clock::time_point UnitStartTime; + long TimeOfLongestUnitInSeconds = 0; }; }; // namespace fuzzer diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index 563fbf4..9dfe30b 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -16,21 +16,49 @@ namespace fuzzer { -// static -Unit Fuzzer::CurrentUnit; -system_clock::time_point Fuzzer::UnitStartTime; +// Only one Fuzzer per process. +static Fuzzer *F; + +Fuzzer::Fuzzer(UserCallback Callback, FuzzingOptions Options) + : Callback(Callback), Options(Options) { + SetDeathCallback(); + InitializeDFSan(); + assert(!F); + F = this; +} void Fuzzer::SetDeathCallback() { - __sanitizer_set_death_callback(DeathCallback); + __sanitizer_set_death_callback(StaticDeathCallback); +} + +void Fuzzer::PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter) { + if (Options.Tokens.empty()) { + PrintASCII(U, PrintAfter); + } else { + auto T = SubstituteTokens(U); + T.push_back(0); + std::cerr << T.data(); + std::cerr << PrintAfter; + } +} + +void Fuzzer::StaticDeathCallback() { + assert(F); + F->DeathCallback(); } void Fuzzer::DeathCallback() { std::cerr << "DEATH: " << std::endl; Print(CurrentUnit, "\n"); - PrintASCII(CurrentUnit, "\n"); + PrintUnitInASCIIOrTokens(CurrentUnit, "\n"); WriteToCrash(CurrentUnit, "crash-"); } +void Fuzzer::StaticAlarmCallback() { + assert(F); + F->AlarmCallback(); +} + void Fuzzer::AlarmCallback() { size_t Seconds = duration_cast(system_clock::now() - UnitStartTime).count(); @@ -38,27 +66,40 @@ void Fuzzer::AlarmCallback() { << std::endl; if (Seconds >= 3) { Print(CurrentUnit, "\n"); - PrintASCII(CurrentUnit, "\n"); + PrintUnitInASCIIOrTokens(CurrentUnit, "\n"); WriteToCrash(CurrentUnit, "timeout-"); } exit(1); } +void Fuzzer::PrintStats(const char *Where, size_t Cov, const char *End) { + if (!Options.Verbosity) return; + size_t Seconds = secondsSinceProcessStartUp(); + size_t ExecPerSec = (Seconds ? TotalNumberOfRuns / Seconds : 0); + std::cerr + << "#" << TotalNumberOfRuns + << "\t" << Where + << " cov " << Cov + << " bits " << TotalBits() + << " units " << Corpus.size() + << " exec/s " << ExecPerSec + << End; +} + void Fuzzer::ShuffleAndMinimize() { + size_t MaxCov = 0; bool PreferSmall = (Options.PreferSmallDuringInitialShuffle == 1 || (Options.PreferSmallDuringInitialShuffle == -1 && rand() % 2)); if (Options.Verbosity) - std::cerr << "Shuffle: Size: " << Corpus.size() - << " prefer small: " << PreferSmall - << "\n"; + std::cerr << "PreferSmall: " << PreferSmall << "\n"; + PrintStats("READ ", 0); std::vector NewCorpus; std::random_shuffle(Corpus.begin(), Corpus.end()); if (PreferSmall) std::stable_sort( Corpus.begin(), Corpus.end(), [](const Unit &A, const Unit &B) { return A.size() < B.size(); }); - size_t MaxCov = 0; Unit &U = CurrentUnit; for (const auto &C : Corpus) { for (size_t First = 0; First < 1; First++) { @@ -77,18 +118,29 @@ void Fuzzer::ShuffleAndMinimize() { } } Corpus = NewCorpus; - if (Options.Verbosity) - std::cerr << "Shuffle done: " << Corpus.size() << " IC: " << MaxCov << "\n"; + PrintStats("INITED", MaxCov); } size_t Fuzzer::RunOne(const Unit &U) { UnitStartTime = system_clock::now(); TotalNumberOfRuns++; + size_t Res = 0; if (Options.UseFullCoverageSet) - return RunOneMaximizeFullCoverageSet(U); - if (Options.UseCoveragePairs) - return RunOneMaximizeCoveragePairs(U); - return RunOneMaximizeTotalCoverage(U); + Res = RunOneMaximizeFullCoverageSet(U); + else if (Options.UseCoveragePairs) + Res = RunOneMaximizeCoveragePairs(U); + else + Res = RunOneMaximizeTotalCoverage(U); + auto UnitStopTime = system_clock::now(); + auto TimeOfUnit = + duration_cast(UnitStopTime - UnitStartTime).count(); + if (TimeOfUnit > TimeOfLongestUnitInSeconds) { + TimeOfLongestUnitInSeconds = TimeOfUnit; + std::cerr << "Longest unit: " << TimeOfLongestUnitInSeconds + << " s:\n"; + Print(U, "\n"); + } + return Res; } static uintptr_t HashOfArrayOfPCs(uintptr_t *PCs, uintptr_t NumPCs) { @@ -99,12 +151,35 @@ static uintptr_t HashOfArrayOfPCs(uintptr_t *PCs, uintptr_t NumPCs) { return Res; } +Unit Fuzzer::SubstituteTokens(const Unit &U) const { + Unit Res; + for (auto Idx : U) { + if (Idx < Options.Tokens.size()) { + std::string Token = Options.Tokens[Idx]; + Res.insert(Res.end(), Token.begin(), Token.end()); + } else { + Res.push_back(' '); + } + } + // FIXME: Apply DFSan labels. + return Res; +} + +void Fuzzer::ExecuteCallback(const Unit &U) { + if (Options.Tokens.empty()) { + Callback(U.data(), U.size()); + } else { + auto T = SubstituteTokens(U); + Callback(T.data(), T.size()); + } +} + // Experimental. Does not yet scale. // Fuly reset the current coverage state, run a single unit, // collect all coverage pairs and return non-zero if a new pair is observed. size_t Fuzzer::RunOneMaximizeCoveragePairs(const Unit &U) { __sanitizer_reset_coverage(); - Callback(U.data(), U.size()); + ExecuteCallback(U); uintptr_t *PCs; uintptr_t NumPCs = __sanitizer_get_coverage_guards(&PCs); bool HasNewPairs = false; @@ -129,7 +204,7 @@ size_t Fuzzer::RunOneMaximizeCoveragePairs(const Unit &U) { // e.g. test/FullCoverageSetTest.cpp. FIXME: make it scale. size_t Fuzzer::RunOneMaximizeFullCoverageSet(const Unit &U) { __sanitizer_reset_coverage(); - Callback(U.data(), U.size()); + ExecuteCallback(U); uintptr_t *PCs; uintptr_t NumPCs =__sanitizer_get_coverage_guards(&PCs); if (FullCoverageSets.insert(HashOfArrayOfPCs(PCs, NumPCs)).second) @@ -144,21 +219,16 @@ size_t Fuzzer::RunOneMaximizeTotalCoverage(const Unit &U) { __sanitizer_update_counter_bitset_and_clear_counters(0); } size_t OldCoverage = __sanitizer_get_total_unique_coverage(); - Callback(U.data(), U.size()); + ExecuteCallback(U); size_t NewCoverage = __sanitizer_get_total_unique_coverage(); size_t NumNewBits = 0; if (Options.UseCounters) NumNewBits = __sanitizer_update_counter_bitset_and_clear_counters( CounterBitmap.data()); - if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && Options.Verbosity) { - size_t Seconds = secondsSinceProcessStartUp(); - std::cerr - << "#" << TotalNumberOfRuns - << "\tcov: " << NewCoverage - << "\tbits: " << TotalBits() - << "\texec/s: " << (Seconds ? TotalNumberOfRuns / Seconds : 0) << "\n"; - } + if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && Options.Verbosity) + PrintStats("pulse ", NewCoverage); + if (NewCoverage > OldCoverage || NumNewBits) return NewCoverage; return 0; @@ -192,22 +262,18 @@ size_t Fuzzer::MutateAndTestOne(Unit *U) { for (int i = 0; i < Options.MutateDepth; i++) { if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) return NewUnits; + MutateWithDFSan(U); Mutate(U, Options.MaxLen); size_t NewCoverage = RunOne(*U); if (NewCoverage) { Corpus.push_back(*U); NewUnits++; + PrintStats("NEW ", NewCoverage, ""); if (Options.Verbosity) { - std::cerr << "#" << TotalNumberOfRuns - << "\tNEW: " << NewCoverage - << " B: " << TotalBits() - << " L: " << U->size() - << " S: " << Corpus.size() - << " I: " << i - << "\t"; + std::cerr << " L: " << U->size(); if (U->size() < 30) { - PrintASCII(*U); - std::cerr << "\t"; + std::cerr << " "; + PrintUnitInASCIIOrTokens(*U, "\t"); Print(*U); } std::cerr << "\n"; diff --git a/lib/Fuzzer/FuzzerUtil.cpp b/lib/Fuzzer/FuzzerUtil.cpp index 679f289..3635f39 100644 --- a/lib/Fuzzer/FuzzerUtil.cpp +++ b/lib/Fuzzer/FuzzerUtil.cpp @@ -19,15 +19,18 @@ namespace fuzzer { void Print(const Unit &v, const char *PrintAfter) { - std::cerr << v.size() << ": "; for (auto x : v) - std::cerr << (unsigned) x << " "; + std::cerr << "0x" << std::hex << (unsigned) x << std::dec << ","; std::cerr << PrintAfter; } void PrintASCII(const Unit &U, const char *PrintAfter) { - for (auto X : U) - std::cerr << (char)((isascii(X) && X >= ' ') ? X : '?'); + for (auto X : U) { + if (isprint(X)) + std::cerr << X; + else + std::cerr << "\\x" << std::hex << (int)(unsigned)X << std::dec; + } std::cerr << PrintAfter; } @@ -43,7 +46,7 @@ std::string Hash(const Unit &in) { } static void AlarmHandler(int, siginfo_t *, void *) { - Fuzzer::AlarmCallback(); + Fuzzer::StaticAlarmCallback(); } void SetTimer(int Seconds) { diff --git a/lib/Fuzzer/README.txt b/lib/Fuzzer/README.txt index e4d6b4f..79f49b5 100644 --- a/lib/Fuzzer/README.txt +++ b/lib/Fuzzer/README.txt @@ -1,112 +1,2 @@ -=============================== -Fuzzer -- a library for coverage-guided fuzz testing. -=============================== +Move to http://llvm.org/docs/LibFuzzer.html -This library is intended primarily for in-process coverage-guided fuzz testing -(fuzzing) of other libraries. The typical workflow looks like this: - - * Build the Fuzzer library as a static archive (or just a set of .o files). - Note that the Fuzzer contains the main() function. - Preferably do *not* use sanitizers while building the Fuzzer. - * Build the library you are going to test with -fsanitize-coverage=[234] - and one of the sanitizers. We recommend to build the library in several - different modes (e.g. asan, msan, lsan, ubsan, etc) and even using different - optimizations options (e.g. -O0, -O1, -O2) to diversify testing. - * Build a test driver using the same options as the library. - The test driver is a C/C++ file containing interesting calls to the library - inside a single function: - extern "C" void TestOneInput(const uint8_t *Data, size_t Size); - * Link the Fuzzer, the library and the driver together into an executable - using the same sanitizer options as for the library. - * Collect the initial corpus of inputs for the - fuzzer (a directory with test inputs, one file per input). - The better your inputs are the faster you will find something interesting. - Also try to keep your inputs small, otherwise the Fuzzer will run too slow. - * Run the fuzzer with the test corpus. As new interesting test cases are - discovered they will be added to the corpus. If a bug is discovered by - the sanitizer (asan, etc) it will be reported as usual and the reproducer - will be written to disk. - Each Fuzzer process is single-threaded (unless the library starts its own - threads). You can run the Fuzzer on the same corpus in multiple processes. - in parallel. For run-time options run the Fuzzer binary with '-help=1'. - - -The Fuzzer is similar in concept to AFL (http://lcamtuf.coredump.cx/afl/), -but uses in-process Fuzzing, which is more fragile, more restrictive, but -potentially much faster as it has no overhead for process start-up. -It uses LLVM's "Sanitizer Coverage" instrumentation to get in-process -coverage-feedback https://code.google.com/p/address-sanitizer/wiki/AsanCoverage - -The code resides in the LLVM repository and is (or will be) used by various -parts of LLVM, but the Fuzzer itself does not (and should not) depend on any -part of LLVM and can be used for other projects. Ideally, the Fuzzer's code -should not have any external dependencies. Right now it uses STL, which may need -to be fixed later. See also F.A.Q. below. - -Examples of usage in LLVM: - * clang-format-fuzzer. The inputs are random pieces of C++-like text. - * Build (make sure to use fresh clang as the host compiler): - cmake -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ - -DLLVM_USE_SANITIZER=Address -DLLVM_USE_SANITIZE_COVERAGE=YES \ - /path/to/llvm -DCMAKE_BUILD_TYPE=Release - ninja clang-format-fuzzer - * Optionally build other kinds of binaries (asan+Debug, msan, ubsan, etc) - * TODO: commit the pre-fuzzed corpus to svn (?). - * Run: - clang-format-fuzzer CORPUS_DIR - -Toy example (see SimpleTest.cpp): -a simple function that does something interesting if it receives bytes "Hi!". - # Build the Fuzzer with asan: - % clang++ -std=c++11 -fsanitize=address -fsanitize-coverage=3 -O1 -g \ - Fuzzer*.cpp test/SimpleTest.cpp - # Run the fuzzer with no corpus (assuming on empty input) - % ./a.out - -=============================================================================== -F.A.Q. - -Q. Why Fuzzer does not use any of the LLVM support? -A. There are two reasons. -First, we want this library to be used outside of the LLVM w/o users having to -build the rest of LLVM. This may sound unconvincing for many LLVM folks, -but in practice the need for building the whole LLVM frightens many potential -users -- and we want more users to use this code. -Second, there is a subtle technical reason not to rely on the rest of LLVM, or -any other large body of code (maybe not even STL). When coverage instrumentation -is enabled, it will also instrument the LLVM support code which will blow up the -coverage set of the process (since the fuzzer is in-process). In other words, by -using more external dependencies we will slow down the fuzzer while the main -reason for it to exist is extreme speed. - -Q. What about Windows then? The Fuzzer contains code that does not build on -Windows. -A. The sanitizer coverage support does not work on Windows either as of 01/2015. -Once it's there, we'll need to re-implement OS-specific parts (I/O, signals). - -Q. When this Fuzzer is not a good solution for a problem? -A. - * If the test inputs are validated by the target library and the validator - asserts/crashes on invalid inputs, the in-process fuzzer is not applicable - (we could use fork() w/o exec, but it comes with extra overhead). - * Bugs in the target library may accumulate w/o being detected. E.g. a memory - corruption that goes undetected at first and then leads to a crash while - testing another input. This is why it is highly recommended to run this - in-process fuzzer with all sanitizers to detect most bugs on the spot. - * It is harder to protect the in-process fuzzer from excessive memory - consumption and infinite loops in the target library (still possible). - * The target library should not have significant global state that is not - reset between the runs. - * Many interesting target libs are not designed in a way that supports - the in-process fuzzer interface (e.g. require a file path instead of a - byte array). - * If a single test run takes a considerable fraction of a second (or - more) the speed benefit from the in-process fuzzer is negligible. - * If the target library runs persistent threads (that outlive - execution of one test) the fuzzing results will be unreliable. - -Q. So, what exactly this Fuzzer is good for? -A. This Fuzzer might be a good choice for testing libraries that have relatively -small inputs, each input takes < 1ms to run, and the library code is not expected -to crash on invalid inputs. -Examples: regular expression matchers, text or binary format parsers. diff --git a/lib/Fuzzer/cxx_fuzzer_tokens.txt b/lib/Fuzzer/cxx_fuzzer_tokens.txt new file mode 100644 index 0000000..f3c4f80 --- /dev/null +++ b/lib/Fuzzer/cxx_fuzzer_tokens.txt @@ -0,0 +1,218 @@ +# +## +` +~ +! +@ +$ +% +^ +& +* +( +) +_ +- +_ += ++ +{ +} +[ +] +| +\ +, +. +/ +? +> +< +; +: +' +" +++ +-- +<< +>> ++= +-= +*= +/= +>>= +<<= +&= +|= +^= +%= +!= +&& +|| +== +>= +<= +-> +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +alignas +alignof +and +and_eq +asm +auto +bitand +bitor +bool +break +case +catch +char +char16_t +char32_t +class +compl +concept +const +constexpr +const_cast +continue +decltype +default +delete +do +double +dynamic_cast +else +enum +explicit +export +extern +false +float +for +friend +goto +if +inline +int +long +mutable +namespace +new +noexcept +not +not_eq +nullptr +operator +or +or_eq +private +protected +public +register +reinterpret_cast +requires +return +short +signed +sizeof +static +static_assert +static_cast +struct +switch +template +this +thread_local +throw +true +try +typedef +typeid +typename +union +unsigned +using +virtual +void +volatile +wchar_t +while +xor +xor_eq +if +elif +else +endif +defined +ifdef +ifndef +define +undef +include +line +error +pragma +override +final diff --git a/lib/Fuzzer/dfsan_fuzzer_abi.list b/lib/Fuzzer/dfsan_fuzzer_abi.list new file mode 100644 index 0000000..7da7522 --- /dev/null +++ b/lib/Fuzzer/dfsan_fuzzer_abi.list @@ -0,0 +1,12 @@ +# Replaces __sanitizer_cov_trace_cmp with __dfsw___sanitizer_cov_trace_cmp +fun:__sanitizer_cov_trace_cmp=custom +fun:__sanitizer_cov_trace_cmp=uninstrumented + +# Ignores coverage callbacks. +fun:__sanitizer_cov=uninstrumented +fun:__sanitizer_cov=discard +fun:__sanitizer_cov_module_init=uninstrumented +fun:__sanitizer_cov_module_init=discard + +# Don't add extra parameters to the Fuzzer callback. +fun:TestOneInput=uninstrumented diff --git a/lib/Fuzzer/test/CMakeLists.txt b/lib/Fuzzer/test/CMakeLists.txt index 08130c6..fb3bf20 100644 --- a/lib/Fuzzer/test/CMakeLists.txt +++ b/lib/Fuzzer/test/CMakeLists.txt @@ -2,10 +2,11 @@ # basic blocks and we'll fail to discover the targets. # Also enable the coverage instrumentation back (it is disabled # for the Fuzzer lib) -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O0 -fsanitize-coverage=4") +set(CMAKE_CXX_FLAGS_RELEASE "${LIBFUZZER_FLAGS_BASE} -O0 -fsanitize-coverage=4") set(Tests CounterTest + CxxTokensTest FourIndependentBranchesTest FullCoverageSetTest InfiniteTest @@ -14,11 +15,14 @@ set(Tests TimeoutTest ) +set(DFSanTests + DFSanSimpleCmpTest + ) + set(TestBinaries) foreach(Test ${Tests}) add_executable(LLVMFuzzer-${Test} - EXCLUDE_FROM_ALL ${Test}.cpp ) target_link_libraries(LLVMFuzzer-${Test} @@ -52,6 +56,13 @@ target_link_libraries(LLVMFuzzer-Unittest set(TestBinaries ${TestBinaries} LLVMFuzzer-Unittest) +add_subdirectory(dfsan) + +foreach(Test ${DFSanTests}) + set(TestBinaries ${TestBinaries} LLVMFuzzer-${Test}) +endforeach() + + set_target_properties(${TestBinaries} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/lib/Fuzzer/test/CxxTokensTest.cpp b/lib/Fuzzer/test/CxxTokensTest.cpp new file mode 100644 index 0000000..1addccb --- /dev/null +++ b/lib/Fuzzer/test/CxxTokensTest.cpp @@ -0,0 +1,24 @@ +// Simple test for a fuzzer. The fuzzer must find a sequence of C++ tokens. +#include +#include +#include +#include +#include + +static void Found() { + std::cout << "Found the target, exiting\n"; + exit(1); +} + +extern "C" void TestOneInput(const uint8_t *Data, size_t Size) { + // looking for "thread_local unsigned A;" + if (Size < 24) return; + if (0 == memcmp(&Data[0], "thread_local", 12)) + if (Data[12] == ' ') + if (0 == memcmp(&Data[13], "unsigned", 8)) + if (Data[21] == ' ') + if (Data[22] == 'A') + if (Data[23] == ';') + Found(); +} + diff --git a/lib/Fuzzer/test/dfsan/CMakeLists.txt b/lib/Fuzzer/test/dfsan/CMakeLists.txt new file mode 100644 index 0000000..b5b874f --- /dev/null +++ b/lib/Fuzzer/test/dfsan/CMakeLists.txt @@ -0,0 +1,17 @@ +# These tests depend on both coverage and dfsan instrumentation. + +set(DFSAN_FUZZER_ABI_LIST "${CMAKE_CURRENT_SOURCE_DIR}/../../dfsan_fuzzer_abi.list") + +set(CMAKE_CXX_FLAGS_RELEASE + "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow -mllvm -sanitizer-coverage-experimental-trace-compares=1 -fsanitize-blacklist=${DFSAN_FUZZER_ABI_LIST}") + +foreach(Test ${DFSanTests}) + set_source_files_properties(${Test}.cpp PROPERTIES OBJECT_DEPENDS ${DFSAN_FUZZER_ABI_LIST}) + add_executable(LLVMFuzzer-${Test} + ${Test}.cpp + ) + target_link_libraries(LLVMFuzzer-${Test} + LLVMFuzzer + ) +endforeach() + diff --git a/lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp b/lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp new file mode 100644 index 0000000..1162092 --- /dev/null +++ b/lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp @@ -0,0 +1,30 @@ +// Simple test for a fuzzer. The fuzzer must find several narrow ranges. +#include +#include +#include +#include + +extern "C" void TestOneInput(const uint8_t *Data, size_t Size) { + if (Size < 14) return; + uint64_t x = 0; + int64_t y = 0; + int z = 0; + unsigned short a = 0; + memcpy(&x, Data, 8); + memcpy(&y, Data + Size - 8, 8); + memcpy(&z, Data + Size / 2, sizeof(z)); + memcpy(&a, Data + Size / 2 + 4, sizeof(a)); + + if (x > 1234567890 && + x < 1234567895 && + y >= 987654321 && + y <= 987654325 && + z < -10000 && + z >= -10005 && + z != -10003 && + a == 4242) { + fprintf(stderr, "Found the target: size %zd (%zd, %zd, %d, %d), exiting.\n", + Size, x, y, z, a); + exit(1); + } +} diff --git a/lib/Fuzzer/test/fuzzer.test b/lib/Fuzzer/test/fuzzer.test index 45691f5..2a0e95f 100644 --- a/lib/Fuzzer/test/fuzzer.test +++ b/lib/Fuzzer/test/fuzzer.test @@ -20,3 +20,9 @@ FourIndependentBranchesTest: BINGO RUN: not ./LLVMFuzzer-CounterTest -use_counters=1 -max_len=6 -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=CounterTest CounterTest: BINGO + +RUN: not ./LLVMFuzzer-DFSanSimpleCmpTest -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=DFSanSimpleCmpTest +DFSanSimpleCmpTest: Found the target: + +RUN: not ./LLVMFuzzer-CxxTokensTest -seed=1 -timeout=15 -tokens=%S/../cxx_fuzzer_tokens.txt 2>&1 | FileCheck %s --check-prefix=CxxTokensTest +CxxTokensTest: Found the target, exiting diff --git a/lib/IR/Android.mk b/lib/IR/Android.mk index 2ca02f7..32ebfae 100644 --- a/lib/IR/Android.mk +++ b/lib/IR/Android.mk @@ -43,7 +43,6 @@ vmcore_SRC_FILES := \ Type.cpp \ TypeFinder.cpp \ Use.cpp \ - UseListOrder.cpp \ User.cpp \ Value.cpp \ ValueSymbolTable.cpp \ diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index ae0beba..48737b5 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -401,9 +401,7 @@ public: /// NumberedTypes - The numbered types, along with their value. DenseMap NumberedTypes; - - TypePrinting() {} - ~TypePrinting() {} + TypePrinting() = default; void incorporateTypes(const Module &M); @@ -1443,11 +1441,11 @@ void MDFieldPrinter::printDIFlags(StringRef Name, unsigned Flags) { Out << FS << Name << ": "; SmallVector SplitFlags; - unsigned Extra = DIDescriptor::splitFlags(Flags, SplitFlags); + unsigned Extra = DebugNode::splitFlags(Flags, SplitFlags); FieldSeparator FlagsFS(" | "); for (unsigned F : SplitFlags) { - const char *StringF = DIDescriptor::getFlagString(F); + const char *StringF = DebugNode::getFlagString(F); assert(StringF && "Expected valid flag"); Out << FlagsFS << StringF; } @@ -1505,7 +1503,7 @@ static void writeMDSubrange(raw_ostream &Out, const MDSubrange *N, Out << "!MDSubrange("; MDFieldPrinter Printer(Out); Printer.printInt("count", N->getCount(), /* ShouldSkipZero */ false); - Printer.printInt("lowerBound", N->getLo()); + Printer.printInt("lowerBound", N->getLowerBound()); Out << ")"; } @@ -1539,16 +1537,16 @@ static void writeMDDerivedType(raw_ostream &Out, const MDDerivedType *N, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printTag(N); Printer.printString("name", N->getName()); - Printer.printMetadata("scope", N->getScope()); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope()); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); - Printer.printMetadata("baseType", N->getBaseType(), + Printer.printMetadata("baseType", N->getRawBaseType(), /* ShouldSkipNull */ false); Printer.printInt("size", N->getSizeInBits()); Printer.printInt("align", N->getAlignInBits()); Printer.printInt("offset", N->getOffsetInBits()); Printer.printDIFlags("flags", N->getFlags()); - Printer.printMetadata("extraData", N->getExtraData()); + Printer.printMetadata("extraData", N->getRawExtraData()); Out << ")"; } @@ -1559,19 +1557,19 @@ static void writeMDCompositeType(raw_ostream &Out, const MDCompositeType *N, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printTag(N); Printer.printString("name", N->getName()); - Printer.printMetadata("scope", N->getScope()); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope()); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); - Printer.printMetadata("baseType", N->getBaseType()); + Printer.printMetadata("baseType", N->getRawBaseType()); Printer.printInt("size", N->getSizeInBits()); Printer.printInt("align", N->getAlignInBits()); Printer.printInt("offset", N->getOffsetInBits()); Printer.printDIFlags("flags", N->getFlags()); - Printer.printMetadata("elements", N->getElements()); + Printer.printMetadata("elements", N->getRawElements()); Printer.printDwarfEnum("runtimeLang", N->getRuntimeLang(), dwarf::LanguageString); - Printer.printMetadata("vtableHolder", N->getVTableHolder()); - Printer.printMetadata("templateParams", N->getTemplateParams()); + Printer.printMetadata("vtableHolder", N->getRawVTableHolder()); + Printer.printMetadata("templateParams", N->getRawTemplateParams()); Printer.printString("identifier", N->getIdentifier()); Out << ")"; } @@ -1582,7 +1580,8 @@ static void writeMDSubroutineType(raw_ostream &Out, const MDSubroutineType *N, Out << "!MDSubroutineType("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printDIFlags("flags", N->getFlags()); - Printer.printMetadata("types", N->getTypeArray(), /* ShouldSkipNull */ false); + Printer.printMetadata("types", N->getRawTypeArray(), + /* ShouldSkipNull */ false); Out << ")"; } @@ -1604,7 +1603,7 @@ static void writeMDCompileUnit(raw_ostream &Out, const MDCompileUnit *N, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printDwarfEnum("language", N->getSourceLanguage(), dwarf::LanguageString, /* ShouldSkipZero */ false); - Printer.printMetadata("file", N->getFile(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile(), /* ShouldSkipNull */ false); Printer.printString("producer", N->getProducer()); Printer.printBool("isOptimized", N->isOptimized()); Printer.printString("flags", N->getFlags()); @@ -1613,11 +1612,11 @@ static void writeMDCompileUnit(raw_ostream &Out, const MDCompileUnit *N, Printer.printString("splitDebugFilename", N->getSplitDebugFilename()); Printer.printInt("emissionKind", N->getEmissionKind(), /* ShouldSkipZero */ false); - Printer.printMetadata("enums", N->getEnumTypes()); - Printer.printMetadata("retainedTypes", N->getRetainedTypes()); - Printer.printMetadata("subprograms", N->getSubprograms()); - Printer.printMetadata("globals", N->getGlobalVariables()); - Printer.printMetadata("imports", N->getImportedEntities()); + Printer.printMetadata("enums", N->getRawEnumTypes()); + Printer.printMetadata("retainedTypes", N->getRawRetainedTypes()); + Printer.printMetadata("subprograms", N->getRawSubprograms()); + Printer.printMetadata("globals", N->getRawGlobalVariables()); + Printer.printMetadata("imports", N->getRawImportedEntities()); Out << ")"; } @@ -1628,23 +1627,23 @@ static void writeMDSubprogram(raw_ostream &Out, const MDSubprogram *N, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printString("name", N->getName()); Printer.printString("linkageName", N->getLinkageName()); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); - Printer.printMetadata("type", N->getType()); + Printer.printMetadata("type", N->getRawType()); Printer.printBool("isLocal", N->isLocalToUnit()); Printer.printBool("isDefinition", N->isDefinition()); Printer.printInt("scopeLine", N->getScopeLine()); - Printer.printMetadata("containingType", N->getContainingType()); + Printer.printMetadata("containingType", N->getRawContainingType()); Printer.printDwarfEnum("virtuality", N->getVirtuality(), dwarf::VirtualityString); Printer.printInt("virtualIndex", N->getVirtualIndex()); Printer.printDIFlags("flags", N->getFlags()); Printer.printBool("isOptimized", N->isOptimized()); - Printer.printMetadata("function", N->getFunction()); - Printer.printMetadata("templateParams", N->getTemplateParams()); - Printer.printMetadata("declaration", N->getDeclaration()); - Printer.printMetadata("variables", N->getVariables()); + Printer.printMetadata("function", N->getRawFunction()); + Printer.printMetadata("templateParams", N->getRawTemplateParams()); + Printer.printMetadata("declaration", N->getRawDeclaration()); + Printer.printMetadata("variables", N->getRawVariables()); Out << ")"; } @@ -1653,8 +1652,8 @@ static void writeMDLexicalBlock(raw_ostream &Out, const MDLexicalBlock *N, const Module *Context) { Out << "!MDLexicalBlock("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); Printer.printInt("column", N->getColumn()); Out << ")"; @@ -1667,8 +1666,8 @@ static void writeMDLexicalBlockFile(raw_ostream &Out, const Module *Context) { Out << "!MDLexicalBlockFile("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("discriminator", N->getDiscriminator(), /* ShouldSkipZero */ false); Out << ")"; @@ -1680,8 +1679,8 @@ static void writeMDNamespace(raw_ostream &Out, const MDNamespace *N, Out << "!MDNamespace("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printString("name", N->getName()); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); Out << ")"; } @@ -1694,7 +1693,7 @@ static void writeMDTemplateTypeParameter(raw_ostream &Out, Out << "!MDTemplateTypeParameter("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printString("name", N->getName()); - Printer.printMetadata("type", N->getType(), /* ShouldSkipNull */ false); + Printer.printMetadata("type", N->getRawType(), /* ShouldSkipNull */ false); Out << ")"; } @@ -1708,7 +1707,7 @@ static void writeMDTemplateValueParameter(raw_ostream &Out, if (N->getTag() != dwarf::DW_TAG_template_value_parameter) Printer.printTag(N); Printer.printString("name", N->getName()); - Printer.printMetadata("type", N->getType()); + Printer.printMetadata("type", N->getRawType()); Printer.printMetadata("value", N->getValue(), /* ShouldSkipNull */ false); Out << ")"; } @@ -1720,14 +1719,14 @@ static void writeMDGlobalVariable(raw_ostream &Out, const MDGlobalVariable *N, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printString("name", N->getName()); Printer.printString("linkageName", N->getLinkageName()); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); - Printer.printMetadata("type", N->getType()); + Printer.printMetadata("type", N->getRawType()); Printer.printBool("isLocal", N->isLocalToUnit()); Printer.printBool("isDefinition", N->isDefinition()); - Printer.printMetadata("variable", N->getVariable()); - Printer.printMetadata("declaration", N->getStaticDataMemberDeclaration()); + Printer.printMetadata("variable", N->getRawVariable()); + Printer.printMetadata("declaration", N->getRawStaticDataMemberDeclaration()); Out << ")"; } @@ -1741,12 +1740,11 @@ static void writeMDLocalVariable(raw_ostream &Out, const MDLocalVariable *N, Printer.printInt("arg", N->getArg(), /* ShouldSkipZero */ N->getTag() == dwarf::DW_TAG_auto_variable); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); - Printer.printMetadata("type", N->getType()); + Printer.printMetadata("type", N->getRawType()); Printer.printDIFlags("flags", N->getFlags()); - Printer.printMetadata("inlinedAt", N->getInlinedAt()); Out << ")"; } @@ -1777,12 +1775,12 @@ static void writeMDObjCProperty(raw_ostream &Out, const MDObjCProperty *N, Out << "!MDObjCProperty("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printString("name", N->getName()); - Printer.printMetadata("file", N->getFile()); + Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); Printer.printString("setter", N->getSetterName()); Printer.printString("getter", N->getGetterName()); Printer.printInt("attributes", N->getAttributes()); - Printer.printMetadata("type", N->getType()); + Printer.printMetadata("type", N->getRawType()); Out << ")"; } @@ -1793,8 +1791,8 @@ static void writeMDImportedEntity(raw_ostream &Out, const MDImportedEntity *N, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printTag(N); Printer.printString("name", N->getName()); - Printer.printMetadata("scope", N->getScope(), /* ShouldSkipNull */ false); - Printer.printMetadata("entity", N->getEntity()); + Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); + Printer.printMetadata("entity", N->getRawEntity()); Printer.printInt("line", N->getLine()); Out << ")"; } @@ -1943,16 +1941,19 @@ class AssemblyWriter { TypePrinting TypePrinter; AssemblyAnnotationWriter *AnnotationWriter; SetVector Comdats; + bool ShouldPreserveUseListOrder; UseListOrderStack UseListOrders; public: /// Construct an AssemblyWriter with an external SlotTracker - AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac, - const Module *M, AssemblyAnnotationWriter *AAW); + AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac, const Module *M, + AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder = false); /// Construct an AssemblyWriter with an internally allocated SlotTracker AssemblyWriter(formatted_raw_ostream &o, const Module *M, - AssemblyAnnotationWriter *AAW); + AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder = false); void printMDNodeBody(const MDNode *MD); void printNamedMDNode(const NamedMDNode *NMD); @@ -2004,18 +2005,20 @@ void AssemblyWriter::init() { Comdats.insert(C); } - AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac, - const Module *M, - AssemblyAnnotationWriter *AAW) - : Out(o), TheModule(M), Machine(Mac), AnnotationWriter(AAW) { + const Module *M, AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder) + : Out(o), TheModule(M), Machine(Mac), AnnotationWriter(AAW), + ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) { init(); } AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, const Module *M, - AssemblyAnnotationWriter *AAW) - : Out(o), TheModule(M), ModuleSlotTracker(createSlotTracker(M)), - Machine(*ModuleSlotTracker), AnnotationWriter(AAW) { + AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder) + : Out(o), TheModule(M), ModuleSlotTracker(createSlotTracker(M)), + Machine(*ModuleSlotTracker), AnnotationWriter(AAW), + ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) { init(); } @@ -2103,7 +2106,7 @@ void AssemblyWriter::writeParamOperand(const Value *Operand, void AssemblyWriter::printModule(const Module *M) { Machine.initialize(); - if (shouldPreserveAssemblyUseListOrder()) + if (ShouldPreserveUseListOrder) UseListOrders = predictUseListOrder(M); if (!M->getModuleIdentifier().empty() && @@ -2778,8 +2781,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { } Operand = CI->getCalledValue(); - PointerType *PTy = cast(Operand->getType()); - FunctionType *FTy = cast(PTy->getElementType()); + FunctionType *FTy = cast(CI->getFunctionType()); Type *RetTy = FTy->getReturnType(); const AttributeSet &PAL = CI->getAttributes(); @@ -2791,15 +2793,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) { // and if the return type is not a pointer to a function. // Out << ' '; - if (!FTy->isVarArg() && - (!RetTy->isPointerTy() || - !cast(RetTy)->getElementType()->isFunctionTy())) { - TypePrinter.print(RetTy, Out); - Out << ' '; - writeOperand(Operand, false); - } else { - writeOperand(Operand, true); - } + TypePrinter.print(FTy->isVarArg() ? FTy : RetTy, Out); + Out << ' '; + writeOperand(Operand, false); Out << '('; for (unsigned op = 0, Eop = CI->getNumArgOperands(); op < Eop; ++op) { if (op > 0) @@ -3060,10 +3056,18 @@ void AssemblyWriter::printUseLists(const Function *F) { // External Interface declarations //===----------------------------------------------------------------------===// -void Module::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW) const { +void Function::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW) const { + SlotTracker SlotTable(this->getParent()); + formatted_raw_ostream OS(ROS); + AssemblyWriter W(OS, SlotTable, this->getParent(), AAW); + W.printFunction(this); +} + +void Module::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder) const { SlotTracker SlotTable(this); formatted_raw_ostream OS(ROS); - AssemblyWriter W(OS, SlotTable, this, AAW); + AssemblyWriter W(OS, SlotTable, this, AAW, ShouldPreserveUseListOrder); W.printModule(this); } diff --git a/lib/IR/AttributeImpl.h b/lib/IR/AttributeImpl.h index 199c318..d544689 100644 --- a/lib/IR/AttributeImpl.h +++ b/lib/IR/AttributeImpl.h @@ -115,10 +115,10 @@ class IntAttributeImpl : public EnumAttributeImpl { public: IntAttributeImpl(Attribute::AttrKind Kind, uint64_t Val) : EnumAttributeImpl(IntAttrEntry, Kind), Val(Val) { - assert( - (Kind == Attribute::Alignment || Kind == Attribute::StackAlignment || - Kind == Attribute::Dereferenceable) && - "Wrong kind for int attribute!"); + assert((Kind == Attribute::Alignment || Kind == Attribute::StackAlignment || + Kind == Attribute::Dereferenceable || + Kind == Attribute::DereferenceableOrNull) && + "Wrong kind for int attribute!"); } uint64_t getValue() const { return Val; } diff --git a/lib/IR/Attributes.cpp b/lib/IR/Attributes.cpp index daac6b5..be5b74f 100644 --- a/lib/IR/Attributes.cpp +++ b/lib/IR/Attributes.cpp @@ -94,6 +94,12 @@ Attribute Attribute::getWithDereferenceableBytes(LLVMContext &Context, return get(Context, Dereferenceable, Bytes); } +Attribute Attribute::getWithDereferenceableOrNullBytes(LLVMContext &Context, + uint64_t Bytes) { + assert(Bytes && "Bytes must be non-zero."); + return get(Context, DereferenceableOrNull, Bytes); +} + //===----------------------------------------------------------------------===// // Attribute Accessor Methods //===----------------------------------------------------------------------===// @@ -170,6 +176,13 @@ uint64_t Attribute::getDereferenceableBytes() const { return pImpl->getValueAsInt(); } +uint64_t Attribute::getDereferenceableOrNullBytes() const { + assert(hasAttribute(Attribute::DereferenceableOrNull) && + "Trying to get dereferenceable bytes from " + "non-dereferenceable attribute!"); + return pImpl->getValueAsInt(); +} + std::string Attribute::getAsString(bool InAttrGrp) const { if (!pImpl) return ""; @@ -263,9 +276,9 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return Result; } - if (hasAttribute(Attribute::StackAlignment)) { + auto AttrWithBytesToString = [&](const char *Name) { std::string Result; - Result += "alignstack"; + Result += Name; if (InAttrGrp) { Result += "="; Result += utostr(getValueAsInt()); @@ -275,21 +288,16 @@ std::string Attribute::getAsString(bool InAttrGrp) const { Result += ")"; } return Result; - } + }; - if (hasAttribute(Attribute::Dereferenceable)) { - std::string Result; - Result += "dereferenceable"; - if (InAttrGrp) { - Result += "="; - Result += utostr(getValueAsInt()); - } else { - Result += "("; - Result += utostr(getValueAsInt()); - Result += ")"; - } - return Result; - } + if (hasAttribute(Attribute::StackAlignment)) + return AttrWithBytesToString("alignstack"); + + if (hasAttribute(Attribute::Dereferenceable)) + return AttrWithBytesToString("dereferenceable"); + + if (hasAttribute(Attribute::DereferenceableOrNull)) + return AttrWithBytesToString("dereferenceable_or_null"); // Convert target-dependent attributes to strings of the form: // @@ -298,12 +306,12 @@ std::string Attribute::getAsString(bool InAttrGrp) const { // if (isStringAttribute()) { std::string Result; - Result += '\"' + getKindAsString().str() + '"'; + Result += (Twine('"') + getKindAsString() + Twine('"')).str(); StringRef Val = pImpl->getValueAsString(); if (Val.empty()) return Result; - Result += "=\"" + Val.str() + '"'; + Result += ("=\"" + Val + Twine('"')).str(); return Result; } @@ -428,6 +436,11 @@ uint64_t AttributeImpl::getAttrMask(Attribute::AttrKind Val) { case Attribute::JumpTable: return 1ULL << 45; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); + break; + case Attribute::DereferenceableOrNull: + llvm_unreachable("dereferenceable_or_null attribute not supported in raw " + "format"); + break; } llvm_unreachable("Unsupported attribute type"); } @@ -663,6 +676,10 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, Attrs.push_back(std::make_pair(Index, Attribute::getWithDereferenceableBytes(C, B.getDereferenceableBytes()))); + else if (Kind == Attribute::DereferenceableOrNull) + Attrs.push_back( + std::make_pair(Index, Attribute::getWithDereferenceableOrNullBytes( + C, B.getDereferenceableOrNullBytes()))); else Attrs.push_back(std::make_pair(Index, Attribute::get(C, Kind))); } @@ -842,6 +859,14 @@ AttributeSet AttributeSet::addDereferenceableAttr(LLVMContext &C, unsigned Index return addAttributes(C, Index, AttributeSet::get(C, Index, B)); } +AttributeSet AttributeSet::addDereferenceableOrNullAttr(LLVMContext &C, + unsigned Index, + uint64_t Bytes) const { + llvm::AttrBuilder B; + B.addDereferenceableOrNullAttr(Bytes); + return addAttributes(C, Index, AttributeSet::get(C, Index, B)); +} + //===----------------------------------------------------------------------===// // AttributeSet Accessor Methods //===----------------------------------------------------------------------===// @@ -1011,7 +1036,8 @@ void AttributeSet::dump() const { //===----------------------------------------------------------------------===// AttrBuilder::AttrBuilder(AttributeSet AS, unsigned Index) - : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0) { + : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0), + DerefOrNullBytes(0) { AttributeSetImpl *pImpl = AS.pImpl; if (!pImpl) return; @@ -1028,7 +1054,7 @@ AttrBuilder::AttrBuilder(AttributeSet AS, unsigned Index) void AttrBuilder::clear() { Attrs.reset(); - Alignment = StackAlignment = DerefBytes = 0; + Alignment = StackAlignment = DerefBytes = DerefOrNullBytes = 0; } AttrBuilder &AttrBuilder::addAttribute(Attribute::AttrKind Val) { @@ -1055,6 +1081,8 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) { StackAlignment = Attr.getStackAlignment(); else if (Kind == Attribute::Dereferenceable) DerefBytes = Attr.getDereferenceableBytes(); + else if (Kind == Attribute::DereferenceableOrNull) + DerefOrNullBytes = Attr.getDereferenceableOrNullBytes(); return *this; } @@ -1073,6 +1101,8 @@ AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) { StackAlignment = 0; else if (Val == Attribute::Dereferenceable) DerefBytes = 0; + else if (Val == Attribute::DereferenceableOrNull) + DerefOrNullBytes = 0; return *this; } @@ -1099,6 +1129,8 @@ AttrBuilder &AttrBuilder::removeAttributes(AttributeSet A, uint64_t Index) { StackAlignment = 0; else if (Kind == Attribute::Dereferenceable) DerefBytes = 0; + else if (Kind == Attribute::DereferenceableOrNull) + DerefOrNullBytes = 0; } else { assert(Attr.isStringAttribute() && "Invalid attribute type!"); std::map::iterator @@ -1149,6 +1181,15 @@ AttrBuilder &AttrBuilder::addDereferenceableAttr(uint64_t Bytes) { return *this; } +AttrBuilder &AttrBuilder::addDereferenceableOrNullAttr(uint64_t Bytes) { + if (Bytes == 0) + return *this; + + Attrs[Attribute::DereferenceableOrNull] = true; + DerefOrNullBytes = Bytes; + return *this; +} + AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) { // FIXME: What if both have alignments, but they don't match?! if (!Alignment) @@ -1225,7 +1266,8 @@ AttrBuilder &AttrBuilder::addRawValue(uint64_t Val) { for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds; I = Attribute::AttrKind(I + 1)) { - if (I == Attribute::Dereferenceable) + if (I == Attribute::Dereferenceable || + I == Attribute::DereferenceableOrNull) continue; if (uint64_t A = (Val & AttributeImpl::getAttrMask(I))) { Attrs[I] = true; @@ -1261,6 +1303,7 @@ AttributeSet AttributeFuncs::typeIncompatible(Type *Ty, uint64_t Index) { .addAttribute(Attribute::NoCapture) .addAttribute(Attribute::NonNull) .addDereferenceableAttr(1) // the int here is ignored + .addDereferenceableOrNullAttr(1) // the int here is ignored .addAttribute(Attribute::ReadNone) .addAttribute(Attribute::ReadOnly) .addAttribute(Attribute::StructRet) diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp index d2dfeaa..bb23d2c 100644 --- a/lib/IR/AutoUpgrade.cpp +++ b/lib/IR/AutoUpgrade.cpp @@ -124,19 +124,6 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } break; } - case 'd': { - if (Name.startswith("dbg.declare") && F->arg_size() == 2) { - F->setName(Name + ".old"); - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare); - return true; - } - if (Name.startswith("dbg.value") && F->arg_size() == 3) { - F->setName(Name + ".old"); - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); - return true; - } - break; - } case 'o': // We only need to change the name to match the mangling including the @@ -354,23 +341,6 @@ bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { return false; } -static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) { - if (!DbgNode || Elt >= DbgNode->getNumOperands()) - return nullptr; - return dyn_cast_or_null(DbgNode->getOperand(Elt)); -} - -static MetadataAsValue *getExpression(Value *VarOperand, Function *F) { - // Old-style DIVariables have an optional expression as the 8th element. - DIExpression Expr(getNodeField( - cast(cast(VarOperand)->getMetadata()), 8)); - if (!Expr) { - DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false); - Expr = DIB.createExpression(); - } - return MetadataAsValue::get(F->getContext(), Expr); -} - // Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them // to byte shuffles. static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, @@ -745,7 +715,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { return; } - std::string Name = CI->getName().str(); + std::string Name = CI->getName(); if (!Name.empty()) CI->setName(Name + ".old"); @@ -753,25 +723,6 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { default: llvm_unreachable("Unknown function for CallInst upgrade."); - // Upgrade debug intrinsics to use an additional DIExpression argument. - case Intrinsic::dbg_declare: { - auto NewCI = - Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1), - getExpression(CI->getArgOperand(1), F), Name); - NewCI->setDebugLoc(CI->getDebugLoc()); - CI->replaceAllUsesWith(NewCI); - CI->eraseFromParent(); - return; - } - case Intrinsic::dbg_value: { - auto NewCI = Builder.CreateCall4( - NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), - getExpression(CI->getArgOperand(2), F), Name); - NewCI->setDebugLoc(CI->getDebugLoc()); - CI->replaceAllUsesWith(NewCI); - CI->eraseFromParent(); - return; - } case Intrinsic::ctlz: case Intrinsic::cttz: assert(CI->getNumArgOperands() == 1 && diff --git a/lib/IR/BasicBlock.cpp b/lib/IR/BasicBlock.cpp index fe38385..23ec705 100644 --- a/lib/IR/BasicBlock.cpp +++ b/lib/IR/BasicBlock.cpp @@ -94,8 +94,8 @@ void BasicBlock::removeFromParent() { getParent()->getBasicBlockList().remove(this); } -void BasicBlock::eraseFromParent() { - getParent()->getBasicBlockList().erase(this); +iplist::iterator BasicBlock::eraseFromParent() { + return getParent()->getBasicBlockList().erase(this); } /// Unlink this basic block from its current function and diff --git a/lib/IR/CMakeLists.txt b/lib/IR/CMakeLists.txt index 9fef0b2..d2e0c38 100644 --- a/lib/IR/CMakeLists.txt +++ b/lib/IR/CMakeLists.txt @@ -41,7 +41,6 @@ add_llvm_library(LLVMCore Type.cpp TypeFinder.cpp Use.cpp - UseListOrder.cpp User.cpp Value.cpp ValueSymbolTable.cpp diff --git a/lib/IR/ConstantFold.cpp b/lib/IR/ConstantFold.cpp index d97d2c4..d3caf04 100644 --- a/lib/IR/ConstantFold.cpp +++ b/lib/IR/ConstantFold.cpp @@ -132,7 +132,8 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { if (ElTy == DPTy->getElementType()) // This GEP is inbounds because all indices are zero. - return ConstantExpr::getInBoundsGetElementPtr(V, IdxList); + return ConstantExpr::getInBoundsGetElementPtr(PTy->getElementType(), + V, IdxList); } // Handle casts from one vector constant to another. We know that the src @@ -169,7 +170,8 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { // be the same. Consequently, we just fold to V. return V; - if (DestTy->isFloatingPointTy()) + // See note below regarding the PPC_FP128 restriction. + if (DestTy->isFloatingPointTy() && !DestTy->isPPC_FP128Ty()) return ConstantFP::get(DestTy->getContext(), APFloat(DestTy->getFltSemantics(), CI->getValue())); @@ -179,9 +181,19 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { } // Handle ConstantFP input: FP -> Integral. - if (ConstantFP *FP = dyn_cast(V)) + if (ConstantFP *FP = dyn_cast(V)) { + // PPC_FP128 is really the sum of two consecutive doubles, where the first + // double is always stored first in memory, regardless of the target + // endianness. The memory layout of i128, however, depends on the target + // endianness, and so we can't fold this without target endianness + // information. This should instead be handled by + // Analysis/ConstantFolding.cpp + if (FP->getType()->isPPC_FP128Ty()) + return nullptr; + return ConstantInt::get(FP->getContext(), FP->getValueAPF().bitcastToAPInt()); + } return nullptr; } @@ -2020,7 +2032,8 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C, if (isa(C)) { PointerType *Ptr = cast(C->getType()); - Type *Ty = GetElementPtrInst::getIndexedType(Ptr, Idxs); + Type *Ty = GetElementPtrInst::getIndexedType( + cast(Ptr->getScalarType())->getElementType(), Idxs); assert(Ty && "Invalid indices for GEP!"); return UndefValue::get(PointerType::get(Ty, Ptr->getAddressSpace())); } @@ -2034,7 +2047,8 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C, } if (isNull) { PointerType *Ptr = cast(C->getType()); - Type *Ty = GetElementPtrInst::getIndexedType(Ptr, Idxs); + Type *Ty = GetElementPtrInst::getIndexedType( + cast(Ptr->getScalarType())->getElementType(), Idxs); assert(Ty && "Invalid indices for GEP!"); return ConstantPointerNull::get(PointerType::get(Ty, Ptr->getAddressSpace())); @@ -2107,10 +2121,9 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C, NewIndices.push_back(Combined); NewIndices.append(Idxs.begin() + 1, Idxs.end()); - return - ConstantExpr::getGetElementPtr(CE->getOperand(0), NewIndices, - inBounds && - cast(CE)->isInBounds()); + return ConstantExpr::getGetElementPtr( + cast(CE)->getSourceElementType(), CE->getOperand(0), + NewIndices, inBounds && cast(CE)->isInBounds()); } } @@ -2135,8 +2148,8 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C, if (SrcArrayTy && DstArrayTy && SrcArrayTy->getElementType() == DstArrayTy->getElementType() && SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace()) - return ConstantExpr::getGetElementPtr((Constant*)CE->getOperand(0), - Idxs, inBounds); + return ConstantExpr::getGetElementPtr( + SrcArrayTy, (Constant *)CE->getOperand(0), Idxs, inBounds); } } } @@ -2202,7 +2215,7 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C, if (!NewIdxs.empty()) { for (unsigned i = 0, e = Idxs.size(); i != e; ++i) if (!NewIdxs[i]) NewIdxs[i] = cast(Idxs[i]); - return ConstantExpr::getGetElementPtr(C, NewIdxs, inBounds); + return ConstantExpr::getGetElementPtr(nullptr, C, NewIdxs, inBounds); } // If all indices are known integers and normalized, we can do a simple @@ -2210,7 +2223,7 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C, if (!Unknown && !inBounds) if (auto *GV = dyn_cast(C)) if (!GV->hasExternalWeakLinkage() && isInBoundsIndices(Idxs)) - return ConstantExpr::getInBoundsGetElementPtr(C, Idxs); + return ConstantExpr::getInBoundsGetElementPtr(nullptr, C, Idxs); return nullptr; } diff --git a/lib/IR/Constants.cpp b/lib/IR/Constants.cpp index e51a396..3f8d1f1 100644 --- a/lib/IR/Constants.cpp +++ b/lib/IR/Constants.cpp @@ -1252,7 +1252,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef Ops, Type *Ty, return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2], OnlyIfReducedTy); case Instruction::GetElementPtr: - return ConstantExpr::getGetElementPtr(Ops[0], Ops.slice(1), + return ConstantExpr::getGetElementPtr(nullptr, Ops[0], Ops.slice(1), cast(this)->isInBounds(), OnlyIfReducedTy); case Instruction::ICmp: @@ -1925,7 +1925,7 @@ Constant *ConstantExpr::getSizeOf(Type* Ty) { // Note that a non-inbounds gep is used, as null isn't within any object. Constant *GEPIdx = ConstantInt::get(Type::getInt32Ty(Ty->getContext()), 1); Constant *GEP = getGetElementPtr( - Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx); + Ty, Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx); return getPtrToInt(GEP, Type::getInt64Ty(Ty->getContext())); } @@ -1939,7 +1939,7 @@ Constant *ConstantExpr::getAlignOf(Type* Ty) { Constant *Zero = ConstantInt::get(Type::getInt64Ty(Ty->getContext()), 0); Constant *One = ConstantInt::get(Type::getInt32Ty(Ty->getContext()), 1); Constant *Indices[2] = { Zero, One }; - Constant *GEP = getGetElementPtr(NullPtr, Indices); + Constant *GEP = getGetElementPtr(AligningTy, NullPtr, Indices); return getPtrToInt(GEP, Type::getInt64Ty(Ty->getContext())); } @@ -1957,7 +1957,7 @@ Constant *ConstantExpr::getOffsetOf(Type* Ty, Constant *FieldNo) { FieldNo }; Constant *GEP = getGetElementPtr( - Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx); + Ty, Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx); return getPtrToInt(GEP, Type::getInt64Ty(Ty->getContext())); } @@ -2001,19 +2001,22 @@ Constant *ConstantExpr::getSelect(Constant *C, Constant *V1, Constant *V2, return pImpl->ExprConstants.getOrCreate(V1->getType(), Key); } -Constant *ConstantExpr::getGetElementPtr(Constant *C, ArrayRef Idxs, - bool InBounds, Type *OnlyIfReducedTy) { - assert(C->getType()->isPtrOrPtrVectorTy() && - "Non-pointer type for constant GetElementPtr expression"); - +Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, + ArrayRef Idxs, bool InBounds, + Type *OnlyIfReducedTy) { if (Constant *FC = ConstantFoldGetElementPtr(C, InBounds, Idxs)) return FC; // Fold a few common cases. + if (!Ty) + Ty = cast(C->getType()->getScalarType())->getElementType(); + else + assert(Ty == + cast(C->getType()->getScalarType())->getElementType()); // Get the result type of the getelementptr! - Type *Ty = GetElementPtrInst::getIndexedType(C->getType(), Idxs); - assert(Ty && "GEP indices invalid!"); + Type *DestTy = GetElementPtrInst::getIndexedType(Ty, Idxs); + assert(DestTy && "GEP indices invalid!"); unsigned AS = C->getType()->getPointerAddressSpace(); - Type *ReqTy = Ty->getPointerTo(AS); + Type *ReqTy = DestTy->getPointerTo(AS); if (VectorType *VecTy = dyn_cast(C->getType())) ReqTy = VectorType::get(ReqTy, VecTy->getNumElements()); diff --git a/lib/IR/Core.cpp b/lib/IR/Core.cpp index 613147e..7fe7beb 100644 --- a/lib/IR/Core.cpp +++ b/lib/IR/Core.cpp @@ -1153,8 +1153,8 @@ LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal, LLVMValueRef *ConstantIndices, unsigned NumIndices) { ArrayRef IdxList(unwrap(ConstantIndices, NumIndices), NumIndices); - return wrap(ConstantExpr::getGetElementPtr(unwrap(ConstantVal), - IdxList)); + return wrap(ConstantExpr::getGetElementPtr( + nullptr, unwrap(ConstantVal), IdxList)); } LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal, @@ -1163,7 +1163,7 @@ LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal, Constant* Val = unwrap(ConstantVal); ArrayRef IdxList(unwrap(ConstantIndices, NumIndices), NumIndices); - return wrap(ConstantExpr::getInBoundsGetElementPtr(Val, IdxList)); + return wrap(ConstantExpr::getInBoundsGetElementPtr(nullptr, Val, IdxList)); } LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType) { @@ -2181,14 +2181,13 @@ void LLVMDisposeBuilder(LLVMBuilderRef Builder) { void LLVMSetCurrentDebugLocation(LLVMBuilderRef Builder, LLVMValueRef L) { MDNode *Loc = L ? cast(unwrap(L)->getMetadata()) : nullptr; - unwrap(Builder)->SetCurrentDebugLocation(DebugLoc::getFromDILocation(Loc)); + unwrap(Builder)->SetCurrentDebugLocation(DebugLoc(Loc)); } LLVMValueRef LLVMGetCurrentDebugLocation(LLVMBuilderRef Builder) { LLVMContext &Context = unwrap(Builder)->getContext(); return wrap(MetadataAsValue::get( - Context, - unwrap(Builder)->getCurrentDebugLocation().getAsMDNode(Context))); + Context, unwrap(Builder)->getCurrentDebugLocation().getAsMDNode())); } void LLVMSetInstDebugLocation(LLVMBuilderRef Builder, LLVMValueRef Inst) { @@ -2513,12 +2512,13 @@ LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer, LLVMValueRef *Indices, unsigned NumIndices, const char *Name) { ArrayRef IdxList(unwrap(Indices), NumIndices); - return wrap(unwrap(B)->CreateInBoundsGEP(unwrap(Pointer), IdxList, Name)); + return wrap( + unwrap(B)->CreateInBoundsGEP(nullptr, unwrap(Pointer), IdxList, Name)); } LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer, unsigned Idx, const char *Name) { - return wrap(unwrap(B)->CreateStructGEP(unwrap(Pointer), Idx, Name)); + return wrap(unwrap(B)->CreateStructGEP(nullptr, unwrap(Pointer), Idx, Name)); } LLVMValueRef LLVMBuildGlobalString(LLVMBuilderRef B, const char *Str, diff --git a/lib/IR/DIBuilder.cpp b/lib/IR/DIBuilder.cpp index 9677de4..891fb86 100644 --- a/lib/IR/DIBuilder.cpp +++ b/lib/IR/DIBuilder.cpp @@ -74,8 +74,7 @@ void DIBuilder::trackIfUnresolved(MDNode *N) { } void DIBuilder::finalize() { - DIArray Enums = getOrCreateArray(AllEnumTypes); - DIType(TempEnumTypes).replaceAllUsesWith(Enums); + TempEnumTypes->replaceAllUsesWith(MDTuple::get(VMContext, AllEnumTypes)); SmallVector RetainValues; // Declarations and definitions of the same type may be retained. Some @@ -86,28 +85,24 @@ void DIBuilder::finalize() { for (unsigned I = 0, E = AllRetainTypes.size(); I < E; I++) if (RetainSet.insert(AllRetainTypes[I]).second) RetainValues.push_back(AllRetainTypes[I]); - DIArray RetainTypes = getOrCreateArray(RetainValues); - DIType(TempRetainTypes).replaceAllUsesWith(RetainTypes); - - DIArray SPs = getOrCreateArray(AllSubprograms); - DIType(TempSubprograms).replaceAllUsesWith(SPs); - for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) { - DISubprogram SP(SPs.getElement(i)); - if (MDNode *Temp = SP.getVariablesNodes()) { + TempRetainTypes->replaceAllUsesWith(MDTuple::get(VMContext, RetainValues)); + + MDSubprogramArray SPs = MDTuple::get(VMContext, AllSubprograms); + TempSubprograms->replaceAllUsesWith(SPs.get()); + for (auto *SP : SPs) { + if (MDTuple *Temp = SP->getVariables().get()) { const auto &PV = PreservedVariables.lookup(SP); SmallVector Variables(PV.begin(), PV.end()); DIArray AV = getOrCreateArray(Variables); - DIType(Temp).replaceAllUsesWith(AV); + TempMDTuple(Temp)->replaceAllUsesWith(AV.get()); } } - DIArray GVs = getOrCreateArray(AllGVs); - DIType(TempGVs).replaceAllUsesWith(GVs); + TempGVs->replaceAllUsesWith(MDTuple::get(VMContext, AllGVs)); - SmallVector RetainValuesI(AllImportedModules.begin(), - AllImportedModules.end()); - DIArray IMs = getOrCreateArray(RetainValuesI); - DIType(TempImportedModules).replaceAllUsesWith(IMs); + TempImportedModules->replaceAllUsesWith(MDTuple::get( + VMContext, SmallVector(AllImportedModules.begin(), + AllImportedModules.end()))); // Now that all temp nodes have been replaced or deleted, resolve remaining // cycles. @@ -121,19 +116,16 @@ void DIBuilder::finalize() { } /// If N is compile unit return NULL otherwise return N. -static MDScope *getNonCompileUnitScope(MDNode *N) { +static MDScope *getNonCompileUnitScope(MDScope *N) { if (!N || isa(N)) return nullptr; return cast(N); } -DICompileUnit DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename, - StringRef Directory, - StringRef Producer, bool isOptimized, - StringRef Flags, unsigned RunTimeVer, - StringRef SplitName, - DebugEmissionKind Kind, - bool EmitDebugInfo) { +MDCompileUnit *DIBuilder::createCompileUnit( + unsigned Lang, StringRef Filename, StringRef Directory, StringRef Producer, + bool isOptimized, StringRef Flags, unsigned RunTimeVer, StringRef SplitName, + DebugEmissionKind Kind, bool EmitDebugInfo) { assert(((Lang <= dwarf::DW_LANG_Fortran08 && Lang >= dwarf::DW_LANG_C89) || (Lang <= dwarf::DW_LANG_hi_user && Lang >= dwarf::DW_LANG_lo_user)) && @@ -143,18 +135,19 @@ DICompileUnit DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename, // TODO: Once we make MDCompileUnit distinct, stop using temporaries here // (just start with operands assigned to nullptr). - TempEnumTypes = MDTuple::getTemporary(VMContext, None).release(); - TempRetainTypes = MDTuple::getTemporary(VMContext, None).release(); - TempSubprograms = MDTuple::getTemporary(VMContext, None).release(); - TempGVs = MDTuple::getTemporary(VMContext, None).release(); - TempImportedModules = MDTuple::getTemporary(VMContext, None).release(); + TempEnumTypes = MDTuple::getTemporary(VMContext, None); + TempRetainTypes = MDTuple::getTemporary(VMContext, None); + TempSubprograms = MDTuple::getTemporary(VMContext, None); + TempGVs = MDTuple::getTemporary(VMContext, None); + TempImportedModules = MDTuple::getTemporary(VMContext, None); // TODO: Switch to getDistinct(). We never want to merge compile units based // on contents. - MDNode *CUNode = MDCompileUnit::get( + MDCompileUnit *CUNode = MDCompileUnit::get( VMContext, Lang, MDFile::get(VMContext, Filename, Directory), Producer, - isOptimized, Flags, RunTimeVer, SplitName, Kind, TempEnumTypes, - TempRetainTypes, TempSubprograms, TempGVs, TempImportedModules); + isOptimized, Flags, RunTimeVer, SplitName, Kind, TempEnumTypes.get(), + TempRetainTypes.get(), TempSubprograms.get(), TempGVs.get(), + TempImportedModules.get()); // Create a named metadata so that it is easier to find cu in a module. // Note that we only generate this when the caller wants to actually @@ -167,141 +160,136 @@ DICompileUnit DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename, } trackIfUnresolved(CUNode); - return DICompileUnit(CUNode); + return CUNode; } -static DIImportedEntity -createImportedModule(LLVMContext &C, dwarf::Tag Tag, DIScope Context, +static MDImportedEntity* +createImportedModule(LLVMContext &C, dwarf::Tag Tag, MDScope* Context, Metadata *NS, unsigned Line, StringRef Name, SmallVectorImpl &AllImportedModules) { - DIImportedEntity M = MDImportedEntity::get(C, Tag, Context, NS, Line, Name); - assert(M.Verify() && "Imported module should be valid"); - AllImportedModules.emplace_back(M.get()); + auto *M = + MDImportedEntity::get(C, Tag, Context, DebugNodeRef(NS), Line, Name); + AllImportedModules.emplace_back(M); return M; } -DIImportedEntity DIBuilder::createImportedModule(DIScope Context, - DINameSpace NS, +MDImportedEntity* DIBuilder::createImportedModule(MDScope* Context, + MDNamespace* NS, unsigned Line) { return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module, Context, NS, Line, StringRef(), AllImportedModules); } -DIImportedEntity DIBuilder::createImportedModule(DIScope Context, - DIImportedEntity NS, +MDImportedEntity* DIBuilder::createImportedModule(MDScope* Context, + MDImportedEntity* NS, unsigned Line) { return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module, Context, NS, Line, StringRef(), AllImportedModules); } -DIImportedEntity DIBuilder::createImportedDeclaration(DIScope Context, - DIDescriptor Decl, - unsigned Line, StringRef Name) { +MDImportedEntity *DIBuilder::createImportedDeclaration(MDScope *Context, + DebugNode *Decl, + unsigned Line, + StringRef Name) { // Make sure to use the unique identifier based metadata reference for // types that have one. - Metadata *V = - Decl.isType() ? static_cast(DIType(Decl).getRef()) : Decl; return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_declaration, - Context, V, Line, Name, + Context, DebugNodeRef::get(Decl), Line, Name, AllImportedModules); } -DIImportedEntity DIBuilder::createImportedDeclaration(DIScope Context, - DIImportedEntity Imp, - unsigned Line, StringRef Name) { - return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_declaration, - Context, Imp, Line, Name, AllImportedModules); -} - -DIFile DIBuilder::createFile(StringRef Filename, StringRef Directory) { +MDFile* DIBuilder::createFile(StringRef Filename, StringRef Directory) { return MDFile::get(VMContext, Filename, Directory); } -DIEnumerator DIBuilder::createEnumerator(StringRef Name, int64_t Val) { +MDEnumerator *DIBuilder::createEnumerator(StringRef Name, int64_t Val) { assert(!Name.empty() && "Unable to create enumerator without name"); return MDEnumerator::get(VMContext, Val, Name); } -DIBasicType DIBuilder::createUnspecifiedType(StringRef Name) { +MDBasicType *DIBuilder::createUnspecifiedType(StringRef Name) { assert(!Name.empty() && "Unable to create type without name"); return MDBasicType::get(VMContext, dwarf::DW_TAG_unspecified_type, Name); } -DIBasicType DIBuilder::createNullPtrType() { +MDBasicType *DIBuilder::createNullPtrType() { return createUnspecifiedType("decltype(nullptr)"); } -DIBasicType -DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits, - uint64_t AlignInBits, unsigned Encoding) { +MDBasicType *DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits, + uint64_t AlignInBits, + unsigned Encoding) { assert(!Name.empty() && "Unable to create type without name"); return MDBasicType::get(VMContext, dwarf::DW_TAG_base_type, Name, SizeInBits, AlignInBits, Encoding); } -DIDerivedType DIBuilder::createQualifiedType(unsigned Tag, DIType FromTy) { +MDDerivedType *DIBuilder::createQualifiedType(unsigned Tag, MDType *FromTy) { return MDDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, - FromTy.getRef(), 0, 0, 0, 0); + MDTypeRef::get(FromTy), 0, 0, 0, 0); } -DIDerivedType -DIBuilder::createPointerType(DIType PointeeTy, uint64_t SizeInBits, - uint64_t AlignInBits, StringRef Name) { +MDDerivedType *DIBuilder::createPointerType(MDType *PointeeTy, + uint64_t SizeInBits, + uint64_t AlignInBits, + StringRef Name) { // FIXME: Why is there a name here? return MDDerivedType::get(VMContext, dwarf::DW_TAG_pointer_type, Name, - nullptr, 0, nullptr, PointeeTy.getRef(), SizeInBits, - AlignInBits, 0, 0); + nullptr, 0, nullptr, MDTypeRef::get(PointeeTy), + SizeInBits, AlignInBits, 0, 0); } -DIDerivedType -DIBuilder::createMemberPointerType(DIType PointeeTy, DIType Base, - uint64_t SizeInBits, uint64_t AlignInBits) { +MDDerivedType *DIBuilder::createMemberPointerType(MDType *PointeeTy, + MDType *Base, + uint64_t SizeInBits, + uint64_t AlignInBits) { return MDDerivedType::get(VMContext, dwarf::DW_TAG_ptr_to_member_type, "", - nullptr, 0, nullptr, PointeeTy.getRef(), SizeInBits, - AlignInBits, 0, 0, Base.getRef()); + nullptr, 0, nullptr, MDTypeRef::get(PointeeTy), + SizeInBits, AlignInBits, 0, 0, MDTypeRef::get(Base)); } -DIDerivedType DIBuilder::createReferenceType(unsigned Tag, DIType RTy) { - assert(RTy.isType() && "Unable to create reference type"); +MDDerivedType *DIBuilder::createReferenceType(unsigned Tag, MDType *RTy) { + assert(RTy && "Unable to create reference type"); return MDDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, - RTy.getRef(), 0, 0, 0, 0); + MDTypeRef::get(RTy), 0, 0, 0, 0); } -DIDerivedType DIBuilder::createTypedef(DIType Ty, StringRef Name, DIFile File, - unsigned LineNo, DIDescriptor Context) { - return MDDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, - File.getFileNode(), LineNo, - DIScope(getNonCompileUnitScope(Context)).getRef(), - Ty.getRef(), 0, 0, 0, 0); +MDDerivedType *DIBuilder::createTypedef(MDType *Ty, StringRef Name, + MDFile *File, unsigned LineNo, + MDScope *Context) { + return MDDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, + LineNo, + MDScopeRef::get(getNonCompileUnitScope(Context)), + MDTypeRef::get(Ty), 0, 0, 0, 0); } -DIDerivedType DIBuilder::createFriend(DIType Ty, DIType FriendTy) { - // typedefs are encoded in DIDerivedType format. - assert(Ty.isType() && "Invalid type!"); - assert(FriendTy.isType() && "Invalid friend type!"); +MDDerivedType *DIBuilder::createFriend(MDType *Ty, MDType *FriendTy) { + assert(Ty && "Invalid type!"); + assert(FriendTy && "Invalid friend type!"); return MDDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0, - Ty.getRef(), FriendTy.getRef(), 0, 0, 0, 0); + MDTypeRef::get(Ty), MDTypeRef::get(FriendTy), 0, 0, + 0, 0); } -DIDerivedType DIBuilder::createInheritance(DIType Ty, DIType BaseTy, - uint64_t BaseOffset, - unsigned Flags) { - assert(Ty.isType() && "Unable to create inheritance"); +MDDerivedType *DIBuilder::createInheritance(MDType *Ty, MDType *BaseTy, + uint64_t BaseOffset, + unsigned Flags) { + assert(Ty && "Unable to create inheritance"); return MDDerivedType::get(VMContext, dwarf::DW_TAG_inheritance, "", nullptr, - 0, Ty.getRef(), BaseTy.getRef(), 0, 0, BaseOffset, - Flags); + 0, MDTypeRef::get(Ty), MDTypeRef::get(BaseTy), 0, 0, + BaseOffset, Flags); } -DIDerivedType DIBuilder::createMemberType(DIDescriptor Scope, StringRef Name, - DIFile File, unsigned LineNumber, - uint64_t SizeInBits, - uint64_t AlignInBits, - uint64_t OffsetInBits, unsigned Flags, - DIType Ty) { +MDDerivedType *DIBuilder::createMemberType(MDScope *Scope, StringRef Name, + MDFile *File, unsigned LineNumber, + uint64_t SizeInBits, + uint64_t AlignInBits, + uint64_t OffsetInBits, + unsigned Flags, MDType *Ty) { return MDDerivedType::get( VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, - DIScope(getNonCompileUnitScope(Scope)).getRef(), Ty.getRef(), SizeInBits, - AlignInBits, OffsetInBits, Flags); + MDScopeRef::get(getNonCompileUnitScope(Scope)), MDTypeRef::get(Ty), + SizeInBits, AlignInBits, OffsetInBits, Flags); } static ConstantAsMetadata *getConstantOrNull(Constant *C) { @@ -310,135 +298,124 @@ static ConstantAsMetadata *getConstantOrNull(Constant *C) { return nullptr; } -DIDerivedType DIBuilder::createStaticMemberType(DIDescriptor Scope, - StringRef Name, DIFile File, - unsigned LineNumber, DIType Ty, - unsigned Flags, - llvm::Constant *Val) { - // TAG_member is encoded in DIDerivedType format. - Flags |= DIDescriptor::FlagStaticMember; +MDDerivedType *DIBuilder::createStaticMemberType(MDScope *Scope, StringRef Name, + MDFile *File, + unsigned LineNumber, + MDType *Ty, unsigned Flags, + llvm::Constant *Val) { + Flags |= DebugNode::FlagStaticMember; return MDDerivedType::get( VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, - DIScope(getNonCompileUnitScope(Scope)).getRef(), Ty.getRef(), 0, 0, 0, - Flags, getConstantOrNull(Val)); + MDScopeRef::get(getNonCompileUnitScope(Scope)), MDTypeRef::get(Ty), 0, 0, + 0, Flags, getConstantOrNull(Val)); } -DIDerivedType DIBuilder::createObjCIVar(StringRef Name, DIFile File, - unsigned LineNumber, - uint64_t SizeInBits, - uint64_t AlignInBits, - uint64_t OffsetInBits, unsigned Flags, - DIType Ty, MDNode *PropertyNode) { - return MDDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, - LineNumber, getNonCompileUnitScope(File), - Ty.getRef(), SizeInBits, AlignInBits, OffsetInBits, - Flags, PropertyNode); +MDDerivedType *DIBuilder::createObjCIVar(StringRef Name, MDFile *File, + unsigned LineNumber, + uint64_t SizeInBits, + uint64_t AlignInBits, + uint64_t OffsetInBits, unsigned Flags, + MDType *Ty, MDNode *PropertyNode) { + return MDDerivedType::get( + VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, + MDScopeRef::get(getNonCompileUnitScope(File)), MDTypeRef::get(Ty), + SizeInBits, AlignInBits, OffsetInBits, Flags, PropertyNode); } -DIObjCProperty -DIBuilder::createObjCProperty(StringRef Name, DIFile File, unsigned LineNumber, +MDObjCProperty * +DIBuilder::createObjCProperty(StringRef Name, MDFile *File, unsigned LineNumber, StringRef GetterName, StringRef SetterName, - unsigned PropertyAttributes, DIType Ty) { + unsigned PropertyAttributes, MDType *Ty) { return MDObjCProperty::get(VMContext, Name, File, LineNumber, GetterName, SetterName, PropertyAttributes, Ty); } -DITemplateTypeParameter -DIBuilder::createTemplateTypeParameter(DIDescriptor Context, StringRef Name, - DIType Ty) { - assert(!DIScope(getNonCompileUnitScope(Context)).getRef() && - "Expected compile unit"); - return MDTemplateTypeParameter::get(VMContext, Name, Ty.getRef()); +MDTemplateTypeParameter * +DIBuilder::createTemplateTypeParameter(MDScope *Context, StringRef Name, + MDType *Ty) { + assert((!Context || isa(Context)) && "Expected compile unit"); + return MDTemplateTypeParameter::get(VMContext, Name, MDTypeRef::get(Ty)); } -static DITemplateValueParameter +static MDTemplateValueParameter * createTemplateValueParameterHelper(LLVMContext &VMContext, unsigned Tag, - DIDescriptor Context, StringRef Name, - DIType Ty, Metadata *MD) { - assert(!DIScope(getNonCompileUnitScope(Context)).getRef() && - "Expected compile unit"); - return MDTemplateValueParameter::get(VMContext, Tag, Name, Ty.getRef(), MD); + MDScope *Context, StringRef Name, MDType *Ty, + Metadata *MD) { + assert((!Context || isa(Context)) && "Expected compile unit"); + return MDTemplateValueParameter::get(VMContext, Tag, Name, MDTypeRef::get(Ty), + MD); } -DITemplateValueParameter -DIBuilder::createTemplateValueParameter(DIDescriptor Context, StringRef Name, - DIType Ty, Constant *Val) { +MDTemplateValueParameter * +DIBuilder::createTemplateValueParameter(MDScope *Context, StringRef Name, + MDType *Ty, Constant *Val) { return createTemplateValueParameterHelper( VMContext, dwarf::DW_TAG_template_value_parameter, Context, Name, Ty, getConstantOrNull(Val)); } -DITemplateValueParameter -DIBuilder::createTemplateTemplateParameter(DIDescriptor Context, StringRef Name, - DIType Ty, StringRef Val) { +MDTemplateValueParameter * +DIBuilder::createTemplateTemplateParameter(MDScope *Context, StringRef Name, + MDType *Ty, StringRef Val) { return createTemplateValueParameterHelper( VMContext, dwarf::DW_TAG_GNU_template_template_param, Context, Name, Ty, MDString::get(VMContext, Val)); } -DITemplateValueParameter -DIBuilder::createTemplateParameterPack(DIDescriptor Context, StringRef Name, - DIType Ty, DIArray Val) { +MDTemplateValueParameter * +DIBuilder::createTemplateParameterPack(MDScope *Context, StringRef Name, + MDType *Ty, DIArray Val) { return createTemplateValueParameterHelper( VMContext, dwarf::DW_TAG_GNU_template_parameter_pack, Context, Name, Ty, - Val); + Val.get()); } -DICompositeType DIBuilder::createClassType(DIDescriptor Context, StringRef Name, - DIFile File, unsigned LineNumber, - uint64_t SizeInBits, - uint64_t AlignInBits, - uint64_t OffsetInBits, - unsigned Flags, DIType DerivedFrom, - DIArray Elements, - DIType VTableHolder, - MDNode *TemplateParams, - StringRef UniqueIdentifier) { - assert((!Context || Context.isScope() || Context.isType()) && +MDCompositeType *DIBuilder::createClassType( + MDScope *Context, StringRef Name, MDFile *File, unsigned LineNumber, + uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, + unsigned Flags, MDType *DerivedFrom, DIArray Elements, MDType *VTableHolder, + MDNode *TemplateParams, StringRef UniqueIdentifier) { + assert((!Context || isa(Context)) && "createClassType should be called with a valid Context"); - // TAG_class_type is encoded in DICompositeType format. - DICompositeType R = MDCompositeType::get( + + auto *R = MDCompositeType::get( VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber, - DIScope(getNonCompileUnitScope(Context)).getRef(), DerivedFrom.getRef(), - SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, 0, - VTableHolder.getRef(), TemplateParams, UniqueIdentifier); + MDScopeRef::get(getNonCompileUnitScope(Context)), + MDTypeRef::get(DerivedFrom), SizeInBits, AlignInBits, OffsetInBits, Flags, + Elements, 0, MDTypeRef::get(VTableHolder), + cast_or_null(TemplateParams), UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(R); trackIfUnresolved(R); return R; } -DICompositeType DIBuilder::createStructType(DIDescriptor Context, - StringRef Name, DIFile File, - unsigned LineNumber, - uint64_t SizeInBits, - uint64_t AlignInBits, - unsigned Flags, DIType DerivedFrom, - DIArray Elements, - unsigned RunTimeLang, - DIType VTableHolder, - StringRef UniqueIdentifier) { - DICompositeType R = MDCompositeType::get( +MDCompositeType *DIBuilder::createStructType( + MDScope *Context, StringRef Name, MDFile *File, unsigned LineNumber, + uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, + MDType *DerivedFrom, DIArray Elements, unsigned RunTimeLang, + MDType *VTableHolder, StringRef UniqueIdentifier) { + auto *R = MDCompositeType::get( VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber, - DIScope(getNonCompileUnitScope(Context)).getRef(), DerivedFrom.getRef(), - SizeInBits, AlignInBits, 0, Flags, Elements, RunTimeLang, - VTableHolder.getRef(), nullptr, UniqueIdentifier); + MDScopeRef::get(getNonCompileUnitScope(Context)), + MDTypeRef::get(DerivedFrom), SizeInBits, AlignInBits, 0, Flags, Elements, + RunTimeLang, MDTypeRef::get(VTableHolder), nullptr, UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(R); trackIfUnresolved(R); return R; } -DICompositeType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name, - DIFile File, unsigned LineNumber, +MDCompositeType* DIBuilder::createUnionType(MDScope * Scope, StringRef Name, + MDFile* File, unsigned LineNumber, uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, DIArray Elements, unsigned RunTimeLang, StringRef UniqueIdentifier) { - DICompositeType R = MDCompositeType::get( + auto *R = MDCompositeType::get( VMContext, dwarf::DW_TAG_union_type, Name, File, LineNumber, - DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, SizeInBits, + MDScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits, AlignInBits, 0, Flags, Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier); if (!UniqueIdentifier.empty()) @@ -447,21 +424,21 @@ DICompositeType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name, return R; } -DISubroutineType DIBuilder::createSubroutineType(DIFile File, - DITypeArray ParameterTypes, - unsigned Flags) { +MDSubroutineType *DIBuilder::createSubroutineType(MDFile *File, + DITypeArray ParameterTypes, + unsigned Flags) { return MDSubroutineType::get(VMContext, Flags, ParameterTypes); } -DICompositeType DIBuilder::createEnumerationType( - DIDescriptor Scope, StringRef Name, DIFile File, unsigned LineNumber, +MDCompositeType *DIBuilder::createEnumerationType( + MDScope *Scope, StringRef Name, MDFile *File, unsigned LineNumber, uint64_t SizeInBits, uint64_t AlignInBits, DIArray Elements, - DIType UnderlyingType, StringRef UniqueIdentifier) { - DICompositeType CTy = MDCompositeType::get( + MDType *UnderlyingType, StringRef UniqueIdentifier) { + auto *CTy = MDCompositeType::get( VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber, - DIScope(getNonCompileUnitScope(Scope)).getRef(), UnderlyingType.getRef(), - SizeInBits, AlignInBits, 0, 0, Elements, 0, nullptr, nullptr, - UniqueIdentifier); + MDScopeRef::get(getNonCompileUnitScope(Scope)), + MDTypeRef::get(UnderlyingType), SizeInBits, AlignInBits, 0, 0, Elements, + 0, nullptr, nullptr, UniqueIdentifier); AllEnumTypes.push_back(CTy); if (!UniqueIdentifier.empty()) retainType(CTy); @@ -469,63 +446,66 @@ DICompositeType DIBuilder::createEnumerationType( return CTy; } -DICompositeType DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, - DIType Ty, DIArray Subscripts) { +MDCompositeType *DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, + MDType *Ty, DIArray Subscripts) { auto *R = MDCompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", - nullptr, 0, nullptr, Ty.getRef(), Size, + nullptr, 0, nullptr, MDTypeRef::get(Ty), Size, AlignInBits, 0, 0, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } -DICompositeType DIBuilder::createVectorType(uint64_t Size, uint64_t AlignInBits, - DIType Ty, DIArray Subscripts) { - auto *R = MDCompositeType::get( - VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, nullptr, Ty.getRef(), - Size, AlignInBits, 0, DIType::FlagVector, Subscripts, 0, nullptr); +MDCompositeType *DIBuilder::createVectorType(uint64_t Size, + uint64_t AlignInBits, MDType *Ty, + DIArray Subscripts) { + auto *R = + MDCompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, + nullptr, MDTypeRef::get(Ty), Size, AlignInBits, 0, + DebugNode::FlagVector, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } -static DIType createTypeWithFlags(LLVMContext &Context, DIType Ty, - unsigned FlagsToSet) { - TempMDType NewTy = cast(static_cast(Ty))->clone(); +static MDType *createTypeWithFlags(LLVMContext &Context, MDType *Ty, + unsigned FlagsToSet) { + auto NewTy = Ty->clone(); NewTy->setFlags(NewTy->getFlags() | FlagsToSet); return MDNode::replaceWithUniqued(std::move(NewTy)); } -DIType DIBuilder::createArtificialType(DIType Ty) { +MDType *DIBuilder::createArtificialType(MDType *Ty) { // FIXME: Restrict this to the nodes where it's valid. - if (Ty.isArtificial()) + if (Ty->isArtificial()) return Ty; - return createTypeWithFlags(VMContext, Ty, DIType::FlagArtificial); + return createTypeWithFlags(VMContext, Ty, DebugNode::FlagArtificial); } -DIType DIBuilder::createObjectPointerType(DIType Ty) { +MDType *DIBuilder::createObjectPointerType(MDType *Ty) { // FIXME: Restrict this to the nodes where it's valid. - if (Ty.isObjectPointer()) + if (Ty->isObjectPointer()) return Ty; - unsigned Flags = DIType::FlagObjectPointer | DIType::FlagArtificial; + unsigned Flags = DebugNode::FlagObjectPointer | DebugNode::FlagArtificial; return createTypeWithFlags(VMContext, Ty, Flags); } -void DIBuilder::retainType(DIType T) { AllRetainTypes.emplace_back(T); } - -DIBasicType DIBuilder::createUnspecifiedParameter() { - return DIBasicType(); +void DIBuilder::retainType(MDType *T) { + assert(T && "Expected non-null type"); + AllRetainTypes.emplace_back(T); } -DICompositeType -DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIDescriptor Scope, - DIFile F, unsigned Line, unsigned RuntimeLang, +MDBasicType *DIBuilder::createUnspecifiedParameter() { return nullptr; } + +MDCompositeType* +DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, MDScope * Scope, + MDFile* F, unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, StringRef UniqueIdentifier) { // FIXME: Define in terms of createReplaceableForwardDecl() by calling // replaceWithUniqued(). - DICompositeType RetTy = MDCompositeType::get( - VMContext, Tag, Name, F.getFileNode(), Line, - DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, SizeInBits, - AlignInBits, 0, DIDescriptor::FlagFwdDecl, nullptr, RuntimeLang, nullptr, + auto *RetTy = MDCompositeType::get( + VMContext, Tag, Name, F, Line, + MDScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits, + AlignInBits, 0, DebugNode::FlagFwdDecl, nullptr, RuntimeLang, nullptr, nullptr, UniqueIdentifier); if (!UniqueIdentifier.empty()) retainType(RetTy); @@ -533,16 +513,15 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIDescriptor Scope, return RetTy; } -DICompositeType DIBuilder::createReplaceableCompositeType( - unsigned Tag, StringRef Name, DIDescriptor Scope, DIFile F, unsigned Line, +MDCompositeType* DIBuilder::createReplaceableCompositeType( + unsigned Tag, StringRef Name, MDScope * Scope, MDFile* F, unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, StringRef UniqueIdentifier) { - DICompositeType RetTy = - MDCompositeType::getTemporary( - VMContext, Tag, Name, F.getFileNode(), Line, - DIScope(getNonCompileUnitScope(Scope)).getRef(), nullptr, SizeInBits, - AlignInBits, 0, Flags, nullptr, RuntimeLang, - nullptr, nullptr, UniqueIdentifier).release(); + auto *RetTy = MDCompositeType::getTemporary( + VMContext, Tag, Name, F, Line, + MDScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, + SizeInBits, AlignInBits, 0, Flags, nullptr, RuntimeLang, + nullptr, nullptr, UniqueIdentifier).release(); if (!UniqueIdentifier.empty()) retainType(RetTy); trackIfUnresolved(RetTy); @@ -550,102 +529,102 @@ DICompositeType DIBuilder::createReplaceableCompositeType( } DIArray DIBuilder::getOrCreateArray(ArrayRef Elements) { - return DIArray(MDNode::get(VMContext, Elements)); + return MDTuple::get(VMContext, Elements); } DITypeArray DIBuilder::getOrCreateTypeArray(ArrayRef Elements) { SmallVector Elts; for (unsigned i = 0, e = Elements.size(); i != e; ++i) { if (Elements[i] && isa(Elements[i])) - Elts.push_back(DIType(cast(Elements[i])).getRef()); + Elts.push_back(MDTypeRef::get(cast(Elements[i]))); else Elts.push_back(Elements[i]); } return DITypeArray(MDNode::get(VMContext, Elts)); } -DISubrange DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Count) { +MDSubrange *DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Count) { return MDSubrange::get(VMContext, Count, Lo); } -static void checkGlobalVariableScope(DIDescriptor Context) { - MDNode *TheCtx = getNonCompileUnitScope(Context); - if (DIScope(TheCtx).isCompositeType()) { - assert(!DICompositeType(TheCtx).getIdentifier() && +static void checkGlobalVariableScope(MDScope * Context) { +#ifndef NDEBUG + if (auto *CT = + dyn_cast_or_null(getNonCompileUnitScope(Context))) + assert(CT->getIdentifier().empty() && "Context of a global variable should not be a type with identifier"); - } +#endif } -DIGlobalVariable DIBuilder::createGlobalVariable( - DIDescriptor Context, StringRef Name, StringRef LinkageName, DIFile F, - unsigned LineNumber, DITypeRef Ty, bool isLocalToUnit, Constant *Val, +MDGlobalVariable *DIBuilder::createGlobalVariable( + MDScope *Context, StringRef Name, StringRef LinkageName, MDFile *F, + unsigned LineNumber, MDType *Ty, bool isLocalToUnit, Constant *Val, MDNode *Decl) { checkGlobalVariableScope(Context); - auto *N = MDGlobalVariable::get(VMContext, Context, Name, LinkageName, F, - LineNumber, Ty, isLocalToUnit, true, - getConstantOrNull(Val), Decl); + auto *N = MDGlobalVariable::get(VMContext, cast_or_null(Context), + Name, LinkageName, F, LineNumber, + MDTypeRef::get(Ty), isLocalToUnit, true, Val, + cast_or_null(Decl)); AllGVs.push_back(N); return N; } -DIGlobalVariable DIBuilder::createTempGlobalVariableFwdDecl( - DIDescriptor Context, StringRef Name, StringRef LinkageName, DIFile F, - unsigned LineNumber, DITypeRef Ty, bool isLocalToUnit, Constant *Val, +MDGlobalVariable *DIBuilder::createTempGlobalVariableFwdDecl( + MDScope *Context, StringRef Name, StringRef LinkageName, MDFile *F, + unsigned LineNumber, MDType *Ty, bool isLocalToUnit, Constant *Val, MDNode *Decl) { checkGlobalVariableScope(Context); - return MDGlobalVariable::getTemporary(VMContext, Context, Name, LinkageName, - F, LineNumber, Ty, isLocalToUnit, false, - getConstantOrNull(Val), Decl).release(); + return MDGlobalVariable::getTemporary( + VMContext, cast_or_null(Context), Name, LinkageName, F, + LineNumber, MDTypeRef::get(Ty), isLocalToUnit, false, Val, + cast_or_null(Decl)) + .release(); } -DIVariable DIBuilder::createLocalVariable(unsigned Tag, DIDescriptor Scope, - StringRef Name, DIFile File, - unsigned LineNo, DITypeRef Ty, - bool AlwaysPreserve, unsigned Flags, - unsigned ArgNo) { +MDLocalVariable *DIBuilder::createLocalVariable( + unsigned Tag, MDScope *Scope, StringRef Name, MDFile *File, unsigned LineNo, + MDType *Ty, bool AlwaysPreserve, unsigned Flags, unsigned ArgNo) { // FIXME: Why getNonCompileUnitScope()? // FIXME: Why is "!Context" okay here? // FIXME: WHy doesn't this check for a subprogram or lexical block (AFAICT // the only valid scopes)? - DIDescriptor Context(getNonCompileUnitScope(Scope)); - assert((!Context || Context.isScope()) && - "createLocalVariable should be called with a valid Context"); + MDScope* Context = getNonCompileUnitScope(Scope); - auto *Node = - MDLocalVariable::get(VMContext, Tag, getNonCompileUnitScope(Scope), Name, - File, LineNo, Ty, ArgNo, Flags); + auto *Node = MDLocalVariable::get( + VMContext, Tag, cast_or_null(Context), Name, File, LineNo, + MDTypeRef::get(Ty), ArgNo, Flags); if (AlwaysPreserve) { // The optimizer may remove local variable. If there is an interest // to preserve variable info in such situation then stash it in a // named mdnode. - DISubprogram Fn(getDISubprogram(Scope)); + MDSubprogram *Fn = getDISubprogram(Scope); assert(Fn && "Missing subprogram for local variable"); PreservedVariables[Fn].emplace_back(Node); } return Node; } -DIExpression DIBuilder::createExpression(ArrayRef Addr) { +MDExpression* DIBuilder::createExpression(ArrayRef Addr) { return MDExpression::get(VMContext, Addr); } -DIExpression DIBuilder::createExpression(ArrayRef Signed) { +MDExpression* DIBuilder::createExpression(ArrayRef Signed) { // TODO: Remove the callers of this signed version and delete. SmallVector Addr(Signed.begin(), Signed.end()); return createExpression(Addr); } -DIExpression DIBuilder::createBitPieceExpression(unsigned OffsetInBytes, +MDExpression* DIBuilder::createBitPieceExpression(unsigned OffsetInBytes, unsigned SizeInBytes) { uint64_t Addr[] = {dwarf::DW_OP_bit_piece, OffsetInBytes, SizeInBytes}; return MDExpression::get(VMContext, Addr); } -DISubprogram DIBuilder::createFunction(DIScopeRef Context, StringRef Name, - StringRef LinkageName, DIFile File, - unsigned LineNo, DICompositeType Ty, +MDSubprogram* DIBuilder::createFunction(DIScopeRef Context, StringRef Name, + StringRef LinkageName, MDFile* File, + unsigned LineNo, MDSubroutineType* Ty, bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, Function *Fn, @@ -658,20 +637,21 @@ DISubprogram DIBuilder::createFunction(DIScopeRef Context, StringRef Name, Flags, isOptimized, Fn, TParams, Decl); } -DISubprogram DIBuilder::createFunction(DIDescriptor Context, StringRef Name, - StringRef LinkageName, DIFile File, - unsigned LineNo, DICompositeType Ty, +MDSubprogram* DIBuilder::createFunction(MDScope * Context, StringRef Name, + StringRef LinkageName, MDFile* File, + unsigned LineNo, MDSubroutineType* Ty, bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, Function *Fn, MDNode *TParams, MDNode *Decl) { - assert(Ty.getTag() == dwarf::DW_TAG_subroutine_type && + assert(Ty->getTag() == dwarf::DW_TAG_subroutine_type && "function types should be subroutines"); auto *Node = MDSubprogram::get( - VMContext, DIScope(getNonCompileUnitScope(Context)).getRef(), Name, - LinkageName, File.getFileNode(), LineNo, Ty, isLocalToUnit, isDefinition, - ScopeLine, nullptr, 0, 0, Flags, isOptimized, getConstantOrNull(Fn), - TParams, Decl, MDNode::getTemporary(VMContext, None).release()); + VMContext, MDScopeRef::get(getNonCompileUnitScope(Context)), Name, + LinkageName, File, LineNo, Ty, + isLocalToUnit, isDefinition, ScopeLine, nullptr, 0, 0, Flags, isOptimized, + Fn, cast_or_null(TParams), cast_or_null(Decl), + MDTuple::getTemporary(VMContext, None).release()); if (isDefinition) AllSubprograms.push_back(Node); @@ -679,78 +659,64 @@ DISubprogram DIBuilder::createFunction(DIDescriptor Context, StringRef Name, return Node; } -DISubprogram -DIBuilder::createTempFunctionFwdDecl(DIDescriptor Context, StringRef Name, - StringRef LinkageName, DIFile File, - unsigned LineNo, DICompositeType Ty, +MDSubprogram* +DIBuilder::createTempFunctionFwdDecl(MDScope * Context, StringRef Name, + StringRef LinkageName, MDFile* File, + unsigned LineNo, MDSubroutineType* Ty, bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, Function *Fn, MDNode *TParams, MDNode *Decl) { return MDSubprogram::getTemporary( - VMContext, DIScope(getNonCompileUnitScope(Context)).getRef(), Name, - LinkageName, File.getFileNode(), LineNo, Ty, isLocalToUnit, - isDefinition, ScopeLine, nullptr, 0, 0, Flags, isOptimized, - getConstantOrNull(Fn), TParams, Decl, nullptr).release(); -} - -DISubprogram DIBuilder::createMethod(DIDescriptor Context, StringRef Name, - StringRef LinkageName, DIFile F, - unsigned LineNo, DICompositeType Ty, - bool isLocalToUnit, bool isDefinition, - unsigned VK, unsigned VIndex, - DIType VTableHolder, unsigned Flags, - bool isOptimized, Function *Fn, - MDNode *TParam) { - assert(Ty.getTag() == dwarf::DW_TAG_subroutine_type && + VMContext, MDScopeRef::get(getNonCompileUnitScope(Context)), Name, + LinkageName, File, LineNo, Ty, + isLocalToUnit, isDefinition, ScopeLine, nullptr, 0, 0, Flags, + isOptimized, Fn, cast_or_null(TParams), + cast_or_null(Decl), nullptr).release(); +} + +MDSubprogram * +DIBuilder::createMethod(MDScope *Context, StringRef Name, StringRef LinkageName, + MDFile *F, unsigned LineNo, MDSubroutineType *Ty, + bool isLocalToUnit, bool isDefinition, unsigned VK, + unsigned VIndex, MDType *VTableHolder, unsigned Flags, + bool isOptimized, Function *Fn, MDNode *TParam) { + assert(Ty->getTag() == dwarf::DW_TAG_subroutine_type && "function types should be subroutines"); assert(getNonCompileUnitScope(Context) && "Methods should have both a Context and a context that isn't " "the compile unit."); // FIXME: Do we want to use different scope/lines? - auto *Node = MDSubprogram::get( - VMContext, DIScope(Context).getRef(), Name, LinkageName, F.getFileNode(), - LineNo, Ty, isLocalToUnit, isDefinition, LineNo, VTableHolder.getRef(), - VK, VIndex, Flags, isOptimized, getConstantOrNull(Fn), TParam, nullptr, - nullptr); + auto *SP = MDSubprogram::get( + VMContext, MDScopeRef::get(cast(Context)), Name, LinkageName, F, + LineNo, Ty, isLocalToUnit, isDefinition, LineNo, + MDTypeRef::get(VTableHolder), VK, VIndex, Flags, isOptimized, Fn, + cast_or_null(TParam), nullptr, nullptr); if (isDefinition) - AllSubprograms.push_back(Node); - DISubprogram S(Node); - assert(S.isSubprogram() && "createMethod should return a valid DISubprogram"); - trackIfUnresolved(S); - return S; -} - -DINameSpace DIBuilder::createNameSpace(DIDescriptor Scope, StringRef Name, - DIFile File, unsigned LineNo) { - DINameSpace R = MDNamespace::get(VMContext, getNonCompileUnitScope(Scope), - File.getFileNode(), Name, LineNo); - assert(R.Verify() && - "createNameSpace should return a verifiable DINameSpace"); - return R; + AllSubprograms.push_back(SP); + trackIfUnresolved(SP); + return SP; +} + +MDNamespace* DIBuilder::createNameSpace(MDScope * Scope, StringRef Name, + MDFile* File, unsigned LineNo) { + return MDNamespace::get(VMContext, getNonCompileUnitScope(Scope), File, Name, + LineNo); } -DILexicalBlockFile DIBuilder::createLexicalBlockFile(DIDescriptor Scope, - DIFile File, +MDLexicalBlockFile* DIBuilder::createLexicalBlockFile(MDScope * Scope, + MDFile* File, unsigned Discriminator) { - DILexicalBlockFile R = MDLexicalBlockFile::get( - VMContext, Scope, File.getFileNode(), Discriminator); - assert( - R.Verify() && - "createLexicalBlockFile should return a verifiable DILexicalBlockFile"); - return R; + return MDLexicalBlockFile::get(VMContext, Scope, File, Discriminator); } -DILexicalBlock DIBuilder::createLexicalBlock(DIDescriptor Scope, DIFile File, +MDLexicalBlock* DIBuilder::createLexicalBlock(MDScope * Scope, MDFile* File, unsigned Line, unsigned Col) { // Make these distinct, to avoid merging two lexical blocks on the same // file/line/column. - DILexicalBlock R = MDLexicalBlock::getDistinct( - VMContext, getNonCompileUnitScope(Scope), File.getFileNode(), Line, Col); - assert(R.Verify() && - "createLexicalBlock should return a verifiable DILexicalBlock"); - return R; + return MDLexicalBlock::getDistinct(VMContext, getNonCompileUnitScope(Scope), + File, Line, Col); } static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) { @@ -758,11 +724,19 @@ static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) { return MetadataAsValue::get(VMContext, ValueAsMetadata::get(V)); } -Instruction *DIBuilder::insertDeclare(Value *Storage, DIVariable VarInfo, - DIExpression Expr, +static Instruction *withDebugLoc(Instruction *I, const MDLocation *DL) { + I->setDebugLoc(const_cast(DL)); + return I; +} + +Instruction *DIBuilder::insertDeclare(Value *Storage, MDLocalVariable* VarInfo, + MDExpression* Expr, const MDLocation *DL, Instruction *InsertBefore) { - assert(VarInfo.isVariable() && - "empty or invalid DIVariable passed to dbg.declare"); + assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.declare"); + assert(DL && "Expected debug loc"); + assert(DL->getScope()->getSubprogram() == + VarInfo->getScope()->getSubprogram() && + "Expected matching subprograms"); if (!DeclareFn) DeclareFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_declare); @@ -771,14 +745,17 @@ Instruction *DIBuilder::insertDeclare(Value *Storage, DIVariable VarInfo, Value *Args[] = {getDbgIntrinsicValueImpl(VMContext, Storage), MetadataAsValue::get(VMContext, VarInfo), MetadataAsValue::get(VMContext, Expr)}; - return CallInst::Create(DeclareFn, Args, "", InsertBefore); + return withDebugLoc(CallInst::Create(DeclareFn, Args, "", InsertBefore), DL); } -Instruction *DIBuilder::insertDeclare(Value *Storage, DIVariable VarInfo, - DIExpression Expr, +Instruction *DIBuilder::insertDeclare(Value *Storage, MDLocalVariable* VarInfo, + MDExpression* Expr, const MDLocation *DL, BasicBlock *InsertAtEnd) { - assert(VarInfo.isVariable() && - "empty or invalid DIVariable passed to dbg.declare"); + assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.declare"); + assert(DL && "Expected debug loc"); + assert(DL->getScope()->getSubprogram() == + VarInfo->getScope()->getSubprogram() && + "Expected matching subprograms"); if (!DeclareFn) DeclareFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_declare); @@ -791,18 +768,21 @@ Instruction *DIBuilder::insertDeclare(Value *Storage, DIVariable VarInfo, // If this block already has a terminator then insert this intrinsic // before the terminator. if (TerminatorInst *T = InsertAtEnd->getTerminator()) - return CallInst::Create(DeclareFn, Args, "", T); - else - return CallInst::Create(DeclareFn, Args, "", InsertAtEnd); + return withDebugLoc(CallInst::Create(DeclareFn, Args, "", T), DL); + return withDebugLoc(CallInst::Create(DeclareFn, Args, "", InsertAtEnd), DL); } Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset, - DIVariable VarInfo, - DIExpression Expr, + MDLocalVariable* VarInfo, + MDExpression* Expr, + const MDLocation *DL, Instruction *InsertBefore) { assert(V && "no value passed to dbg.value"); - assert(VarInfo.isVariable() && - "empty or invalid DIVariable passed to dbg.value"); + assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.value"); + assert(DL && "Expected debug loc"); + assert(DL->getScope()->getSubprogram() == + VarInfo->getScope()->getSubprogram() && + "Expected matching subprograms"); if (!ValueFn) ValueFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_value); @@ -812,16 +792,20 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset, ConstantInt::get(Type::getInt64Ty(VMContext), Offset), MetadataAsValue::get(VMContext, VarInfo), MetadataAsValue::get(VMContext, Expr)}; - return CallInst::Create(ValueFn, Args, "", InsertBefore); + return withDebugLoc(CallInst::Create(ValueFn, Args, "", InsertBefore), DL); } Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset, - DIVariable VarInfo, - DIExpression Expr, + MDLocalVariable* VarInfo, + MDExpression* Expr, + const MDLocation *DL, BasicBlock *InsertAtEnd) { assert(V && "no value passed to dbg.value"); - assert(VarInfo.isVariable() && - "empty or invalid DIVariable passed to dbg.value"); + assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.value"); + assert(DL && "Expected debug loc"); + assert(DL->getScope()->getSubprogram() == + VarInfo->getScope()->getSubprogram() && + "Expected matching subprograms"); if (!ValueFn) ValueFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_value); @@ -831,11 +815,16 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset, ConstantInt::get(Type::getInt64Ty(VMContext), Offset), MetadataAsValue::get(VMContext, VarInfo), MetadataAsValue::get(VMContext, Expr)}; - return CallInst::Create(ValueFn, Args, "", InsertAtEnd); + + return withDebugLoc(CallInst::Create(ValueFn, Args, "", InsertAtEnd), DL); } -void DIBuilder::replaceVTableHolder(DICompositeType &T, DICompositeType VTableHolder) { - T.setContainingType(VTableHolder); +void DIBuilder::replaceVTableHolder(MDCompositeType* &T, MDCompositeType* VTableHolder) { + { + TypedTrackingMDRef N(T); + N->replaceVTableHolder(MDTypeRef::get(VTableHolder)); + T = N.get(); + } // If this didn't create a self-reference, just return. if (T != VTableHolder) @@ -849,9 +838,16 @@ void DIBuilder::replaceVTableHolder(DICompositeType &T, DICompositeType VTableHo trackIfUnresolved(N); } -void DIBuilder::replaceArrays(DICompositeType &T, DIArray Elements, +void DIBuilder::replaceArrays(MDCompositeType* &T, DIArray Elements, DIArray TParams) { - T.setArrays(Elements, TParams); + { + TypedTrackingMDRef N(T); + if (Elements) + N->replaceElements(Elements); + if (TParams) + N->replaceTemplateParams(MDTemplateParameterArray(TParams)); + T = N.get(); + } // If T isn't resolved, there's no problem. if (!T->isResolved()) @@ -861,7 +857,7 @@ void DIBuilder::replaceArrays(DICompositeType &T, DIArray Elements, // arrays explicitly if they're unresolved, or else the cycles will be // orphaned. if (Elements) - trackIfUnresolved(Elements); + trackIfUnresolved(Elements.get()); if (TParams) - trackIfUnresolved(TParams); + trackIfUnresolved(TParams.get()); } diff --git a/lib/IR/DebugInfo.cpp b/lib/IR/DebugInfo.cpp index 9a6b953..719c28b 100644 --- a/lib/IR/DebugInfo.cpp +++ b/lib/IR/DebugInfo.cpp @@ -17,7 +17,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" @@ -25,6 +24,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/Module.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Debug.h" @@ -33,612 +33,62 @@ using namespace llvm; using namespace llvm::dwarf; -//===----------------------------------------------------------------------===// -// DIDescriptor -//===----------------------------------------------------------------------===// - -unsigned DIDescriptor::getFlag(StringRef Flag) { - return StringSwitch(Flag) -#define HANDLE_DI_FLAG(ID, NAME) .Case("DIFlag" #NAME, Flag##NAME) -#include "llvm/IR/DebugInfoFlags.def" - .Default(0); -} - -const char *DIDescriptor::getFlagString(unsigned Flag) { - switch (Flag) { - default: - return ""; -#define HANDLE_DI_FLAG(ID, NAME) \ - case Flag##NAME: \ - return "DIFlag" #NAME; -#include "llvm/IR/DebugInfoFlags.def" - } -} - -unsigned DIDescriptor::splitFlags(unsigned Flags, - SmallVectorImpl &SplitFlags) { - // Accessibility flags need to be specially handled, since they're packed - // together. - if (unsigned A = Flags & FlagAccessibility) { - if (A == FlagPrivate) - SplitFlags.push_back(FlagPrivate); - else if (A == FlagProtected) - SplitFlags.push_back(FlagProtected); - else - SplitFlags.push_back(FlagPublic); - Flags &= ~A; - } - -#define HANDLE_DI_FLAG(ID, NAME) \ - if (unsigned Bit = Flags & ID) { \ - SplitFlags.push_back(Bit); \ - Flags &= ~Bit; \ - } -#include "llvm/IR/DebugInfoFlags.def" - - return Flags; -} - -bool DIDescriptor::Verify() const { - return DbgNode && - (DIDerivedType(DbgNode).Verify() || - DICompositeType(DbgNode).Verify() || DIBasicType(DbgNode).Verify() || - DIVariable(DbgNode).Verify() || DISubprogram(DbgNode).Verify() || - DIGlobalVariable(DbgNode).Verify() || DIFile(DbgNode).Verify() || - DICompileUnit(DbgNode).Verify() || DINameSpace(DbgNode).Verify() || - DILexicalBlock(DbgNode).Verify() || - DILexicalBlockFile(DbgNode).Verify() || - DISubrange(DbgNode).Verify() || DIEnumerator(DbgNode).Verify() || - DIObjCProperty(DbgNode).Verify() || - DITemplateTypeParameter(DbgNode).Verify() || - DITemplateValueParameter(DbgNode).Verify() || - DIImportedEntity(DbgNode).Verify()); -} - -static Metadata *getField(const MDNode *DbgNode, unsigned Elt) { - if (!DbgNode || Elt >= DbgNode->getNumOperands()) - return nullptr; - return DbgNode->getOperand(Elt); -} - -static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) { - return dyn_cast_or_null(getField(DbgNode, Elt)); -} - -static StringRef getStringField(const MDNode *DbgNode, unsigned Elt) { - if (MDString *MDS = dyn_cast_or_null(getField(DbgNode, Elt))) - return MDS->getString(); - return StringRef(); -} - -StringRef DIDescriptor::getStringField(unsigned Elt) const { - return ::getStringField(DbgNode, Elt); -} - -uint64_t DIDescriptor::getUInt64Field(unsigned Elt) const { - if (auto *C = getConstantField(Elt)) - if (ConstantInt *CI = dyn_cast(C)) - return CI->getZExtValue(); - - return 0; -} - -int64_t DIDescriptor::getInt64Field(unsigned Elt) const { - if (auto *C = getConstantField(Elt)) - if (ConstantInt *CI = dyn_cast(C)) - return CI->getZExtValue(); - - return 0; -} - -DIDescriptor DIDescriptor::getDescriptorField(unsigned Elt) const { - MDNode *Field = getNodeField(DbgNode, Elt); - return DIDescriptor(Field); -} - -GlobalVariable *DIDescriptor::getGlobalVariableField(unsigned Elt) const { - return dyn_cast_or_null(getConstantField(Elt)); -} - -Constant *DIDescriptor::getConstantField(unsigned Elt) const { - if (!DbgNode) - return nullptr; - - if (Elt < DbgNode->getNumOperands()) - if (auto *C = - dyn_cast_or_null(DbgNode->getOperand(Elt))) - return C->getValue(); - return nullptr; -} - -Function *DIDescriptor::getFunctionField(unsigned Elt) const { - return dyn_cast_or_null(getConstantField(Elt)); -} - -/// \brief Return the size reported by the variable's type. -unsigned DIVariable::getSizeInBits(const DITypeIdentifierMap &Map) { - DIType Ty = getType().resolve(Map); - // Follow derived types until we reach a type that - // reports back a size. - while (Ty.isDerivedType() && !Ty.getSizeInBits()) { - DIDerivedType DT(&*Ty); - Ty = DT.getTypeDerivedFrom().resolve(Map); - } - assert(Ty.getSizeInBits() && "type with size 0"); - return Ty.getSizeInBits(); -} - -bool DIExpression::isBitPiece() const { - unsigned N = getNumElements(); - return N >=3 && getElement(N-3) == dwarf::DW_OP_bit_piece; -} - -uint64_t DIExpression::getBitPieceOffset() const { - assert(isBitPiece() && "not a piece"); - return getElement(getNumElements()-2); -} - -uint64_t DIExpression::getBitPieceSize() const { - assert(isBitPiece() && "not a piece"); - return getElement(getNumElements()-1); -} - -DIExpression::iterator DIExpression::Operand::getNext() const { - iterator it(I); - return ++it; -} - -//===----------------------------------------------------------------------===// -// Simple Descriptor Constructors and other Methods -//===----------------------------------------------------------------------===// - -void DIDescriptor::replaceAllUsesWith(LLVMContext &, DIDescriptor D) { - assert(DbgNode && "Trying to replace an unverified type!"); - assert(DbgNode->isTemporary() && "Expected temporary node"); - TempMDNode Temp(get()); - - // Since we use a TrackingVH for the node, its easy for clients to manufacture - // legitimate situations where they want to replaceAllUsesWith() on something - // which, due to uniquing, has merged with the source. We shield clients from - // this detail by allowing a value to be replaced with replaceAllUsesWith() - // itself. - if (Temp.get() == D.get()) { - DbgNode = MDNode::replaceWithUniqued(std::move(Temp)); - return; - } - - Temp->replaceAllUsesWith(D.get()); - DbgNode = D.get(); -} - -void DIDescriptor::replaceAllUsesWith(MDNode *D) { - assert(DbgNode && "Trying to replace an unverified type!"); - assert(DbgNode != D && "This replacement should always happen"); - assert(DbgNode->isTemporary() && "Expected temporary node"); - TempMDNode Node(get()); - Node->replaceAllUsesWith(D); -} - -bool DICompileUnit::Verify() const { - if (!isCompileUnit()) - return false; - - // Don't bother verifying the compilation directory or producer string - // as those could be empty. - return !getFilename().empty(); -} - -bool DIObjCProperty::Verify() const { return isObjCProperty(); } - -/// \brief Check if a value can be a reference to a type. -static bool isTypeRef(const Metadata *MD) { - if (!MD) - return true; - if (auto *S = dyn_cast(MD)) - return !S->getString().empty(); - return isa(MD); -} - -/// \brief Check if a value can be a ScopeRef. -static bool isScopeRef(const Metadata *MD) { - if (!MD) - return true; - if (auto *S = dyn_cast(MD)) - return !S->getString().empty(); - return isa(MD); -} - -#ifndef NDEBUG -/// \brief Check if a value can be a DescriptorRef. -static bool isDescriptorRef(const Metadata *MD) { - if (!MD) - return true; - if (auto *S = dyn_cast(MD)) - return !S->getString().empty(); - return isa(MD); -} -#endif - -bool DIType::Verify() const { - auto *N = dyn_cast_or_null(DbgNode); - if (!N) - return false; - if (!isScopeRef(N->getScope())) - return false; - - // DIType is abstract, it should be a BasicType, a DerivedType or - // a CompositeType. - if (isBasicType()) - return DIBasicType(DbgNode).Verify(); - - // FIXME: Sink this into the various subclass verifies. - if (getFilename().empty()) { - // Check whether the filename is allowed to be empty. - uint16_t Tag = getTag(); - if (Tag != dwarf::DW_TAG_const_type && Tag != dwarf::DW_TAG_volatile_type && - Tag != dwarf::DW_TAG_pointer_type && - Tag != dwarf::DW_TAG_ptr_to_member_type && - Tag != dwarf::DW_TAG_reference_type && - Tag != dwarf::DW_TAG_rvalue_reference_type && - Tag != dwarf::DW_TAG_restrict_type && Tag != dwarf::DW_TAG_array_type && - Tag != dwarf::DW_TAG_enumeration_type && - Tag != dwarf::DW_TAG_subroutine_type && - Tag != dwarf::DW_TAG_inheritance && Tag != dwarf::DW_TAG_friend && - Tag != dwarf::DW_TAG_structure_type && Tag != dwarf::DW_TAG_member && - Tag != dwarf::DW_TAG_typedef) - return false; - } - - if (isCompositeType()) - return DICompositeType(DbgNode).Verify(); - if (isDerivedType()) - return DIDerivedType(DbgNode).Verify(); - return false; -} - -bool DIBasicType::Verify() const { - return dyn_cast_or_null(DbgNode); -} - -bool DIDerivedType::Verify() const { - auto *N = dyn_cast_or_null(DbgNode); - if (!N) - return false; - if (getTag() == dwarf::DW_TAG_ptr_to_member_type) { - auto *D = dyn_cast(N); - if (!D) - return false; - if (!isTypeRef(D->getExtraData())) - return false; - } - return isTypeRef(N->getBaseType()); -} - -bool DICompositeType::Verify() const { - auto *N = dyn_cast_or_null(DbgNode); - return N && isTypeRef(N->getBaseType()) && isTypeRef(N->getVTableHolder()) && - !(isLValueReference() && isRValueReference()); -} - -bool DISubprogram::Verify() const { - auto *N = dyn_cast_or_null(DbgNode); - if (!N) - return false; - - if (!isScopeRef(N->getScope())) - return false; - - if (auto *Op = N->getType()) - if (!isa(Op)) - return false; - - if (!isTypeRef(getContainingType())) - return false; - - if (isLValueReference() && isRValueReference()) - return false; - - // If a DISubprogram has an llvm::Function*, then scope chains from all - // instructions within the function should lead to this DISubprogram. - if (auto *F = getFunction()) { - for (auto &BB : *F) { - for (auto &I : BB) { - DebugLoc DL = I.getDebugLoc(); - if (DL.isUnknown()) - continue; - - MDNode *Scope = nullptr; - MDNode *IA = nullptr; - // walk the inlined-at scopes - while ((IA = DL.getInlinedAt())) - DL = DebugLoc::getFromDILocation(IA); - DL.getScopeAndInlinedAt(Scope, IA); - if (!Scope) - return false; - assert(!IA); - while (!DIDescriptor(Scope).isSubprogram()) { - DILexicalBlockFile D(Scope); - Scope = D.isLexicalBlockFile() - ? D.getScope() - : DebugLoc::getFromDILexicalBlock(Scope).getScope(); - if (!Scope) - return false; - } - if (!DISubprogram(Scope).describes(F)) - return false; - } - } - } - - return true; -} - -bool DIGlobalVariable::Verify() const { - auto *N = dyn_cast_or_null(DbgNode); - - if (!N) - return false; - - if (N->getDisplayName().empty()) - return false; - - if (auto *Op = N->getScope()) - if (!isa(Op)) - return false; - - if (auto *Op = N->getStaticDataMemberDeclaration()) - if (!isa(Op)) - return false; - - return isTypeRef(N->getType()); -} - -bool DIVariable::Verify() const { - auto *N = dyn_cast_or_null(DbgNode); - - if (!N) - return false; - - if (auto *Op = N->getScope()) - if (!isa(Op)) - return false; - - return isTypeRef(N->getType()); -} - -bool DILocation::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DINameSpace::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DIFile::Verify() const { return dyn_cast_or_null(DbgNode); } -bool DIEnumerator::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DISubrange::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DILexicalBlock::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DILexicalBlockFile::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DITemplateTypeParameter::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DITemplateValueParameter::Verify() const { - return dyn_cast_or_null(DbgNode); -} -bool DIImportedEntity::Verify() const { - return dyn_cast_or_null(DbgNode); -} - -void DICompositeType::setArraysHelper(MDNode *Elements, MDNode *TParams) { - TypedTrackingMDRef N(get()); - if (Elements) - N->replaceElements(cast(Elements)); - if (TParams) - N->replaceTemplateParams(cast(TParams)); - DbgNode = N; -} - -DIScopeRef DIScope::getRef() const { - if (!isCompositeType()) - return DIScopeRef(*this); - DICompositeType DTy(DbgNode); - if (!DTy.getIdentifier()) - return DIScopeRef(*this); - return DIScopeRef(DTy.getIdentifier()); -} - -void DICompositeType::setContainingType(DICompositeType ContainingType) { - TypedTrackingMDRef N(get()); - N->replaceVTableHolder(ContainingType.getRef()); - DbgNode = N; -} - -bool DIVariable::isInlinedFnArgument(const Function *CurFn) { - assert(CurFn && "Invalid function"); - if (!getContext().isSubprogram()) - return false; - // This variable is not inlined function argument if its scope - // does not describe current function. - return !DISubprogram(getContext()).describes(CurFn); -} - -Function *DISubprogram::getFunction() const { - if (auto *N = get()) - if (auto *C = dyn_cast_or_null(N->getFunction())) - return dyn_cast(C->getValue()); - return nullptr; -} - -bool DISubprogram::describes(const Function *F) { - assert(F && "Invalid function"); - if (F == getFunction()) - return true; - StringRef Name = getLinkageName(); - if (Name.empty()) - Name = getName(); - if (F->getName() == Name) - return true; - return false; -} - -GlobalVariable *DIGlobalVariable::getGlobal() const { - return dyn_cast_or_null(getConstant()); -} - -DIScopeRef DIScope::getContext() const { - - if (isType()) - return DIType(DbgNode).getContext(); - - if (isSubprogram()) - return DIScopeRef(DISubprogram(DbgNode).getContext()); - - if (isLexicalBlock()) - return DIScopeRef(DILexicalBlock(DbgNode).getContext()); - - if (isLexicalBlockFile()) - return DIScopeRef(DILexicalBlockFile(DbgNode).getContext()); - - if (isNameSpace()) - return DIScopeRef(DINameSpace(DbgNode).getContext()); - - assert((isFile() || isCompileUnit()) && "Unhandled type of scope."); - return DIScopeRef(nullptr); -} - -StringRef DIScope::getName() const { - if (isType()) - return DIType(DbgNode).getName(); - if (isSubprogram()) - return DISubprogram(DbgNode).getName(); - if (isNameSpace()) - return DINameSpace(DbgNode).getName(); - assert((isLexicalBlock() || isLexicalBlockFile() || isFile() || - isCompileUnit()) && - "Unhandled type of scope."); - return StringRef(); -} - -StringRef DIScope::getFilename() const { - if (auto *N = get()) - return ::getStringField(dyn_cast_or_null(N->getFile()), 0); - return ""; -} - -StringRef DIScope::getDirectory() const { - if (auto *N = get()) - return ::getStringField(dyn_cast_or_null(N->getFile()), 1); - return ""; -} - -void DICompileUnit::replaceSubprograms(DIArray Subprograms) { - assert(Verify() && "Expected compile unit"); - get()->replaceSubprograms(cast_or_null(Subprograms.get())); -} - -void DICompileUnit::replaceGlobalVariables(DIArray GlobalVariables) { - assert(Verify() && "Expected compile unit"); - get()->replaceGlobalVariables(cast_or_null(GlobalVariables.get())); -} - -DILocation DILocation::copyWithNewScope(LLVMContext &Ctx, - DILexicalBlockFile NewScope) { - assert(Verify()); - assert(NewScope && "Expected valid scope"); - - const auto *Old = cast(DbgNode); - return DILocation(MDLocation::get(Ctx, Old->getLine(), Old->getColumn(), - NewScope, Old->getInlinedAt())); -} - -unsigned DILocation::computeNewDiscriminator(LLVMContext &Ctx) { - std::pair Key(getFilename().data(), getLineNumber()); - return ++Ctx.pImpl->DiscriminatorTable[Key]; -} - -DIVariable llvm::createInlinedVariable(MDNode *DV, MDNode *InlinedScope, - LLVMContext &VMContext) { - assert(DIVariable(DV).Verify() && "Expected a DIVariable"); - return cast(DV) - ->withInline(cast_or_null(InlinedScope)); -} - -DIVariable llvm::cleanseInlinedVariable(MDNode *DV, LLVMContext &VMContext) { - assert(DIVariable(DV).Verify() && "Expected a DIVariable"); - return cast(DV)->withoutInline(); -} - DISubprogram llvm::getDISubprogram(const MDNode *Scope) { - DIDescriptor D(Scope); - if (D.isSubprogram()) - return DISubprogram(Scope); - - if (D.isLexicalBlockFile()) - return getDISubprogram(DILexicalBlockFile(Scope).getContext()); - - if (D.isLexicalBlock()) - return getDISubprogram(DILexicalBlock(Scope).getContext()); - - return DISubprogram(); + if (auto *LocalScope = dyn_cast_or_null(Scope)) + return LocalScope->getSubprogram(); + return nullptr; } DISubprogram llvm::getDISubprogram(const Function *F) { // We look for the first instr that has a debug annotation leading back to F. for (auto &BB : *F) { auto Inst = std::find_if(BB.begin(), BB.end(), [](const Instruction &Inst) { - return !Inst.getDebugLoc().isUnknown(); + return Inst.getDebugLoc(); }); if (Inst == BB.end()) continue; DebugLoc DLoc = Inst->getDebugLoc(); - const MDNode *Scope = DLoc.getScopeNode(); + const MDNode *Scope = DLoc.getInlinedAtScope(); DISubprogram Subprogram = getDISubprogram(Scope); - return Subprogram.describes(F) ? Subprogram : DISubprogram(); + return Subprogram->describes(F) ? Subprogram : DISubprogram(); } return DISubprogram(); } DICompositeType llvm::getDICompositeType(DIType T) { - if (T.isCompositeType()) - return DICompositeType(T); + if (auto *C = dyn_cast_or_null(T)) + return C; - if (T.isDerivedType()) { + if (auto *D = dyn_cast_or_null(T)) { // This function is currently used by dragonegg and dragonegg does // not generate identifier for types, so using an empty map to resolve // DerivedFrom should be fine. DITypeIdentifierMap EmptyMap; - return getDICompositeType( - DIDerivedType(T).getTypeDerivedFrom().resolve(EmptyMap)); + return getDICompositeType(D->getBaseType().resolve(EmptyMap)); } - return DICompositeType(); + return nullptr; } DITypeIdentifierMap llvm::generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes) { DITypeIdentifierMap Map; for (unsigned CUi = 0, CUe = CU_Nodes->getNumOperands(); CUi != CUe; ++CUi) { - DICompileUnit CU(CU_Nodes->getOperand(CUi)); - DIArray Retain = CU.getRetainedTypes(); - for (unsigned Ti = 0, Te = Retain.getNumElements(); Ti != Te; ++Ti) { - if (!Retain.getElement(Ti).isCompositeType()) + auto *CU = cast(CU_Nodes->getOperand(CUi)); + DIArray Retain = CU->getRetainedTypes(); + for (unsigned Ti = 0, Te = Retain.size(); Ti != Te; ++Ti) { + if (!isa(Retain[Ti])) continue; - DICompositeType Ty(Retain.getElement(Ti)); - if (MDString *TypeId = Ty.getIdentifier()) { + auto *Ty = cast(Retain[Ti]); + if (MDString *TypeId = Ty->getRawIdentifier()) { // Definition has priority over declaration. // Try to insert (TypeId, Ty) to Map. std::pair P = Map.insert(std::make_pair(TypeId, Ty)); // If TypeId already exists in Map and this is a definition, replace // whatever we had (declaration or definition) with the definition. - if (!P.second && !Ty.isForwardDecl()) + if (!P.second && !Ty->isForwardDecl()) P.first->second = Ty; } } @@ -673,37 +123,28 @@ void DebugInfoFinder::processModule(const Module &M) { InitializeTypeMap(M); if (NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu")) { for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { - DICompileUnit CU(CU_Nodes->getOperand(i)); + DICompileUnit CU = cast(CU_Nodes->getOperand(i)); addCompileUnit(CU); - DIArray GVs = CU.getGlobalVariables(); - for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i) { - DIGlobalVariable DIG(GVs.getElement(i)); + for (DIGlobalVariable DIG : CU->getGlobalVariables()) { if (addGlobalVariable(DIG)) { - processScope(DIG.getContext()); - processType(DIG.getType().resolve(TypeIdentifierMap)); + processScope(DIG->getScope()); + processType(DIG->getType().resolve(TypeIdentifierMap)); } } - DIArray SPs = CU.getSubprograms(); - for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) - processSubprogram(DISubprogram(SPs.getElement(i))); - DIArray EnumTypes = CU.getEnumTypes(); - for (unsigned i = 0, e = EnumTypes.getNumElements(); i != e; ++i) - processType(DIType(EnumTypes.getElement(i))); - DIArray RetainedTypes = CU.getRetainedTypes(); - for (unsigned i = 0, e = RetainedTypes.getNumElements(); i != e; ++i) - processType(DIType(RetainedTypes.getElement(i))); - DIArray Imports = CU.getImportedEntities(); - for (unsigned i = 0, e = Imports.getNumElements(); i != e; ++i) { - DIImportedEntity Import = DIImportedEntity(Imports.getElement(i)); - if (!Import) - continue; - DIDescriptor Entity = Import.getEntity().resolve(TypeIdentifierMap); - if (Entity.isType()) - processType(DIType(Entity)); - else if (Entity.isSubprogram()) - processSubprogram(DISubprogram(Entity)); - else if (Entity.isNameSpace()) - processScope(DINameSpace(Entity).getContext()); + for (auto *SP : CU->getSubprograms()) + processSubprogram(SP); + for (auto *ET : CU->getEnumTypes()) + processType(ET); + for (auto *RT : CU->getRetainedTypes()) + processType(RT); + for (DIImportedEntity Import : CU->getImportedEntities()) { + auto *Entity = Import->getEntity().resolve(TypeIdentifierMap); + if (auto *T = dyn_cast(Entity)) + processType(T); + else if (auto *SP = dyn_cast(Entity)) + processSubprogram(SP); + else if (auto *NS = dyn_cast(Entity)) + processScope(NS->getScope()); } } } @@ -713,79 +154,66 @@ void DebugInfoFinder::processLocation(const Module &M, DILocation Loc) { if (!Loc) return; InitializeTypeMap(M); - processScope(Loc.getScope()); - processLocation(M, Loc.getOrigLocation()); + processScope(Loc->getScope()); + processLocation(M, Loc->getInlinedAt()); } void DebugInfoFinder::processType(DIType DT) { if (!addType(DT)) return; - processScope(DT.getContext().resolve(TypeIdentifierMap)); - if (DT.isCompositeType()) { - DICompositeType DCT(DT); - processType(DCT.getTypeDerivedFrom().resolve(TypeIdentifierMap)); - if (DT.isSubroutineType()) { - DITypeArray DTA = DISubroutineType(DT).getTypeArray(); - for (unsigned i = 0, e = DTA.getNumElements(); i != e; ++i) - processType(DTA.getElement(i).resolve(TypeIdentifierMap)); + processScope(DT->getScope().resolve(TypeIdentifierMap)); + if (auto *DCT = dyn_cast(DT)) { + processType(DCT->getBaseType().resolve(TypeIdentifierMap)); + if (auto *ST = dyn_cast(DCT)) { + for (MDTypeRef Ref : ST->getTypeArray()) + processType(Ref.resolve(TypeIdentifierMap)); return; } - DIArray DA = DCT.getElements(); - for (unsigned i = 0, e = DA.getNumElements(); i != e; ++i) { - DIDescriptor D = DA.getElement(i); - if (D.isType()) - processType(DIType(D)); - else if (D.isSubprogram()) - processSubprogram(DISubprogram(D)); + for (Metadata *D : DCT->getElements()) { + if (auto *T = dyn_cast(D)) + processType(T); + else if (DISubprogram SP = dyn_cast(D)) + processSubprogram(SP); } - } else if (DT.isDerivedType()) { - DIDerivedType DDT(DT); - processType(DDT.getTypeDerivedFrom().resolve(TypeIdentifierMap)); + } else if (auto *DDT = dyn_cast(DT)) { + processType(DDT->getBaseType().resolve(TypeIdentifierMap)); } } void DebugInfoFinder::processScope(DIScope Scope) { - if (Scope.isType()) { - DIType Ty(Scope); + if (!Scope) + return; + if (DIType Ty = dyn_cast(Scope)) { processType(Ty); return; } - if (Scope.isCompileUnit()) { - addCompileUnit(DICompileUnit(Scope)); + if (DICompileUnit CU = dyn_cast(Scope)) { + addCompileUnit(CU); return; } - if (Scope.isSubprogram()) { - processSubprogram(DISubprogram(Scope)); + if (DISubprogram SP = dyn_cast(Scope)) { + processSubprogram(SP); return; } if (!addScope(Scope)) return; - if (Scope.isLexicalBlock()) { - DILexicalBlock LB(Scope); - processScope(LB.getContext()); - } else if (Scope.isLexicalBlockFile()) { - DILexicalBlockFile LBF = DILexicalBlockFile(Scope); - processScope(LBF.getScope()); - } else if (Scope.isNameSpace()) { - DINameSpace NS(Scope); - processScope(NS.getContext()); + if (auto *LB = dyn_cast(Scope)) { + processScope(LB->getScope()); + } else if (auto *NS = dyn_cast(Scope)) { + processScope(NS->getScope()); } } void DebugInfoFinder::processSubprogram(DISubprogram SP) { if (!addSubprogram(SP)) return; - processScope(SP.getContext().resolve(TypeIdentifierMap)); - processType(SP.getType()); - DIArray TParams = SP.getTemplateParams(); - for (unsigned I = 0, E = TParams.getNumElements(); I != E; ++I) { - DIDescriptor Element = TParams.getElement(I); - if (Element.isTemplateTypeParameter()) { - DITemplateTypeParameter TType(Element); - processType(TType.getType().resolve(TypeIdentifierMap)); - } else if (Element.isTemplateValueParameter()) { - DITemplateValueParameter TVal(Element); - processType(TVal.getType().resolve(TypeIdentifierMap)); + processScope(SP->getScope().resolve(TypeIdentifierMap)); + processType(SP->getType()); + for (auto *Element : SP->getTemplateParams()) { + if (auto *TType = dyn_cast(Element)) { + processType(TType->getType().resolve(TypeIdentifierMap)); + } else if (auto *TVal = dyn_cast(Element)) { + processType(TVal->getType().resolve(TypeIdentifierMap)); } } } @@ -797,14 +225,14 @@ void DebugInfoFinder::processDeclare(const Module &M, return; InitializeTypeMap(M); - DIDescriptor DV(N); - if (!DV.isVariable()) + DIVariable DV = dyn_cast(N); + if (!DV) return; if (!NodesSeen.insert(DV).second) return; - processScope(DIVariable(N).getContext()); - processType(DIVariable(N).getType().resolve(TypeIdentifierMap)); + processScope(DV->getScope()); + processType(DV->getType().resolve(TypeIdentifierMap)); } void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) { @@ -813,14 +241,14 @@ void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) { return; InitializeTypeMap(M); - DIDescriptor DV(N); - if (!DV.isVariable()) + DIVariable DV = dyn_cast(N); + if (!DV) return; if (!NodesSeen.insert(DV).second) return; - processScope(DIVariable(N).getContext()); - processType(DIVariable(N).getType().resolve(TypeIdentifierMap)); + processScope(DV->getScope()); + processType(DV->getType().resolve(TypeIdentifierMap)); } bool DebugInfoFinder::addType(DIType DT) { @@ -879,76 +307,17 @@ bool DebugInfoFinder::addScope(DIScope Scope) { return true; } -//===----------------------------------------------------------------------===// -// DIDescriptor: dump routines for all descriptors. -//===----------------------------------------------------------------------===// - -void DIDescriptor::dump() const { - print(dbgs()); - dbgs() << '\n'; -} - -void DIDescriptor::print(raw_ostream &OS) const { - if (!get()) - return; - get()->print(OS); -} - -static void printDebugLoc(DebugLoc DL, raw_ostream &CommentOS, - const LLVMContext &Ctx) { - if (!DL.isUnknown()) { // Print source line info. - DIScope Scope(DL.getScope(Ctx)); - assert(Scope.isScope() && "Scope of a DebugLoc should be a DIScope."); - // Omit the directory, because it's likely to be long and uninteresting. - CommentOS << Scope.getFilename(); - CommentOS << ':' << DL.getLine(); - if (DL.getCol() != 0) - CommentOS << ':' << DL.getCol(); - DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(DL.getInlinedAt(Ctx)); - if (!InlinedAtDL.isUnknown()) { - CommentOS << " @[ "; - printDebugLoc(InlinedAtDL, CommentOS, Ctx); - CommentOS << " ]"; - } - } -} - -void DIVariable::printExtendedName(raw_ostream &OS) const { - const LLVMContext &Ctx = DbgNode->getContext(); - StringRef Res = getName(); - if (!Res.empty()) - OS << Res << "," << getLineNumber(); - if (MDNode *InlinedAt = getInlinedAt()) { - DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(InlinedAt); - if (!InlinedAtDL.isUnknown()) { - OS << " @["; - printDebugLoc(InlinedAtDL, OS, Ctx); - OS << "]"; +bool llvm::stripDebugInfo(Function &F) { + bool Changed = false; + for (BasicBlock &BB : F) { + for (Instruction &I : BB) { + if (I.getDebugLoc()) { + Changed = true; + I.setDebugLoc(DebugLoc()); + } } } -} - -template <> DIRef::DIRef(const Metadata *V) : Val(V) { - assert(isDescriptorRef(V) && - "DIDescriptorRef should be a MDString or MDNode"); -} -template <> DIRef::DIRef(const Metadata *V) : Val(V) { - assert(isScopeRef(V) && "DIScopeRef should be a MDString or MDNode"); -} -template <> DIRef::DIRef(const Metadata *V) : Val(V) { - assert(isTypeRef(V) && "DITypeRef should be a MDString or MDNode"); -} - -template <> -DIDescriptorRef DIDescriptor::getFieldAs(unsigned Elt) const { - return DIDescriptorRef(cast_or_null(getField(DbgNode, Elt))); -} -template <> -DIScopeRef DIDescriptor::getFieldAs(unsigned Elt) const { - return DIScopeRef(cast_or_null(getField(DbgNode, Elt))); -} -template <> DITypeRef DIDescriptor::getFieldAs(unsigned Elt) const { - return DITypeRef(cast_or_null(getField(DbgNode, Elt))); + return Changed; } bool llvm::StripDebugInfo(Module &M) { @@ -984,16 +353,11 @@ bool llvm::StripDebugInfo(Module &M) { } } - for (Module::iterator MI = M.begin(), ME = M.end(); MI != ME; ++MI) - for (Function::iterator FI = MI->begin(), FE = MI->end(); FI != FE; - ++FI) - for (BasicBlock::iterator BI = FI->begin(), BE = FI->end(); BI != BE; - ++BI) { - if (!BI->getDebugLoc().isUnknown()) { - Changed = true; - BI->setDebugLoc(DebugLoc()); - } - } + for (Function &F : M) + Changed |= stripDebugInfo(F); + + if (GVMaterializer *Materializer = M.getMaterializer()) + Materializer->setStripDebugInfo(); return Changed; } @@ -1014,11 +378,9 @@ llvm::makeSubprogramMap(const Module &M) { return R; for (MDNode *N : CU_Nodes->operands()) { - DICompileUnit CUNode(N); - DIArray SPs = CUNode.getSubprograms(); - for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) { - DISubprogram SP(SPs.getElement(i)); - if (Function *F = SP.getFunction()) + DICompileUnit CUNode = cast(N); + for (DISubprogram SP : CUNode->getSubprograms()) { + if (Function *F = SP->getFunction()) R.insert(std::make_pair(F, SP)); } } diff --git a/lib/IR/DebugInfoMetadata.cpp b/lib/IR/DebugInfoMetadata.cpp index 89ec1bc..f6f2ff2 100644 --- a/lib/IR/DebugInfoMetadata.cpp +++ b/lib/IR/DebugInfoMetadata.cpp @@ -14,6 +14,7 @@ #include "llvm/IR/DebugInfoMetadata.h" #include "LLVMContextImpl.h" #include "MetadataImpl.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/IR/Function.h" using namespace llvm; @@ -44,6 +45,7 @@ MDLocation *MDLocation::getImpl(LLVMContext &Context, unsigned Line, // Fixup column. adjustColumn(Column); + assert(Scope && "Expected scope"); if (Storage == Uniqued) { if (auto *N = getUniqued(Context.pImpl->MDLocations, @@ -64,6 +66,96 @@ MDLocation *MDLocation::getImpl(LLVMContext &Context, unsigned Line, Storage, Context.pImpl->MDLocations); } +unsigned MDLocation::computeNewDiscriminator() const { + // FIXME: This seems completely wrong. + // + // 1. If two modules are generated in the same context, then the second + // Module will get different discriminators than it would have if it were + // generated in its own context. + // 2. If this function is called after round-tripping to bitcode instead of + // before, it will give a different (and potentially incorrect!) return. + // + // The discriminator should instead be calculated from local information + // where it's actually needed. This logic should be moved to + // AddDiscriminators::runOnFunction(), where it doesn't pollute the + // LLVMContext. + std::pair Key(getFilename().data(), getLine()); + return ++getContext().pImpl->DiscriminatorTable[Key]; +} + +unsigned DebugNode::getFlag(StringRef Flag) { + return StringSwitch(Flag) +#define HANDLE_DI_FLAG(ID, NAME) .Case("DIFlag" #NAME, Flag##NAME) +#include "llvm/IR/DebugInfoFlags.def" + .Default(0); +} + +const char *DebugNode::getFlagString(unsigned Flag) { + switch (Flag) { + default: + return ""; +#define HANDLE_DI_FLAG(ID, NAME) \ + case Flag##NAME: \ + return "DIFlag" #NAME; +#include "llvm/IR/DebugInfoFlags.def" + } +} + +unsigned DebugNode::splitFlags(unsigned Flags, + SmallVectorImpl &SplitFlags) { + // Accessibility flags need to be specially handled, since they're packed + // together. + if (unsigned A = Flags & FlagAccessibility) { + if (A == FlagPrivate) + SplitFlags.push_back(FlagPrivate); + else if (A == FlagProtected) + SplitFlags.push_back(FlagProtected); + else + SplitFlags.push_back(FlagPublic); + Flags &= ~A; + } + +#define HANDLE_DI_FLAG(ID, NAME) \ + if (unsigned Bit = Flags & ID) { \ + SplitFlags.push_back(Bit); \ + Flags &= ~Bit; \ + } +#include "llvm/IR/DebugInfoFlags.def" + + return Flags; +} + +MDScopeRef MDScope::getScope() const { + if (auto *T = dyn_cast(this)) + return T->getScope(); + + if (auto *SP = dyn_cast(this)) + return SP->getScope(); + + if (auto *LB = dyn_cast(this)) + return MDScopeRef(LB->getScope()); + + if (auto *NS = dyn_cast(this)) + return MDScopeRef(NS->getScope()); + + assert((isa(this) || isa(this)) && + "Unhandled type of scope."); + return nullptr; +} + +StringRef MDScope::getName() const { + if (auto *T = dyn_cast(this)) + return T->getName(); + if (auto *SP = dyn_cast(this)) + return SP->getName(); + if (auto *NS = dyn_cast(this)) + return NS->getName(); + assert((isa(this) || isa(this) || + isa(this)) && + "Unhandled type of scope."); + return ""; +} + static StringRef getString(const MDString *S) { if (S) return S->getString(); @@ -237,6 +329,12 @@ MDCompileUnit *MDCompileUnit::getImpl( (SourceLanguage, IsOptimized, RuntimeVersion, EmissionKind), Ops); } +MDSubprogram *MDLocalScope::getSubprogram() const { + if (auto *Block = dyn_cast(this)) + return Block->getScope()->getSubprogram(); + return const_cast(cast(this)); +} + MDSubprogram *MDSubprogram::getImpl( LLVMContext &Context, Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, @@ -262,6 +360,21 @@ MDSubprogram *MDSubprogram::getImpl( Ops); } +Function *MDSubprogram::getFunction() const { + // FIXME: Should this be looking through bitcasts? + return dyn_cast_or_null(getFunctionConstant()); +} + +bool MDSubprogram::describes(const Function *F) const { + assert(F && "Invalid function"); + if (F == getFunction()) + return true; + StringRef Name = getLinkageName(); + if (Name.empty()) + Name = getName(); + return F->getName() == Name; +} + void MDSubprogram::replaceFunction(Function *F) { replaceFunction(F ? ConstantAsMetadata::get(F) : static_cast(nullptr)); @@ -271,6 +384,7 @@ MDLexicalBlock *MDLexicalBlock::getImpl(LLVMContext &Context, Metadata *Scope, Metadata *File, unsigned Line, unsigned Column, StorageType Storage, bool ShouldCreate) { + assert(Scope && "Expected scope"); DEFINE_GETIMPL_LOOKUP(MDLexicalBlock, (Scope, File, Line, Column)); Metadata *Ops[] = {File, Scope}; DEFINE_GETIMPL_STORE(MDLexicalBlock, (Line, Column), Ops); @@ -281,6 +395,7 @@ MDLexicalBlockFile *MDLexicalBlockFile::getImpl(LLVMContext &Context, unsigned Discriminator, StorageType Storage, bool ShouldCreate) { + assert(Scope && "Expected scope"); DEFINE_GETIMPL_LOOKUP(MDLexicalBlockFile, (Scope, File, Discriminator)); Metadata *Ops[] = {File, Scope}; DEFINE_GETIMPL_STORE(MDLexicalBlockFile, (Discriminator), Ops); @@ -335,20 +450,23 @@ MDGlobalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, Ops); } -MDLocalVariable *MDLocalVariable::getImpl( - LLVMContext &Context, unsigned Tag, Metadata *Scope, MDString *Name, - Metadata *File, unsigned Line, Metadata *Type, unsigned Arg, unsigned Flags, - Metadata *InlinedAt, StorageType Storage, bool ShouldCreate) { +MDLocalVariable *MDLocalVariable::getImpl(LLVMContext &Context, unsigned Tag, + Metadata *Scope, MDString *Name, + Metadata *File, unsigned Line, + Metadata *Type, unsigned Arg, + unsigned Flags, StorageType Storage, + bool ShouldCreate) { // Truncate Arg to 8 bits. // // FIXME: This is gross (and should be changed to an assert or removed), but // it matches historical behaviour for now. Arg &= (1u << 8) - 1; + assert(Scope && "Expected scope"); assert(isCanonical(Name) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(MDLocalVariable, (Tag, Scope, getString(Name), File, - Line, Type, Arg, Flags, InlinedAt)); - Metadata *Ops[] = {Scope, Name, File, Type, InlinedAt}; + Line, Type, Arg, Flags)); + Metadata *Ops[] = {Scope, Name, File, Type}; DEFINE_GETIMPL_STORE(MDLocalVariable, (Tag, Line, Arg, Flags), Ops); } @@ -391,6 +509,24 @@ bool MDExpression::isValid() const { return true; } +bool MDExpression::isBitPiece() const { + assert(isValid() && "Expected valid expression"); + if (unsigned N = getNumElements()) + if (N >= 3) + return getElement(N - 3) == dwarf::DW_OP_bit_piece; + return false; +} + +uint64_t MDExpression::getBitPieceOffset() const { + assert(isBitPiece() && "Expected bit piece"); + return getElement(getNumElements() - 2); +} + +uint64_t MDExpression::getBitPieceSize() const { + assert(isBitPiece() && "Expected bit piece"); + return getElement(getNumElements() - 1); +} + MDObjCProperty *MDObjCProperty::getImpl( LLVMContext &Context, MDString *Name, Metadata *File, unsigned Line, MDString *GetterName, MDString *SetterName, unsigned Attributes, diff --git a/lib/IR/DebugLoc.cpp b/lib/IR/DebugLoc.cpp index e1bf795..4cf7e9e 100644 --- a/lib/IR/DebugLoc.cpp +++ b/lib/IR/DebugLoc.cpp @@ -16,101 +16,87 @@ using namespace llvm; //===----------------------------------------------------------------------===// // DebugLoc Implementation //===----------------------------------------------------------------------===// +DebugLoc::DebugLoc(const MDLocation *L) : Loc(const_cast(L)) {} +DebugLoc::DebugLoc(const MDNode *L) : Loc(const_cast(L)) {} -unsigned DebugLoc::getLine() const { return DILocation(Loc).getLineNumber(); } -unsigned DebugLoc::getCol() const { return DILocation(Loc).getColumnNumber(); } +MDLocation *DebugLoc::get() const { + return cast_or_null(Loc.get()); +} -MDNode *DebugLoc::getScope() const { return DILocation(Loc).getScope(); } +unsigned DebugLoc::getLine() const { + assert(get() && "Expected valid DebugLoc"); + return get()->getLine(); +} -MDNode *DebugLoc::getInlinedAt() const { - return DILocation(Loc).getOrigLocation(); +unsigned DebugLoc::getCol() const { + assert(get() && "Expected valid DebugLoc"); + return get()->getColumn(); } -/// Return both the Scope and the InlinedAt values. -void DebugLoc::getScopeAndInlinedAt(MDNode *&Scope, MDNode *&IA) const { - Scope = getScope(); - IA = getInlinedAt(); +MDNode *DebugLoc::getScope() const { + assert(get() && "Expected valid DebugLoc"); + return get()->getScope(); } -MDNode *DebugLoc::getScopeNode() const { - if (MDNode *InlinedAt = getInlinedAt()) - return DebugLoc::getFromDILocation(InlinedAt).getScopeNode(); - return getScope(); +MDLocation *DebugLoc::getInlinedAt() const { + assert(get() && "Expected valid DebugLoc"); + return get()->getInlinedAt(); +} + +MDNode *DebugLoc::getInlinedAtScope() const { + return cast(Loc)->getInlinedAtScope(); } DebugLoc DebugLoc::getFnDebugLoc() const { - const MDNode *Scope = getScopeNode(); - DISubprogram SP = getDISubprogram(Scope); - if (SP.isSubprogram()) - return DebugLoc::get(SP.getScopeLineNumber(), 0, SP); + // FIXME: Add a method on \a MDLocation that does this work. + const MDNode *Scope = getInlinedAtScope(); + if (DISubprogram SP = getDISubprogram(Scope)) + return DebugLoc::get(SP->getScopeLine(), 0, SP); return DebugLoc(); } -DebugLoc DebugLoc::get(unsigned Line, unsigned Col, - MDNode *Scope, MDNode *InlinedAt) { +DebugLoc DebugLoc::get(unsigned Line, unsigned Col, const MDNode *Scope, + const MDNode *InlinedAt) { // If no scope is available, this is an unknown location. if (!Scope) return DebugLoc(); - return getFromDILocation( - MDLocation::get(Scope->getContext(), Line, Col, Scope, InlinedAt)); -} - -/// getAsMDNode - This method converts the compressed DebugLoc node into a -/// DILocation-compatible MDNode. -MDNode *DebugLoc::getAsMDNode() const { return Loc; } - -/// getFromDILocation - Translate the DILocation quad into a DebugLoc. -DebugLoc DebugLoc::getFromDILocation(MDNode *N) { - DebugLoc Loc; - Loc.Loc.reset(N); - return Loc; -} - -/// getFromDILexicalBlock - Translate the DILexicalBlock into a DebugLoc. -DebugLoc DebugLoc::getFromDILexicalBlock(MDNode *N) { - DILexicalBlock LexBlock(N); - MDNode *Scope = LexBlock.getContext(); - if (!Scope) return DebugLoc(); - return get(LexBlock.getLineNumber(), LexBlock.getColumnNumber(), Scope, - nullptr); + return MDLocation::get(Scope->getContext(), Line, Col, + const_cast(Scope), + const_cast(InlinedAt)); } void DebugLoc::dump() const { #ifndef NDEBUG - if (!isUnknown()) { - dbgs() << getLine(); - if (getCol() != 0) - dbgs() << ',' << getCol(); - DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(getInlinedAt()); - if (!InlinedAtDL.isUnknown()) { - dbgs() << " @ "; - InlinedAtDL.dump(); - } else - dbgs() << "\n"; - } + if (!Loc) + return; + + dbgs() << getLine(); + if (getCol() != 0) + dbgs() << ',' << getCol(); + if (DebugLoc InlinedAtDL = DebugLoc(getInlinedAt())) { + dbgs() << " @ "; + InlinedAtDL.dump(); + } else + dbgs() << "\n"; #endif } void DebugLoc::print(raw_ostream &OS) const { - if (!isUnknown()) { - // Print source line info. - DIScope Scope(getScope()); - assert((!Scope || Scope.isScope()) && - "Scope of a DebugLoc should be null or a DIScope."); - if (Scope) - OS << Scope.getFilename(); - else - OS << ""; - OS << ':' << getLine(); - if (getCol() != 0) - OS << ':' << getCol(); - DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(getInlinedAt()); - if (!InlinedAtDL.isUnknown()) { - OS << " @[ "; - InlinedAtDL.print(OS); - OS << " ]"; - } + if (!Loc) + return; + + // Print source line info. + auto *Scope = cast(getScope()); + OS << Scope->getFilename(); + OS << ':' << getLine(); + if (getCol() != 0) + OS << ':' << getCol(); + + if (DebugLoc InlinedAtDL = getInlinedAt()) { + OS << " @[ "; + InlinedAtDL.print(OS); + OS << " ]"; } } diff --git a/lib/IR/DiagnosticInfo.cpp b/lib/IR/DiagnosticInfo.cpp index 5608589..91635f6 100644 --- a/lib/IR/DiagnosticInfo.cpp +++ b/lib/IR/DiagnosticInfo.cpp @@ -129,16 +129,16 @@ void DiagnosticInfoSampleProfile::print(DiagnosticPrinter &DP) const { } bool DiagnosticInfoOptimizationBase::isLocationAvailable() const { - return !getDebugLoc().isUnknown(); + return getDebugLoc(); } void DiagnosticInfoOptimizationBase::getLocation(StringRef *Filename, unsigned *Line, unsigned *Column) const { - DILocation DIL(getDebugLoc().getAsMDNode(getFunction().getContext())); - *Filename = DIL.getFilename(); - *Line = DIL.getLineNumber(); - *Column = DIL.getColumnNumber(); + MDLocation *L = getDebugLoc(); + *Filename = L->getFilename(); + *Line = L->getLine(); + *Column = L->getColumn(); } const std::string DiagnosticInfoOptimizationBase::getLocationStr() const { @@ -147,7 +147,7 @@ const std::string DiagnosticInfoOptimizationBase::getLocationStr() const { unsigned Column = 0; if (isLocationAvailable()) getLocation(&Filename, &Line, &Column); - return Twine(Filename + ":" + Twine(Line) + ":" + Twine(Column)).str(); + return (Filename + ":" + Twine(Line) + ":" + Twine(Column)).str(); } void DiagnosticInfoOptimizationBase::print(DiagnosticPrinter &DP) const { diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp index 33e1526..d3a0934 100644 --- a/lib/IR/Function.cpp +++ b/lib/IR/Function.cpp @@ -215,9 +215,7 @@ LLVMContext &Function::getContext() const { return getType()->getContext(); } -FunctionType *Function::getFunctionType() const { - return cast(getType()->getElementType()); -} +FunctionType *Function::getFunctionType() const { return Ty; } bool Function::isVarArg() const { return getFunctionType()->isVarArg(); @@ -242,7 +240,8 @@ void Function::eraseFromParent() { Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name, Module *ParentModule) : GlobalObject(PointerType::getUnqual(Ty), Value::FunctionVal, nullptr, 0, - Linkage, name) { + Linkage, name), + Ty(Ty) { assert(FunctionType::isValidReturnType(getReturnType()) && "invalid return type"); setIsMaterializable(false); @@ -349,6 +348,12 @@ void Function::addDereferenceableAttr(unsigned i, uint64_t Bytes) { setAttributes(PAL); } +void Function::addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) { + AttributeSet PAL = getAttributes(); + PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes); + setAttributes(PAL); +} + // Maintain the GC name for each function in an on-the-side table. This saves // allocating an additional word in Function for programs which do not use GC // (i.e., most programs) at the cost of increased overhead for clients which do diff --git a/lib/IR/GCOV.cpp b/lib/IR/GCOV.cpp index 7010ceb..6ed5891 100644 --- a/lib/IR/GCOV.cpp +++ b/lib/IR/GCOV.cpp @@ -555,7 +555,7 @@ FileInfo::openCoveragePath(StringRef CoveragePath) { return llvm::make_unique(); std::error_code EC; - auto OS = llvm::make_unique(CoveragePath.str(), EC, + auto OS = llvm::make_unique(CoveragePath, EC, sys::fs::F_Text); if (EC) { errs() << EC.message() << "\n"; diff --git a/lib/IR/IRBuilder.cpp b/lib/IR/IRBuilder.cpp index 90303b2..06f54c7 100644 --- a/lib/IR/IRBuilder.cpp +++ b/lib/IR/IRBuilder.cpp @@ -23,7 +23,8 @@ using namespace llvm; /// has array of i8 type filled in with the nul terminated string value /// specified. If Name is specified, it is the name of the global variable /// created. -Value *IRBuilderBase::CreateGlobalString(StringRef Str, const Twine &Name) { +GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str, + const Twine &Name) { Constant *StrConstant = ConstantDataArray::getString(Context, Str); Module &M = *BB->getParent()->getParent(); GlobalVariable *GV = new GlobalVariable(M, StrConstant->getType(), diff --git a/lib/IR/IRPrintingPasses.cpp b/lib/IR/IRPrintingPasses.cpp index 91ccfbb..c1ac336 100644 --- a/lib/IR/IRPrintingPasses.cpp +++ b/lib/IR/IRPrintingPasses.cpp @@ -21,11 +21,14 @@ using namespace llvm; PrintModulePass::PrintModulePass() : OS(dbgs()) {} -PrintModulePass::PrintModulePass(raw_ostream &OS, const std::string &Banner) - : OS(OS), Banner(Banner) {} +PrintModulePass::PrintModulePass(raw_ostream &OS, const std::string &Banner, + bool ShouldPreserveUseListOrder) + : OS(OS), Banner(Banner), + ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {} PreservedAnalyses PrintModulePass::run(Module &M) { - OS << Banner << M; + OS << Banner; + M.print(OS, nullptr, ShouldPreserveUseListOrder); return PreservedAnalyses::all(); } @@ -46,8 +49,9 @@ class PrintModulePassWrapper : public ModulePass { public: static char ID; PrintModulePassWrapper() : ModulePass(ID) {} - PrintModulePassWrapper(raw_ostream &OS, const std::string &Banner) - : ModulePass(ID), P(OS, Banner) {} + PrintModulePassWrapper(raw_ostream &OS, const std::string &Banner, + bool ShouldPreserveUseListOrder) + : ModulePass(ID), P(OS, Banner, ShouldPreserveUseListOrder) {} bool runOnModule(Module &M) override { P.run(M); @@ -112,8 +116,9 @@ INITIALIZE_PASS(PrintBasicBlockPass, "print-bb", "Print BB to stderr", false, false) ModulePass *llvm::createPrintModulePass(llvm::raw_ostream &OS, - const std::string &Banner) { - return new PrintModulePassWrapper(OS, Banner); + const std::string &Banner, + bool ShouldPreserveUseListOrder) { + return new PrintModulePassWrapper(OS, Banner, ShouldPreserveUseListOrder); } FunctionPass *llvm::createPrintFunctionPass(llvm::raw_ostream &OS, diff --git a/lib/IR/InlineAsm.cpp b/lib/IR/InlineAsm.cpp index b456d9f..aa9e027 100644 --- a/lib/IR/InlineAsm.cpp +++ b/lib/IR/InlineAsm.cpp @@ -167,7 +167,9 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, // Note that operand #n has a matching input. scInfo.MatchingInput = ConstraintsSoFar.size(); } else { - if (ConstraintsSoFar[N].hasMatchingInput()) + if (ConstraintsSoFar[N].hasMatchingInput() && + (size_t)ConstraintsSoFar[N].MatchingInput != + ConstraintsSoFar.size()) return true; // Note that operand #n has a matching input. ConstraintsSoFar[N].MatchingInput = ConstraintsSoFar.size(); diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp index 7d9bd7e..57c143c 100644 --- a/lib/IR/Instruction.cpp +++ b/lib/IR/Instruction.cpp @@ -62,8 +62,8 @@ void Instruction::removeFromParent() { getParent()->getInstList().remove(this); } -void Instruction::eraseFromParent() { - getParent()->getInstList().erase(this); +iplist::iterator Instruction::eraseFromParent() { + return getParent()->getInstList().erase(this); } /// insertBefore - Insert an unlinked instructions into a basic block diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp index af2aeb9..85b7521 100644 --- a/lib/IR/Instructions.cpp +++ b/lib/IR/Instructions.cpp @@ -352,6 +352,12 @@ void CallInst::addDereferenceableAttr(unsigned i, uint64_t Bytes) { setAttributes(PAL); } +void CallInst::addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) { + AttributeSet PAL = getAttributes(); + PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes); + setAttributes(PAL); +} + bool CallInst::hasFnAttrImpl(Attribute::AttrKind A) const { if (AttributeList.hasAttribute(AttributeSet::FunctionIndex, A)) return true; @@ -617,6 +623,12 @@ void InvokeInst::addDereferenceableAttr(unsigned i, uint64_t Bytes) { setAttributes(PAL); } +void InvokeInst::addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) { + AttributeSet PAL = getAttributes(); + PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes); + setAttributes(PAL); +} + LandingPadInst *InvokeInst::getLandingPadInst() const { return cast(getUnwindDest()->getFirstNonPHI()); } @@ -943,12 +955,10 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, : LoadInst(Ptr, Name, isVolatile, Align, NotAtomic, CrossThread, InsertAE) { } -LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, +LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, - Instruction *InsertBef) - : UnaryInstruction(cast(Ptr->getType())->getElementType(), - Load, Ptr, InsertBef) { + SynchronizationScope SynchScope, Instruction *InsertBef) + : UnaryInstruction(Ty, Load, Ptr, InsertBef) { setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SynchScope); @@ -1258,11 +1268,7 @@ GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI) /// pointer type. /// template -static Type *getIndexedTypeInternal(Type *Ptr, ArrayRef IdxList) { - PointerType *PTy = dyn_cast(Ptr->getScalarType()); - if (!PTy) return nullptr; // Type isn't a pointer type! - Type *Agg = PTy->getElementType(); - +static Type *getIndexedTypeInternal(Type *Agg, ArrayRef IdxList) { // Handle the special case of the empty set index set, which is always valid. if (IdxList.empty()) return Agg; @@ -1283,17 +1289,17 @@ static Type *getIndexedTypeInternal(Type *Ptr, ArrayRef IdxList) { return CurIdx == IdxList.size() ? Agg : nullptr; } -Type *GetElementPtrInst::getIndexedType(Type *Ptr, ArrayRef IdxList) { - return getIndexedTypeInternal(Ptr, IdxList); +Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef IdxList) { + return getIndexedTypeInternal(Ty, IdxList); } -Type *GetElementPtrInst::getIndexedType(Type *Ptr, +Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef IdxList) { - return getIndexedTypeInternal(Ptr, IdxList); + return getIndexedTypeInternal(Ty, IdxList); } -Type *GetElementPtrInst::getIndexedType(Type *Ptr, ArrayRef IdxList) { - return getIndexedTypeInternal(Ptr, IdxList); +Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef IdxList) { + return getIndexedTypeInternal(Ty, IdxList); } /// hasAllZeroIndices - Return true if all of the indices of this GEP are diff --git a/lib/IR/LLVMContextImpl.h b/lib/IR/LLVMContextImpl.h index e380665..c096a83 100644 --- a/lib/IR/LLVMContextImpl.h +++ b/lib/IR/LLVMContextImpl.h @@ -275,15 +275,17 @@ template <> struct MDNodeKeyImpl : MDNodeOpsKey { template <> struct MDNodeKeyImpl { int64_t Count; - int64_t Lo; + int64_t LowerBound; - MDNodeKeyImpl(int64_t Count, int64_t Lo) : Count(Count), Lo(Lo) {} - MDNodeKeyImpl(const MDSubrange *N) : Count(N->getCount()), Lo(N->getLo()) {} + MDNodeKeyImpl(int64_t Count, int64_t LowerBound) + : Count(Count), LowerBound(LowerBound) {} + MDNodeKeyImpl(const MDSubrange *N) + : Count(N->getCount()), LowerBound(N->getLowerBound()) {} bool isKeyOf(const MDSubrange *RHS) const { - return Count == RHS->getCount() && Lo == RHS->getLo(); + return Count == RHS->getCount() && LowerBound == RHS->getLowerBound(); } - unsigned getHashValue() const { return hash_combine(Count, Lo); } + unsigned getHashValue() const { return hash_combine(Count, LowerBound); } }; template <> struct MDNodeKeyImpl { @@ -347,20 +349,20 @@ template <> struct MDNodeKeyImpl { BaseType(BaseType), SizeInBits(SizeInBits), AlignInBits(AlignInBits), OffsetInBits(OffsetInBits), Flags(Flags), ExtraData(ExtraData) {} MDNodeKeyImpl(const MDDerivedType *N) - : Tag(N->getTag()), Name(N->getName()), File(N->getFile()), - Line(N->getLine()), Scope(N->getScope()), BaseType(N->getBaseType()), - SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()), - OffsetInBits(N->getOffsetInBits()), Flags(N->getFlags()), - ExtraData(N->getExtraData()) {} + : Tag(N->getTag()), Name(N->getName()), File(N->getRawFile()), + Line(N->getLine()), Scope(N->getRawScope()), + BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()), + AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()), + Flags(N->getFlags()), ExtraData(N->getRawExtraData()) {} bool isKeyOf(const MDDerivedType *RHS) const { return Tag == RHS->getTag() && Name == RHS->getName() && - File == RHS->getFile() && Line == RHS->getLine() && - Scope == RHS->getScope() && BaseType == RHS->getBaseType() && + File == RHS->getRawFile() && Line == RHS->getLine() && + Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() && SizeInBits == RHS->getSizeInBits() && AlignInBits == RHS->getAlignInBits() && OffsetInBits == RHS->getOffsetInBits() && Flags == RHS->getFlags() && - ExtraData == RHS->getExtraData(); + ExtraData == RHS->getRawExtraData(); } unsigned getHashValue() const { return hash_combine(Tag, Name, File, Line, Scope, BaseType, SizeInBits, @@ -397,26 +399,26 @@ template <> struct MDNodeKeyImpl { RuntimeLang(RuntimeLang), VTableHolder(VTableHolder), TemplateParams(TemplateParams), Identifier(Identifier) {} MDNodeKeyImpl(const MDCompositeType *N) - : Tag(N->getTag()), Name(N->getName()), File(N->getFile()), - Line(N->getLine()), Scope(N->getScope()), BaseType(N->getBaseType()), - SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()), - OffsetInBits(N->getOffsetInBits()), Flags(N->getFlags()), - Elements(N->getElements()), RuntimeLang(N->getRuntimeLang()), - VTableHolder(N->getVTableHolder()), - TemplateParams(N->getTemplateParams()), Identifier(N->getIdentifier()) { - } + : Tag(N->getTag()), Name(N->getName()), File(N->getRawFile()), + Line(N->getLine()), Scope(N->getRawScope()), + BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()), + AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()), + Flags(N->getFlags()), Elements(N->getRawElements()), + RuntimeLang(N->getRuntimeLang()), VTableHolder(N->getRawVTableHolder()), + TemplateParams(N->getRawTemplateParams()), + Identifier(N->getIdentifier()) {} bool isKeyOf(const MDCompositeType *RHS) const { return Tag == RHS->getTag() && Name == RHS->getName() && - File == RHS->getFile() && Line == RHS->getLine() && - Scope == RHS->getScope() && BaseType == RHS->getBaseType() && + File == RHS->getRawFile() && Line == RHS->getLine() && + Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() && SizeInBits == RHS->getSizeInBits() && AlignInBits == RHS->getAlignInBits() && OffsetInBits == RHS->getOffsetInBits() && Flags == RHS->getFlags() && - Elements == RHS->getElements() && + Elements == RHS->getRawElements() && RuntimeLang == RHS->getRuntimeLang() && - VTableHolder == RHS->getVTableHolder() && - TemplateParams == RHS->getTemplateParams() && + VTableHolder == RHS->getRawVTableHolder() && + TemplateParams == RHS->getRawTemplateParams() && Identifier == RHS->getIdentifier(); } unsigned getHashValue() const { @@ -433,10 +435,10 @@ template <> struct MDNodeKeyImpl { MDNodeKeyImpl(int64_t Flags, Metadata *TypeArray) : Flags(Flags), TypeArray(TypeArray) {} MDNodeKeyImpl(const MDSubroutineType *N) - : Flags(N->getFlags()), TypeArray(N->getTypeArray()) {} + : Flags(N->getFlags()), TypeArray(N->getRawTypeArray()) {} bool isKeyOf(const MDSubroutineType *RHS) const { - return Flags == RHS->getFlags() && TypeArray == RHS->getTypeArray(); + return Flags == RHS->getFlags() && TypeArray == RHS->getRawTypeArray(); } unsigned getHashValue() const { return hash_combine(Flags, TypeArray); } }; @@ -484,27 +486,28 @@ template <> struct MDNodeKeyImpl { Subprograms(Subprograms), GlobalVariables(GlobalVariables), ImportedEntities(ImportedEntities) {} MDNodeKeyImpl(const MDCompileUnit *N) - : SourceLanguage(N->getSourceLanguage()), File(N->getFile()), + : SourceLanguage(N->getSourceLanguage()), File(N->getRawFile()), Producer(N->getProducer()), IsOptimized(N->isOptimized()), Flags(N->getFlags()), RuntimeVersion(N->getRuntimeVersion()), SplitDebugFilename(N->getSplitDebugFilename()), - EmissionKind(N->getEmissionKind()), EnumTypes(N->getEnumTypes()), - RetainedTypes(N->getRetainedTypes()), Subprograms(N->getSubprograms()), - GlobalVariables(N->getGlobalVariables()), - ImportedEntities(N->getImportedEntities()) {} + EmissionKind(N->getEmissionKind()), EnumTypes(N->getRawEnumTypes()), + RetainedTypes(N->getRawRetainedTypes()), + Subprograms(N->getRawSubprograms()), + GlobalVariables(N->getRawGlobalVariables()), + ImportedEntities(N->getRawImportedEntities()) {} bool isKeyOf(const MDCompileUnit *RHS) const { return SourceLanguage == RHS->getSourceLanguage() && - File == RHS->getFile() && Producer == RHS->getProducer() && + File == RHS->getRawFile() && Producer == RHS->getProducer() && IsOptimized == RHS->isOptimized() && Flags == RHS->getFlags() && RuntimeVersion == RHS->getRuntimeVersion() && SplitDebugFilename == RHS->getSplitDebugFilename() && EmissionKind == RHS->getEmissionKind() && - EnumTypes == RHS->getEnumTypes() && - RetainedTypes == RHS->getRetainedTypes() && - Subprograms == RHS->getSubprograms() && - GlobalVariables == RHS->getGlobalVariables() && - ImportedEntities == RHS->getImportedEntities(); + EnumTypes == RHS->getRawEnumTypes() && + RetainedTypes == RHS->getRawRetainedTypes() && + Subprograms == RHS->getRawSubprograms() && + GlobalVariables == RHS->getRawGlobalVariables() && + ImportedEntities == RHS->getRawImportedEntities(); } unsigned getHashValue() const { return hash_combine(SourceLanguage, File, Producer, IsOptimized, Flags, @@ -549,31 +552,32 @@ template <> struct MDNodeKeyImpl { Function(Function), TemplateParams(TemplateParams), Declaration(Declaration), Variables(Variables) {} MDNodeKeyImpl(const MDSubprogram *N) - : Scope(N->getScope()), Name(N->getName()), - LinkageName(N->getLinkageName()), File(N->getFile()), - Line(N->getLine()), Type(N->getType()), + : Scope(N->getRawScope()), Name(N->getName()), + LinkageName(N->getLinkageName()), File(N->getRawFile()), + Line(N->getLine()), Type(N->getRawType()), IsLocalToUnit(N->isLocalToUnit()), IsDefinition(N->isDefinition()), - ScopeLine(N->getScopeLine()), ContainingType(N->getContainingType()), + ScopeLine(N->getScopeLine()), ContainingType(N->getRawContainingType()), Virtuality(N->getVirtuality()), VirtualIndex(N->getVirtualIndex()), Flags(N->getFlags()), IsOptimized(N->isOptimized()), - Function(N->getFunction()), TemplateParams(N->getTemplateParams()), - Declaration(N->getDeclaration()), Variables(N->getVariables()) {} + Function(N->getRawFunction()), + TemplateParams(N->getRawTemplateParams()), + Declaration(N->getRawDeclaration()), Variables(N->getRawVariables()) {} bool isKeyOf(const MDSubprogram *RHS) const { - return Scope == RHS->getScope() && Name == RHS->getName() && - LinkageName == RHS->getLinkageName() && File == RHS->getFile() && - Line == RHS->getLine() && Type == RHS->getType() && + return Scope == RHS->getRawScope() && Name == RHS->getName() && + LinkageName == RHS->getLinkageName() && File == RHS->getRawFile() && + Line == RHS->getLine() && Type == RHS->getRawType() && IsLocalToUnit == RHS->isLocalToUnit() && IsDefinition == RHS->isDefinition() && ScopeLine == RHS->getScopeLine() && - ContainingType == RHS->getContainingType() && + ContainingType == RHS->getRawContainingType() && Virtuality == RHS->getVirtuality() && VirtualIndex == RHS->getVirtualIndex() && Flags == RHS->getFlags() && IsOptimized == RHS->isOptimized() && - Function == RHS->getFunction() && - TemplateParams == RHS->getTemplateParams() && - Declaration == RHS->getDeclaration() && - Variables == RHS->getVariables(); + Function == RHS->getRawFunction() && + TemplateParams == RHS->getRawTemplateParams() && + Declaration == RHS->getRawDeclaration() && + Variables == RHS->getRawVariables(); } unsigned getHashValue() const { return hash_combine(Scope, Name, LinkageName, File, Line, Type, @@ -592,11 +596,11 @@ template <> struct MDNodeKeyImpl { MDNodeKeyImpl(Metadata *Scope, Metadata *File, unsigned Line, unsigned Column) : Scope(Scope), File(File), Line(Line), Column(Column) {} MDNodeKeyImpl(const MDLexicalBlock *N) - : Scope(N->getScope()), File(N->getFile()), Line(N->getLine()), + : Scope(N->getRawScope()), File(N->getRawFile()), Line(N->getLine()), Column(N->getColumn()) {} bool isKeyOf(const MDLexicalBlock *RHS) const { - return Scope == RHS->getScope() && File == RHS->getFile() && + return Scope == RHS->getRawScope() && File == RHS->getRawFile() && Line == RHS->getLine() && Column == RHS->getColumn(); } unsigned getHashValue() const { @@ -612,11 +616,11 @@ template <> struct MDNodeKeyImpl { MDNodeKeyImpl(Metadata *Scope, Metadata *File, unsigned Discriminator) : Scope(Scope), File(File), Discriminator(Discriminator) {} MDNodeKeyImpl(const MDLexicalBlockFile *N) - : Scope(N->getScope()), File(N->getFile()), + : Scope(N->getRawScope()), File(N->getRawFile()), Discriminator(N->getDiscriminator()) {} bool isKeyOf(const MDLexicalBlockFile *RHS) const { - return Scope == RHS->getScope() && File == RHS->getFile() && + return Scope == RHS->getRawScope() && File == RHS->getRawFile() && Discriminator == RHS->getDiscriminator(); } unsigned getHashValue() const { @@ -633,11 +637,11 @@ template <> struct MDNodeKeyImpl { MDNodeKeyImpl(Metadata *Scope, Metadata *File, StringRef Name, unsigned Line) : Scope(Scope), File(File), Name(Name), Line(Line) {} MDNodeKeyImpl(const MDNamespace *N) - : Scope(N->getScope()), File(N->getFile()), Name(N->getName()), + : Scope(N->getRawScope()), File(N->getRawFile()), Name(N->getName()), Line(N->getLine()) {} bool isKeyOf(const MDNamespace *RHS) const { - return Scope == RHS->getScope() && File == RHS->getFile() && + return Scope == RHS->getRawScope() && File == RHS->getRawFile() && Name == RHS->getName() && Line == RHS->getLine(); } unsigned getHashValue() const { @@ -651,10 +655,10 @@ template <> struct MDNodeKeyImpl { MDNodeKeyImpl(StringRef Name, Metadata *Type) : Name(Name), Type(Type) {} MDNodeKeyImpl(const MDTemplateTypeParameter *N) - : Name(N->getName()), Type(N->getType()) {} + : Name(N->getName()), Type(N->getRawType()) {} bool isKeyOf(const MDTemplateTypeParameter *RHS) const { - return Name == RHS->getName() && Type == RHS->getType(); + return Name == RHS->getName() && Type == RHS->getRawType(); } unsigned getHashValue() const { return hash_combine(Name, Type); } }; @@ -668,12 +672,12 @@ template <> struct MDNodeKeyImpl { MDNodeKeyImpl(unsigned Tag, StringRef Name, Metadata *Type, Metadata *Value) : Tag(Tag), Name(Name), Type(Type), Value(Value) {} MDNodeKeyImpl(const MDTemplateValueParameter *N) - : Tag(N->getTag()), Name(N->getName()), Type(N->getType()), + : Tag(N->getTag()), Name(N->getName()), Type(N->getRawType()), Value(N->getValue()) {} bool isKeyOf(const MDTemplateValueParameter *RHS) const { return Tag == RHS->getTag() && Name == RHS->getName() && - Type == RHS->getType() && Value == RHS->getValue(); + Type == RHS->getRawType() && Value == RHS->getValue(); } unsigned getHashValue() const { return hash_combine(Tag, Name, Type, Value); } }; @@ -699,21 +703,22 @@ template <> struct MDNodeKeyImpl { IsDefinition(IsDefinition), Variable(Variable), StaticDataMemberDeclaration(StaticDataMemberDeclaration) {} MDNodeKeyImpl(const MDGlobalVariable *N) - : Scope(N->getScope()), Name(N->getName()), - LinkageName(N->getLinkageName()), File(N->getFile()), - Line(N->getLine()), Type(N->getType()), + : Scope(N->getRawScope()), Name(N->getName()), + LinkageName(N->getLinkageName()), File(N->getRawFile()), + Line(N->getLine()), Type(N->getRawType()), IsLocalToUnit(N->isLocalToUnit()), IsDefinition(N->isDefinition()), - Variable(N->getVariable()), - StaticDataMemberDeclaration(N->getStaticDataMemberDeclaration()) {} + Variable(N->getRawVariable()), + StaticDataMemberDeclaration(N->getRawStaticDataMemberDeclaration()) {} bool isKeyOf(const MDGlobalVariable *RHS) const { - return Scope == RHS->getScope() && Name == RHS->getName() && - LinkageName == RHS->getLinkageName() && File == RHS->getFile() && - Line == RHS->getLine() && Type == RHS->getType() && + return Scope == RHS->getRawScope() && Name == RHS->getName() && + LinkageName == RHS->getLinkageName() && File == RHS->getRawFile() && + Line == RHS->getLine() && Type == RHS->getRawType() && IsLocalToUnit == RHS->isLocalToUnit() && IsDefinition == RHS->isDefinition() && - Variable == RHS->getVariable() && - StaticDataMemberDeclaration == RHS->getStaticDataMemberDeclaration(); + Variable == RHS->getRawVariable() && + StaticDataMemberDeclaration == + RHS->getRawStaticDataMemberDeclaration(); } unsigned getHashValue() const { return hash_combine(Scope, Name, LinkageName, File, Line, Type, @@ -731,28 +736,24 @@ template <> struct MDNodeKeyImpl { Metadata *Type; unsigned Arg; unsigned Flags; - Metadata *InlinedAt; MDNodeKeyImpl(unsigned Tag, Metadata *Scope, StringRef Name, Metadata *File, - unsigned Line, Metadata *Type, unsigned Arg, unsigned Flags, - Metadata *InlinedAt) + unsigned Line, Metadata *Type, unsigned Arg, unsigned Flags) : Tag(Tag), Scope(Scope), Name(Name), File(File), Line(Line), Type(Type), - Arg(Arg), Flags(Flags), InlinedAt(InlinedAt) {} + Arg(Arg), Flags(Flags) {} MDNodeKeyImpl(const MDLocalVariable *N) - : Tag(N->getTag()), Scope(N->getScope()), Name(N->getName()), - File(N->getFile()), Line(N->getLine()), Type(N->getType()), - Arg(N->getArg()), Flags(N->getFlags()), InlinedAt(N->getInlinedAt()) {} + : Tag(N->getTag()), Scope(N->getRawScope()), Name(N->getName()), + File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()), + Arg(N->getArg()), Flags(N->getFlags()) {} bool isKeyOf(const MDLocalVariable *RHS) const { - return Tag == RHS->getTag() && Scope == RHS->getScope() && - Name == RHS->getName() && File == RHS->getFile() && - Line == RHS->getLine() && Type == RHS->getType() && - Arg == RHS->getArg() && Flags == RHS->getFlags() && - InlinedAt == RHS->getInlinedAt(); + return Tag == RHS->getTag() && Scope == RHS->getRawScope() && + Name == RHS->getName() && File == RHS->getRawFile() && + Line == RHS->getLine() && Type == RHS->getRawType() && + Arg == RHS->getArg() && Flags == RHS->getFlags(); } unsigned getHashValue() const { - return hash_combine(Tag, Scope, Name, File, Line, Type, Arg, Flags, - InlinedAt); + return hash_combine(Tag, Scope, Name, File, Line, Type, Arg, Flags); } }; @@ -785,15 +786,15 @@ template <> struct MDNodeKeyImpl { : Name(Name), File(File), Line(Line), GetterName(GetterName), SetterName(SetterName), Attributes(Attributes), Type(Type) {} MDNodeKeyImpl(const MDObjCProperty *N) - : Name(N->getName()), File(N->getFile()), Line(N->getLine()), + : Name(N->getName()), File(N->getRawFile()), Line(N->getLine()), GetterName(N->getGetterName()), SetterName(N->getSetterName()), - Attributes(N->getAttributes()), Type(N->getType()) {} + Attributes(N->getAttributes()), Type(N->getRawType()) {} bool isKeyOf(const MDObjCProperty *RHS) const { - return Name == RHS->getName() && File == RHS->getFile() && + return Name == RHS->getName() && File == RHS->getRawFile() && Line == RHS->getLine() && GetterName == RHS->getGetterName() && SetterName == RHS->getSetterName() && - Attributes == RHS->getAttributes() && Type == RHS->getType(); + Attributes == RHS->getAttributes() && Type == RHS->getRawType(); } unsigned getHashValue() const { return hash_combine(Name, File, Line, GetterName, SetterName, Attributes, @@ -812,12 +813,12 @@ template <> struct MDNodeKeyImpl { StringRef Name) : Tag(Tag), Scope(Scope), Entity(Entity), Line(Line), Name(Name) {} MDNodeKeyImpl(const MDImportedEntity *N) - : Tag(N->getTag()), Scope(N->getScope()), Entity(N->getEntity()), + : Tag(N->getTag()), Scope(N->getRawScope()), Entity(N->getRawEntity()), Line(N->getLine()), Name(N->getName()) {} bool isKeyOf(const MDImportedEntity *RHS) const { - return Tag == RHS->getTag() && Scope == RHS->getScope() && - Entity == RHS->getEntity() && Line == RHS->getLine() && + return Tag == RHS->getTag() && Scope == RHS->getRawScope() && + Entity == RHS->getRawEntity() && Line == RHS->getLine() && Name == RHS->getName(); } unsigned getHashValue() const { diff --git a/lib/IR/LegacyPassManager.cpp b/lib/IR/LegacyPassManager.cpp index 9a365d1..6870032 100644 --- a/lib/IR/LegacyPassManager.cpp +++ b/lib/IR/LegacyPassManager.cpp @@ -293,7 +293,7 @@ public: Pass(PT_PassManager, ID), PMDataManager() { } // Delete on the fly managers. - virtual ~MPPassManager() { + ~MPPassManager() override { for (std::map::iterator I = OnTheFlyManagers.begin(), E = OnTheFlyManagers.end(); I != E; ++I) { diff --git a/lib/IR/Metadata.cpp b/lib/IR/Metadata.cpp index 0ad3c5c..93098b9 100644 --- a/lib/IR/Metadata.cpp +++ b/lib/IR/Metadata.cpp @@ -446,6 +446,10 @@ void MDNode::makeUniqued() { assert(isTemporary() && "Expected this to be temporary"); assert(!isResolved() && "Expected this to be unresolved"); + // Enable uniquing callbacks. + for (auto &Op : mutable_operands()) + Op.reset(Op.get(), this); + // Make this 'uniqued'. Storage = Uniqued; if (!countUnresolvedOperands()) @@ -1035,7 +1039,7 @@ void Instruction::setMetadata(unsigned KindID, MDNode *Node) { // Handle 'dbg' as a special case since it is not stored in the hash table. if (KindID == LLVMContext::MD_dbg) { - DbgLoc = DebugLoc::getFromDILocation(Node); + DbgLoc = DebugLoc(Node); return; } @@ -1114,7 +1118,7 @@ void Instruction::getAllMetadataImpl( Result.clear(); // Handle 'dbg' as a special case since it is not stored in the hash table. - if (!DbgLoc.isUnknown()) { + if (DbgLoc) { Result.push_back( std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode())); if (!hasMetadataHashEntry()) return; diff --git a/lib/IR/UseListOrder.cpp b/lib/IR/UseListOrder.cpp deleted file mode 100644 index d064e67..0000000 --- a/lib/IR/UseListOrder.cpp +++ /dev/null @@ -1,43 +0,0 @@ -//===- UseListOrder.cpp - Implement Use List Order ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Implement structures and command-line options for preserving use-list order. -// -//===----------------------------------------------------------------------===// - -#include "llvm/IR/UseListOrder.h" -#include "llvm/Support/CommandLine.h" - -using namespace llvm; - -static cl::opt PreserveBitcodeUseListOrder( - "preserve-bc-use-list-order", - cl::desc("Experimental support to preserve bitcode use-list order."), - cl::init(false), cl::Hidden); - -static cl::opt PreserveAssemblyUseListOrder( - "preserve-ll-use-list-order", - cl::desc("Experimental support to preserve assembly use-list order."), - cl::init(false), cl::Hidden); - -bool llvm::shouldPreserveBitcodeUseListOrder() { - return PreserveBitcodeUseListOrder; -} - -bool llvm::shouldPreserveAssemblyUseListOrder() { - return PreserveAssemblyUseListOrder; -} - -void llvm::setPreserveBitcodeUseListOrder(bool ShouldPreserve) { - PreserveBitcodeUseListOrder = ShouldPreserve; -} - -void llvm::setPreserveAssemblyUseListOrder(bool ShouldPreserve) { - PreserveAssemblyUseListOrder = ShouldPreserve; -} diff --git a/lib/IR/Value.cpp b/lib/IR/Value.cpp index 78bfca4..f6eb427 100644 --- a/lib/IR/Value.cpp +++ b/lib/IR/Value.cpp @@ -525,7 +525,7 @@ static bool isDereferenceablePointer(const Value *V, const DataLayout &DL, // Return values from call sites specifically marked as dereferenceable are // also okay. - if (ImmutableCallSite CS = V) { + if (auto CS = ImmutableCallSite(V)) { if (uint64_t Bytes = CS.getDereferenceableBytes(0)) { Type *Ty = V->getType()->getPointerElementType(); if (Ty->isSized() && DL.getTypeStoreSize(Ty) <= Bytes) @@ -595,7 +595,7 @@ bool Value::isDereferenceablePointer(const DataLayout &DL) const { APInt DerefBytes(Offset.getBitWidth(), 0); if (const Argument *A = dyn_cast(BV)) DerefBytes = A->getDereferenceableBytes(); - else if (ImmutableCallSite CS = BV) + else if (auto CS = ImmutableCallSite(BV)) DerefBytes = CS.getDereferenceableBytes(0); if (DerefBytes.getBoolValue() && Offset.isNonNegative()) { diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index fcf48c4..fba78e9 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -87,10 +87,9 @@ struct VerifierSupport { /// \brief Track the brokenness of the module while recursively visiting. bool Broken; - bool EverBroken; explicit VerifierSupport(raw_ostream &OS) - : OS(OS), M(nullptr), Broken(false), EverBroken(false) {} + : OS(OS), M(nullptr), Broken(false) {} private: void Write(const Value *V) { @@ -111,6 +110,10 @@ private: OS << '\n'; } + template void Write(const MDTupleTypedArrayWrapper &MD) { + Write(MD.get()); + } + void Write(const NamedMDNode *NMD) { if (!NMD) return; @@ -145,7 +148,7 @@ public: /// something is not correct. void CheckFailed(const Twine &Message) { OS << Message << '\n'; - EverBroken = Broken = true; + Broken = true; } /// \brief A check failed (with values to print). @@ -175,6 +178,9 @@ class Verifier : public InstVisitor, VerifierSupport { /// \brief Keep track of the metadata nodes that have been checked already. SmallPtrSet MDNodes; + /// \brief Track unresolved string-based type references. + SmallDenseMap UnresolvedTypeRefs; + /// \brief The personality function referenced by the LandingPadInsts. /// All LandingPadInsts within the same function must use the same /// personality function. @@ -268,8 +274,8 @@ public: visitModuleFlags(M); visitModuleIdents(M); - // Verify debug info last. - verifyDebugInfo(); + // Verify type referneces last. + verifyTypeRefs(); return !Broken; } @@ -296,8 +302,37 @@ private: void visitBasicBlock(BasicBlock &BB); void visitRangeMetadata(Instruction& I, MDNode* Range, Type* Ty); + template bool isValidMetadataArray(const MDTuple &N); #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N); #include "llvm/IR/Metadata.def" + void visitMDScope(const MDScope &N); + void visitMDDerivedTypeBase(const MDDerivedTypeBase &N); + void visitMDVariable(const MDVariable &N); + void visitMDLexicalBlockBase(const MDLexicalBlockBase &N); + void visitMDTemplateParameter(const MDTemplateParameter &N); + + void visitTemplateParams(const MDNode &N, const Metadata &RawParams); + + /// \brief Check for a valid string-based type reference. + /// + /// Checks if \c MD is a string-based type reference. If it is, keeps track + /// of it (and its user, \c N) for error messages later. + bool isValidUUID(const MDNode &N, const Metadata *MD); + + /// \brief Check for a valid type reference. + /// + /// Checks for subclasses of \a MDType, or \a isValidUUID(). + bool isTypeRef(const MDNode &N, const Metadata *MD); + + /// \brief Check for a valid scope reference. + /// + /// Checks for subclasses of \a MDScope, or \a isValidUUID(). + bool isScopeRef(const MDNode &N, const Metadata *MD); + + /// \brief Check for a valid debug info reference. + /// + /// Checks for subclasses of \a DebugNode, or \a isValidUUID(). + bool isDIRef(const MDNode &N, const Metadata *MD); // InstVisitor overrides... using InstVisitor::visit; @@ -371,9 +406,11 @@ private: void verifyFrameRecoverIndices(); // Module-level debug info verification... - void verifyDebugInfo(); - void processInstructions(DebugInfoFinder &Finder); - void processCallInst(DebugInfoFinder &Finder, const CallInst &CI); + void verifyTypeRefs(); + template + void verifyBitPieceExpression(const DbgInfoIntrinsic &I, + const MapTy &TypeRefs); + void visitUnresolvedTypeRef(const MDString *S, const MDNode *N); }; } // End anonymous namespace @@ -566,13 +603,14 @@ void Verifier::visitGlobalAlias(const GlobalAlias &GA) { void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i) { MDNode *MD = NMD.getOperand(i); - if (!MD) - continue; if (NMD.getName() == "llvm.dbg.cu") { - Assert(isa(MD), "invalid compile unit", &NMD, MD); + Assert(MD && isa(MD), "invalid compile unit", &NMD, MD); } + if (!MD) + continue; + visitMDNode(*MD); } } @@ -658,6 +696,58 @@ void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { visitValueAsMetadata(*V, F); } +bool Verifier::isValidUUID(const MDNode &N, const Metadata *MD) { + auto *S = dyn_cast(MD); + if (!S) + return false; + if (S->getString().empty()) + return false; + + // Keep track of names of types referenced via UUID so we can check that they + // actually exist. + UnresolvedTypeRefs.insert(std::make_pair(S, &N)); + return true; +} + +/// \brief Check if a value can be a reference to a type. +bool Verifier::isTypeRef(const MDNode &N, const Metadata *MD) { + return !MD || isValidUUID(N, MD) || isa(MD); +} + +/// \brief Check if a value can be a ScopeRef. +bool Verifier::isScopeRef(const MDNode &N, const Metadata *MD) { + return !MD || isValidUUID(N, MD) || isa(MD); +} + +/// \brief Check if a value can be a debug info ref. +bool Verifier::isDIRef(const MDNode &N, const Metadata *MD) { + return !MD || isValidUUID(N, MD) || isa(MD); +} + +template +bool isValidMetadataArrayImpl(const MDTuple &N, bool AllowNull) { + for (Metadata *MD : N.operands()) { + if (MD) { + if (!isa(MD)) + return false; + } else { + if (!AllowNull) + return false; + } + } + return true; +} + +template +bool isValidMetadataArray(const MDTuple &N) { + return isValidMetadataArrayImpl(N, /* AllowNull */ false); +} + +template +bool isValidMetadataNullArray(const MDTuple &N) { + return isValidMetadataArrayImpl(N, /* AllowNull */ true); +} + void Verifier::visitMDLocation(const MDLocation &N) { Assert(N.getRawScope() && isa(N.getRawScope()), "location requires a valid scope", &N, N.getRawScope()); @@ -669,8 +759,14 @@ void Verifier::visitGenericDebugNode(const GenericDebugNode &N) { Assert(N.getTag(), "invalid tag", &N); } +void Verifier::visitMDScope(const MDScope &N) { + if (auto *F = N.getRawFile()) + Assert(isa(F), "invalid file", &N, F); +} + void Verifier::visitMDSubrange(const MDSubrange &N) { Assert(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); + Assert(N.getCount() >= -1, "invalid subrange count", &N); } void Verifier::visitMDEnumerator(const MDEnumerator &N) { @@ -683,7 +779,39 @@ void Verifier::visitMDBasicType(const MDBasicType &N) { "invalid tag", &N); } +void Verifier::visitMDDerivedTypeBase(const MDDerivedTypeBase &N) { + // Common scope checks. + visitMDScope(N); + + Assert(isScopeRef(N, N.getScope()), "invalid scope", &N, N.getScope()); + Assert(isTypeRef(N, N.getBaseType()), "invalid base type", &N, + N.getBaseType()); + + // FIXME: Sink this into the subclass verifies. + if (!N.getFile() || N.getFile()->getFilename().empty()) { + // Check whether the filename is allowed to be empty. + uint16_t Tag = N.getTag(); + Assert( + Tag == dwarf::DW_TAG_const_type || Tag == dwarf::DW_TAG_volatile_type || + Tag == dwarf::DW_TAG_pointer_type || + Tag == dwarf::DW_TAG_ptr_to_member_type || + Tag == dwarf::DW_TAG_reference_type || + Tag == dwarf::DW_TAG_rvalue_reference_type || + Tag == dwarf::DW_TAG_restrict_type || + Tag == dwarf::DW_TAG_array_type || + Tag == dwarf::DW_TAG_enumeration_type || + Tag == dwarf::DW_TAG_subroutine_type || + Tag == dwarf::DW_TAG_inheritance || Tag == dwarf::DW_TAG_friend || + Tag == dwarf::DW_TAG_structure_type || + Tag == dwarf::DW_TAG_member || Tag == dwarf::DW_TAG_typedef, + "derived/composite type requires a filename", &N, N.getFile()); + } +} + void Verifier::visitMDDerivedType(const MDDerivedType &N) { + // Common derived type checks. + visitMDDerivedTypeBase(N); + Assert(N.getTag() == dwarf::DW_TAG_typedef || N.getTag() == dwarf::DW_TAG_pointer_type || N.getTag() == dwarf::DW_TAG_ptr_to_member_type || @@ -696,9 +824,30 @@ void Verifier::visitMDDerivedType(const MDDerivedType &N) { N.getTag() == dwarf::DW_TAG_inheritance || N.getTag() == dwarf::DW_TAG_friend, "invalid tag", &N); + if (N.getTag() == dwarf::DW_TAG_ptr_to_member_type) { + Assert(isTypeRef(N, N.getExtraData()), "invalid pointer to member type", &N, + N.getExtraData()); + } +} + +static bool hasConflictingReferenceFlags(unsigned Flags) { + return (Flags & DebugNode::FlagLValueReference) && + (Flags & DebugNode::FlagRValueReference); +} + +void Verifier::visitTemplateParams(const MDNode &N, const Metadata &RawParams) { + auto *Params = dyn_cast(&RawParams); + Assert(Params, "invalid template params", &N, &RawParams); + for (Metadata *Op : Params->operands()) { + Assert(Op && isa(Op), "invalid template parameter", &N, + Params, Op); + } } void Verifier::visitMDCompositeType(const MDCompositeType &N) { + // Common derived type checks. + visitMDDerivedTypeBase(N); + Assert(N.getTag() == dwarf::DW_TAG_array_type || N.getTag() == dwarf::DW_TAG_structure_type || N.getTag() == dwarf::DW_TAG_union_type || @@ -706,10 +855,29 @@ void Verifier::visitMDCompositeType(const MDCompositeType &N) { N.getTag() == dwarf::DW_TAG_subroutine_type || N.getTag() == dwarf::DW_TAG_class_type, "invalid tag", &N); + + Assert(!N.getRawElements() || isa(N.getRawElements()), + "invalid composite elements", &N, N.getRawElements()); + Assert(isTypeRef(N, N.getRawVTableHolder()), "invalid vtable holder", &N, + N.getRawVTableHolder()); + Assert(!N.getRawElements() || isa(N.getRawElements()), + "invalid composite elements", &N, N.getRawElements()); + Assert(!hasConflictingReferenceFlags(N.getFlags()), "invalid reference flags", + &N); + if (auto *Params = N.getRawTemplateParams()) + visitTemplateParams(N, *Params); } void Verifier::visitMDSubroutineType(const MDSubroutineType &N) { Assert(N.getTag() == dwarf::DW_TAG_subroutine_type, "invalid tag", &N); + if (auto *Types = N.getRawTypeArray()) { + Assert(isa(Types), "invalid composite elements", &N, Types); + for (Metadata *Ty : N.getTypeArray()->operands()) { + Assert(isTypeRef(N, Ty), "invalid subroutine type ref", &N, Types, Ty); + } + } + Assert(!hasConflictingReferenceFlags(N.getFlags()), "invalid reference flags", + &N); } void Verifier::visitMDFile(const MDFile &N) { @@ -718,45 +886,195 @@ void Verifier::visitMDFile(const MDFile &N) { void Verifier::visitMDCompileUnit(const MDCompileUnit &N) { Assert(N.getTag() == dwarf::DW_TAG_compile_unit, "invalid tag", &N); + + // Don't bother verifying the compilation directory or producer string + // as those could be empty. + Assert(N.getRawFile() && isa(N.getRawFile()), + "invalid file", &N, N.getRawFile()); + Assert(!N.getFile()->getFilename().empty(), "invalid filename", &N, + N.getFile()); + + if (auto *Array = N.getRawEnumTypes()) { + Assert(isa(Array), "invalid enum list", &N, Array); + for (Metadata *Op : N.getEnumTypes()->operands()) { + auto *Enum = dyn_cast_or_null(Op); + Assert(Enum && Enum->getTag() == dwarf::DW_TAG_enumeration_type, + "invalid enum type", &N, N.getEnumTypes(), Op); + } + } + if (auto *Array = N.getRawRetainedTypes()) { + Assert(isa(Array), "invalid retained type list", &N, Array); + for (Metadata *Op : N.getRetainedTypes()->operands()) { + Assert(Op && isa(Op), "invalid retained type", &N, Op); + } + } + if (auto *Array = N.getRawSubprograms()) { + Assert(isa(Array), "invalid subprogram list", &N, Array); + for (Metadata *Op : N.getSubprograms()->operands()) { + Assert(Op && isa(Op), "invalid subprogram ref", &N, Op); + } + } + if (auto *Array = N.getRawGlobalVariables()) { + Assert(isa(Array), "invalid global variable list", &N, Array); + for (Metadata *Op : N.getGlobalVariables()->operands()) { + Assert(Op && isa(Op), "invalid global variable ref", &N, + Op); + } + } + if (auto *Array = N.getRawImportedEntities()) { + Assert(isa(Array), "invalid imported entity list", &N, Array); + for (Metadata *Op : N.getImportedEntities()->operands()) { + Assert(Op && isa(Op), "invalid imported entity ref", &N, + Op); + } + } } void Verifier::visitMDSubprogram(const MDSubprogram &N) { Assert(N.getTag() == dwarf::DW_TAG_subprogram, "invalid tag", &N); + Assert(isScopeRef(N, N.getRawScope()), "invalid scope", &N, N.getRawScope()); + if (auto *T = N.getRawType()) + Assert(isa(T), "invalid subroutine type", &N, T); + Assert(isTypeRef(N, N.getRawContainingType()), "invalid containing type", &N, + N.getRawContainingType()); + if (auto *RawF = N.getRawFunction()) { + auto *FMD = dyn_cast(RawF); + auto *F = FMD ? FMD->getValue() : nullptr; + auto *FT = F ? dyn_cast(F->getType()) : nullptr; + Assert(F && FT && isa(FT->getElementType()), + "invalid function", &N, F, FT); + } + if (auto *Params = N.getRawTemplateParams()) + visitTemplateParams(N, *Params); + if (auto *S = N.getRawDeclaration()) { + Assert(isa(S) && !cast(S)->isDefinition(), + "invalid subprogram declaration", &N, S); + } + if (auto *RawVars = N.getRawVariables()) { + auto *Vars = dyn_cast(RawVars); + Assert(Vars, "invalid variable list", &N, RawVars); + for (Metadata *Op : Vars->operands()) { + Assert(Op && isa(Op), "invalid local variable", &N, Vars, + Op); + } + } + Assert(!hasConflictingReferenceFlags(N.getFlags()), "invalid reference flags", + &N); + + auto *F = N.getFunction(); + if (!F) + return; + + // Check that all !dbg attachments lead to back to N (or, at least, another + // subprogram that describes the same function). + // + // FIXME: Check this incrementally while visiting !dbg attachments. + // FIXME: Only check when N is the canonical subprogram for F. + SmallPtrSet Seen; + for (auto &BB : *F) + for (auto &I : BB) { + // Be careful about using MDLocation here since we might be dealing with + // broken code (this is the Verifier after all). + MDLocation *DL = + dyn_cast_or_null(I.getDebugLoc().getAsMDNode()); + if (!DL) + continue; + if (!Seen.insert(DL).second) + continue; + + MDLocalScope *Scope = DL->getInlinedAtScope(); + if (Scope && !Seen.insert(Scope).second) + continue; + + MDSubprogram *SP = Scope ? Scope->getSubprogram() : nullptr; + if (SP && !Seen.insert(SP).second) + continue; + + // FIXME: Once N is canonical, check "SP == &N". + Assert(SP->describes(F), + "!dbg attachment points at wrong subprogram for function", &N, F, + &I, DL, Scope, SP); + } } -void Verifier::visitMDLexicalBlock(const MDLexicalBlock &N) { +void Verifier::visitMDLexicalBlockBase(const MDLexicalBlockBase &N) { Assert(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); + Assert(N.getRawScope() && isa(N.getRawScope()), + "invalid local scope", &N, N.getRawScope()); +} + +void Verifier::visitMDLexicalBlock(const MDLexicalBlock &N) { + visitMDLexicalBlockBase(N); + + Assert(N.getLine() || !N.getColumn(), + "cannot have column info without line info", &N); } void Verifier::visitMDLexicalBlockFile(const MDLexicalBlockFile &N) { - Assert(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); + visitMDLexicalBlockBase(N); } void Verifier::visitMDNamespace(const MDNamespace &N) { Assert(N.getTag() == dwarf::DW_TAG_namespace, "invalid tag", &N); + if (auto *S = N.getRawScope()) + Assert(isa(S), "invalid scope ref", &N, S); +} + +void Verifier::visitMDTemplateParameter(const MDTemplateParameter &N) { + Assert(isTypeRef(N, N.getType()), "invalid type ref", &N, N.getType()); } void Verifier::visitMDTemplateTypeParameter(const MDTemplateTypeParameter &N) { + visitMDTemplateParameter(N); + Assert(N.getTag() == dwarf::DW_TAG_template_type_parameter, "invalid tag", &N); } void Verifier::visitMDTemplateValueParameter( const MDTemplateValueParameter &N) { + visitMDTemplateParameter(N); + Assert(N.getTag() == dwarf::DW_TAG_template_value_parameter || N.getTag() == dwarf::DW_TAG_GNU_template_template_param || N.getTag() == dwarf::DW_TAG_GNU_template_parameter_pack, "invalid tag", &N); } +void Verifier::visitMDVariable(const MDVariable &N) { + if (auto *S = N.getRawScope()) + Assert(isa(S), "invalid scope", &N, S); + Assert(isTypeRef(N, N.getRawType()), "invalid type ref", &N, N.getRawType()); + if (auto *F = N.getRawFile()) + Assert(isa(F), "invalid file", &N, F); +} + void Verifier::visitMDGlobalVariable(const MDGlobalVariable &N) { + // Checks common to all variables. + visitMDVariable(N); + Assert(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); + Assert(!N.getName().empty(), "missing global variable name", &N); + if (auto *V = N.getRawVariable()) { + Assert(isa(V) && + !isa(cast(V)->getValue()), + "invalid global varaible ref", &N, V); + } + if (auto *Member = N.getRawStaticDataMemberDeclaration()) { + Assert(isa(Member), "invalid static data member declaration", + &N, Member); + } } void Verifier::visitMDLocalVariable(const MDLocalVariable &N) { + // Checks common to all variables. + visitMDVariable(N); + Assert(N.getTag() == dwarf::DW_TAG_auto_variable || N.getTag() == dwarf::DW_TAG_arg_variable, "invalid tag", &N); + Assert(N.getRawScope() && isa(N.getRawScope()), + "local variable requires a valid scope", &N, N.getRawScope()); } void Verifier::visitMDExpression(const MDExpression &N) { @@ -765,12 +1083,20 @@ void Verifier::visitMDExpression(const MDExpression &N) { void Verifier::visitMDObjCProperty(const MDObjCProperty &N) { Assert(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N); + if (auto *T = N.getRawType()) + Assert(isa(T), "invalid type ref", &N, T); + if (auto *F = N.getRawFile()) + Assert(isa(F), "invalid file", &N, F); } void Verifier::visitMDImportedEntity(const MDImportedEntity &N) { Assert(N.getTag() == dwarf::DW_TAG_imported_module || N.getTag() == dwarf::DW_TAG_imported_declaration, "invalid tag", &N); + if (auto *S = N.getRawScope()) + Assert(isa(S), "invalid scope for imported entity", &N, S); + Assert(isDIRef(N, N.getEntity()), "invalid imported entity", &N, + N.getEntity()); } void Verifier::visitComdat(const Comdat &C) { @@ -2133,7 +2459,7 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { SmallVector Idxs(GEP.idx_begin(), GEP.idx_end()); Type *ElTy = - GetElementPtrInst::getIndexedType(GEP.getPointerOperandType(), Idxs); + GetElementPtrInst::getIndexedType(GEP.getSourceElementType(), Idxs); Assert(ElTy, "Invalid indices for GEP pointer type!", &GEP); Assert(GEP.getType()->getScalarType()->isPointerTy() && @@ -2214,9 +2540,7 @@ void Verifier::visitRangeMetadata(Instruction& I, void Verifier::visitLoadInst(LoadInst &LI) { PointerType *PTy = dyn_cast(LI.getOperand(0)->getType()); Assert(PTy, "Load operand must be a pointer.", &LI); - Type *ElTy = PTy->getElementType(); - Assert(ElTy == LI.getType(), - "Load result type does not match pointer operand type!", &LI, ElTy); + Type *ElTy = LI.getType(); Assert(LI.getAlignment() <= Value::MaximumAlignment, "huge alignment values are unsupported", &LI); if (LI.isAtomic()) { @@ -2885,6 +3209,8 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { Assert(!SawFrameEscape, "multiple calls to llvm.frameescape in one function", &CI); for (Value *Arg : CI.arg_operands()) { + if (isa(Arg)) + continue; // Null values are allowed as placeholders. auto *AI = dyn_cast(Arg->stripPointerCasts()); Assert(AI && AI->isStaticAlloca(), "llvm.frameescape only accepts static allocas", &CI); @@ -2909,16 +3235,11 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { break; } - case Intrinsic::eh_unwindhelp: { - auto *AI = dyn_cast(CI.getArgOperand(0)->stripPointerCasts()); - Assert(AI && AI->isStaticAlloca(), - "llvm.eh.unwindhelp requires a static alloca", &CI); - break; - } - case Intrinsic::experimental_gc_statepoint: Assert(!CI.isInlineAsm(), "gc.statepoint support for inline assembly unimplemented", &CI); + Assert(CI.getParent()->getParent()->hasGC(), + "Enclosing function does not use GC.", &CI); VerifyStatepoint(ImmutableCallSite(&CI)); break; @@ -2926,6 +3247,8 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { case Intrinsic::experimental_gc_result_float: case Intrinsic::experimental_gc_result_ptr: case Intrinsic::experimental_gc_result: { + Assert(CI.getParent()->getParent()->hasGC(), + "Enclosing function does not use GC.", &CI); // Are we tied to a statepoint properly? CallSite StatepointCS(CI.getArgOperand(0)); const Function *StatepointFn = @@ -3035,6 +3358,25 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) { }; } +/// \brief Carefully grab the subprogram from a local scope. +/// +/// This carefully grabs the subprogram from a local scope, avoiding the +/// built-in assertions that would typically fire. +static MDSubprogram *getSubprogram(Metadata *LocalScope) { + if (!LocalScope) + return nullptr; + + if (auto *SP = dyn_cast(LocalScope)) + return SP; + + if (auto *LB = dyn_cast(LocalScope)) + return getSubprogram(LB->getRawScope()); + + // Just return null; broken scope chains are checked elsewhere. + assert(!isa(LocalScope) && "Unknown type of local scope"); + return nullptr; +} + template void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) { auto *MD = cast(DII.getArgOperand(0))->getMetadata(); @@ -3047,61 +3389,145 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) { Assert(isa(DII.getRawExpression()), "invalid llvm.dbg." + Kind + " intrinsic expression", &DII, DII.getRawExpression()); -} -void Verifier::verifyDebugInfo() { - // Run the debug info verifier only if the regular verifier succeeds, since - // sometimes checks that have already failed will cause crashes here. - if (EverBroken || !VerifyDebugInfo) - return; + // Ignore broken !dbg attachments; they're checked elsewhere. + if (MDNode *N = DII.getDebugLoc().getAsMDNode()) + if (!isa(N)) + return; - DebugInfoFinder Finder; - Finder.processModule(*M); - processInstructions(Finder); + BasicBlock *BB = DII.getParent(); + Function *F = BB ? BB->getParent() : nullptr; + + // The scopes for variables and !dbg attachments must agree. + MDLocalVariable *Var = DII.getVariable(); + MDLocation *Loc = DII.getDebugLoc(); + Assert(Loc, "llvm.dbg." + Kind + " intrinsic requires a !dbg attachment", + &DII, BB, F); + + MDSubprogram *VarSP = getSubprogram(Var->getRawScope()); + MDSubprogram *LocSP = getSubprogram(Loc->getRawScope()); + if (!VarSP || !LocSP) + return; // Broken scope chains are checked elsewhere. + + Assert(VarSP == LocSP, "mismatched subprogram between llvm.dbg." + Kind + + " variable and !dbg attachment", + &DII, BB, F, Var, Var->getScope()->getSubprogram(), Loc, + Loc->getScope()->getSubprogram()); +} + +template +static uint64_t getVariableSize(const MDLocalVariable &V, const MapTy &Map) { + // Be careful of broken types (checked elsewhere). + const Metadata *RawType = V.getRawType(); + while (RawType) { + // Try to get the size directly. + if (auto *T = dyn_cast(RawType)) + if (uint64_t Size = T->getSizeInBits()) + return Size; + + if (auto *DT = dyn_cast(RawType)) { + // Look at the base type. + RawType = DT->getRawBaseType(); + continue; + } - // Verify Debug Info. - // - // NOTE: The loud braces are necessary for MSVC compatibility. - for (DICompileUnit CU : Finder.compile_units()) { - Assert(CU.Verify(), "DICompileUnit does not Verify!", CU); - } - for (DISubprogram S : Finder.subprograms()) { - Assert(S.Verify(), "DISubprogram does not Verify!", S); - } - for (DIGlobalVariable GV : Finder.global_variables()) { - Assert(GV.Verify(), "DIGlobalVariable does not Verify!", GV); - } - for (DIType T : Finder.types()) { - Assert(T.Verify(), "DIType does not Verify!", T); + if (auto *S = dyn_cast(RawType)) { + // Don't error on missing types (checked elsewhere). + RawType = Map.lookup(S); + continue; + } + + // Missing type or size. + break; } - for (DIScope S : Finder.scopes()) { - Assert(S.Verify(), "DIScope does not Verify!", S); + + // Fail gracefully. + return 0; +} + +template +void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I, + const MapTy &TypeRefs) { + MDLocalVariable *V; + MDExpression *E; + if (auto *DVI = dyn_cast(&I)) { + V = dyn_cast_or_null(DVI->getRawVariable()); + E = dyn_cast_or_null(DVI->getRawExpression()); + } else { + auto *DDI = cast(&I); + V = dyn_cast_or_null(DDI->getRawVariable()); + E = dyn_cast_or_null(DDI->getRawExpression()); } + + // We don't know whether this intrinsic verified correctly. + if (!V || !E || !E->isValid()) + return; + + // Nothing to do if this isn't a bit piece expression. + if (!E->isBitPiece()) + return; + + // If there's no size, the type is broken, but that should be checked + // elsewhere. + uint64_t VarSize = getVariableSize(*V, TypeRefs); + if (!VarSize) + return; + + unsigned PieceSize = E->getBitPieceSize(); + unsigned PieceOffset = E->getBitPieceOffset(); + Assert(PieceSize + PieceOffset <= VarSize, + "piece is larger than or outside of variable", &I, V, E); + Assert(PieceSize != VarSize, "piece covers entire variable", &I, V, E); } -void Verifier::processInstructions(DebugInfoFinder &Finder) { - for (const Function &F : *M) - for (auto I = inst_begin(&F), E = inst_end(&F); I != E; ++I) { - if (MDNode *MD = I->getMetadata(LLVMContext::MD_dbg)) - Finder.processLocation(*M, DILocation(MD)); - if (const CallInst *CI = dyn_cast(&*I)) - processCallInst(Finder, *CI); - } +void Verifier::visitUnresolvedTypeRef(const MDString *S, const MDNode *N) { + // This is in its own function so we get an error for each bad type ref (not + // just the first). + Assert(false, "unresolved type ref", S, N); } -void Verifier::processCallInst(DebugInfoFinder &Finder, const CallInst &CI) { - if (Function *F = CI.getCalledFunction()) - if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID()) - switch (ID) { - case Intrinsic::dbg_declare: - Finder.processDeclare(*M, cast(&CI)); - break; - case Intrinsic::dbg_value: - Finder.processValue(*M, cast(&CI)); - break; - default: - break; - } +void Verifier::verifyTypeRefs() { + auto *CUs = M->getNamedMetadata("llvm.dbg.cu"); + if (!CUs) + return; + + // Visit all the compile units again to map the type references. + SmallDenseMap TypeRefs; + for (auto *CU : CUs->operands()) + if (auto Ts = cast(CU)->getRetainedTypes()) + for (MDType *Op : Ts) + if (auto *T = dyn_cast(Op)) + if (auto *S = T->getRawIdentifier()) { + UnresolvedTypeRefs.erase(S); + TypeRefs.insert(std::make_pair(S, T)); + } + + // Verify debug info intrinsic bit piece expressions. This needs a second + // pass through the intructions, since we haven't built TypeRefs yet when + // verifying functions, and simply queuing the DbgInfoIntrinsics to evaluate + // later/now would queue up some that could be later deleted. + for (const Function &F : *M) + for (const BasicBlock &BB : F) + for (const Instruction &I : BB) + if (auto *DII = dyn_cast(&I)) + verifyBitPieceExpression(*DII, TypeRefs); + + // Return early if all typerefs were resolved. + if (UnresolvedTypeRefs.empty()) + return; + + // Sort the unresolved references by name so the output is deterministic. + typedef std::pair TypeRef; + SmallVector Unresolved(UnresolvedTypeRefs.begin(), + UnresolvedTypeRefs.end()); + std::sort(Unresolved.begin(), Unresolved.end(), + [](const TypeRef &LHS, const TypeRef &RHS) { + return LHS.first->getString() < RHS.first->getString(); + }); + + // Visit the unresolved refs (printing out the errors). + for (const TypeRef &TR : Unresolved) + visitUnresolvedTypeRef(TR.first, TR.second); } //===----------------------------------------------------------------------===// diff --git a/lib/LTO/LTOCodeGenerator.cpp b/lib/LTO/LTOCodeGenerator.cpp index a6f980b..b4a7011 100644 --- a/lib/LTO/LTOCodeGenerator.cpp +++ b/lib/LTO/LTOCodeGenerator.cpp @@ -38,7 +38,6 @@ #include "llvm/MC/SubtargetFeature.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Signals.h" @@ -215,7 +214,8 @@ bool LTOCodeGenerator::writeMergedModules(const char *path, } // write bitcode to it - WriteBitcodeToFile(IRLinker.getModule(), Out.os()); + WriteBitcodeToFile(IRLinker.getModule(), Out.os(), + /* ShouldPreserveUseListOrder */ true); Out.os().close(); if (Out.os().has_error()) { @@ -566,24 +566,20 @@ bool LTOCodeGenerator::optimize(bool DisableInline, return true; } -bool LTOCodeGenerator::compileOptimized(raw_ostream &out, std::string &errMsg) { +bool LTOCodeGenerator::compileOptimized(raw_pwrite_stream &out, + std::string &errMsg) { if (!this->determineTarget(errMsg)) return false; Module *mergedModule = IRLinker.getModule(); - // Mark which symbols can not be internalized - this->applyScopeRestrictions(); - legacy::PassManager codeGenPasses; - formatted_raw_ostream Out(out); - // If the bitcode files contain ARC code and were compiled with optimization, // the ObjCARCContractPass must be run, so do it unconditionally here. codeGenPasses.add(createObjCARCContractPass()); - if (TargetMach->addPassesToEmitFile(codeGenPasses, Out, + if (TargetMach->addPassesToEmitFile(codeGenPasses, out, TargetMachine::CGFT_ObjectFile)) { errMsg = "target file type not supported"; return false; diff --git a/lib/LTO/LTOModule.cpp b/lib/LTO/LTOModule.cpp index 49aa97d..5cdbca6 100644 --- a/lib/LTO/LTOModule.cpp +++ b/lib/LTO/LTOModule.cpp @@ -267,7 +267,7 @@ LTOModule::objcClassNameFromExpression(const Constant *c, std::string &name) { Constant *cn = gvn->getInitializer(); if (ConstantDataArray *ca = dyn_cast(cn)) { if (ca->isCString()) { - name = ".objc_class_name_" + ca->getAsCString().str(); + name = (".objc_class_name_" + ca->getAsCString()).str(); return true; } } diff --git a/lib/Linker/LinkModules.cpp b/lib/Linker/LinkModules.cpp index 21edc50..03ab9fb 100644 --- a/lib/Linker/LinkModules.cpp +++ b/lib/Linker/LinkModules.cpp @@ -1269,15 +1269,11 @@ void ModuleLinker::stripReplacedSubprograms() { if (!CompileUnits) return; for (unsigned I = 0, E = CompileUnits->getNumOperands(); I != E; ++I) { - DICompileUnit CU(CompileUnits->getOperand(I)); + DICompileUnit CU = cast(CompileUnits->getOperand(I)); assert(CU && "Expected valid compile unit"); - DITypedArray SPs(CU.getSubprograms()); - assert(SPs && "Expected valid subprogram array"); - - for (unsigned S = 0, SE = SPs.getNumElements(); S != SE; ++S) { - DISubprogram SP = SPs.getElement(S); - if (!SP || !SP.getFunction() || !Functions.count(SP.getFunction())) + for (MDSubprogram *SP : CU->getSubprograms()) { + if (!SP || !SP->getFunction() || !Functions.count(SP->getFunction())) continue; // Prevent DebugInfoFinder from tagging this as the canonical subprogram, diff --git a/lib/MC/ELFObjectWriter.cpp b/lib/MC/ELFObjectWriter.cpp index c99a3ee..8cb01c4 100644 --- a/lib/MC/ELFObjectWriter.cpp +++ b/lib/MC/ELFObjectWriter.cpp @@ -79,17 +79,6 @@ public: uint8_t other, uint32_t shndx, bool Reserved); }; -struct ELFRelocationEntry { - uint64_t Offset; // Where is the relocation. - const MCSymbol *Symbol; // The symbol to relocate with. - unsigned Type; // The type of the relocation. - uint64_t Addend; // The addend to use. - - ELFRelocationEntry(uint64_t Offset, const MCSymbol *Symbol, unsigned Type, - uint64_t Addend) - : Offset(Offset), Symbol(Symbol), Type(Type), Addend(Addend) {} -}; - class ELFObjectWriter : public MCObjectWriter { FragmentWriter FWriter; @@ -103,22 +92,13 @@ class ELFObjectWriter : public MCObjectWriter { static bool isLocal(const MCSymbolData &Data, bool isUsedInReloc); static bool IsELFMetaDataSection(const MCSectionData &SD); static uint64_t DataSectionSize(const MCSectionData &SD); - static uint64_t GetSectionFileSize(const MCAsmLayout &Layout, - const MCSectionData &SD); static uint64_t GetSectionAddressSize(const MCAsmLayout &Layout, const MCSectionData &SD); - void WriteDataSectionData(MCAssembler &Asm, - const MCAsmLayout &Layout, - const MCSectionELF &Section); - - /*static bool isFixupKindX86RIPRel(unsigned Kind) { - return Kind == X86::reloc_riprel_4byte || - Kind == X86::reloc_riprel_4byte_movq_load; - }*/ + void writeDataSectionData(MCAssembler &Asm, const MCAsmLayout &Layout, + const MCSectionData &SD); - /// ELFSymbolData - Helper struct for containing some precomputed - /// information on symbols. + /// Helper struct for containing some precomputed information on symbols. struct ELFSymbolData { MCSymbolData *SymbolData; uint64_t StringIndex; @@ -185,7 +165,7 @@ class ELFObjectWriter : public MCObjectWriter { } public: - ELFObjectWriter(MCELFObjectTargetWriter *MOTW, raw_ostream &OS, + ELFObjectWriter(MCELFObjectTargetWriter *MOTW, raw_pwrite_stream &OS, bool IsLittleEndian) : MCObjectWriter(OS, IsLittleEndian), FWriter(IsLittleEndian), TargetObjectWriter(MOTW), NeedsGOT(false) {} @@ -204,7 +184,7 @@ class ELFObjectWriter : public MCObjectWriter { MCObjectWriter::reset(); } - virtual ~ELFObjectWriter(); + ~ELFObjectWriter() override; void WriteWord(uint64_t W) { if (is64Bit()) @@ -218,7 +198,6 @@ class ELFObjectWriter : public MCObjectWriter { } void WriteHeader(const MCAssembler &Asm, - uint64_t SectionDataSize, unsigned NumberOfSections); void WriteSymbol(SymbolTableWriter &Writer, ELFSymbolData &MSD, @@ -245,8 +224,6 @@ class ELFObjectWriter : public MCObjectWriter { typedef DenseMap GroupMapTy; // Map from a signature symbol to the group section typedef DenseMap RevGroupMapTy; - // Map from a section to the section with the relocations - typedef DenseMap RelMapTy; // Map from a section to its offset typedef DenseMap SectionOffsetMapTy; @@ -255,23 +232,18 @@ class ELFObjectWriter : public MCObjectWriter { /// \param Asm - The assembler. /// \param SectionIndexMap - Maps a section to its index. /// \param RevGroupMap - Maps a signature symbol to the group section. - /// \param NumRegularSections - Number of non-relocation sections. void computeSymbolTable(MCAssembler &Asm, const MCAsmLayout &Layout, const SectionIndexMapTy &SectionIndexMap, - const RevGroupMapTy &RevGroupMap, - unsigned NumRegularSections); + const RevGroupMapTy &RevGroupMap); - void computeIndexMap(MCAssembler &Asm, - SectionIndexMapTy &SectionIndexMap, - RelMapTy &RelMap); + void computeIndexMap(MCAssembler &Asm, SectionIndexMapTy &SectionIndexMap); MCSectionData *createRelocationSection(MCAssembler &Asm, const MCSectionData &SD); void CompressDebugSections(MCAssembler &Asm, MCAsmLayout &Layout); - void WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout, - const RelMapTy &RelMap); + void WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout); void CreateMetadataSections(MCAssembler &Asm, MCAsmLayout &Layout, SectionIndexMapTy &SectionIndexMap); @@ -279,23 +251,18 @@ class ELFObjectWriter : public MCObjectWriter { // Create the sections that show up in the symbol table. Currently // those are the .note.GNU-stack section and the group sections. void createIndexedSections(MCAssembler &Asm, MCAsmLayout &Layout, - GroupMapTy &GroupMap, - RevGroupMapTy &RevGroupMap, - SectionIndexMapTy &SectionIndexMap, - RelMapTy &RelMap); + GroupMapTy &GroupMap, RevGroupMapTy &RevGroupMap, + SectionIndexMapTy &SectionIndexMap); void ExecutePostLayoutBinding(MCAssembler &Asm, const MCAsmLayout &Layout) override; - void writeSectionHeader(MCAssembler &Asm, const GroupMapTy &GroupMap, + void writeSectionHeader(ArrayRef Sections, + MCAssembler &Asm, const GroupMapTy &GroupMap, const MCAsmLayout &Layout, const SectionIndexMapTy &SectionIndexMap, - const RelMapTy &RelMap, const SectionOffsetMapTy &SectionOffsetMap); - void ComputeSectionOrder(MCAssembler &Asm, - std::vector &Sections); - void WriteSecHdrEntry(uint32_t Name, uint32_t Type, uint64_t Flags, uint64_t Address, uint64_t Offset, uint64_t Size, uint32_t Link, uint32_t Info, @@ -308,6 +275,7 @@ class ELFObjectWriter : public MCObjectWriter { bool IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, const MCSymbolData &DataA, + const MCSymbolData *DataB, const MCFragment &FB, bool InSet, bool IsPCRel) const override; @@ -317,7 +285,6 @@ class ELFObjectWriter : public MCObjectWriter { void WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) override; void writeSection(MCAssembler &Asm, const SectionIndexMapTy &SectionIndexMap, - const RelMapTy &RelMap, uint32_t GroupSymbolIndex, uint64_t Offset, uint64_t Size, uint64_t Alignment, const MCSectionELF &Section); @@ -384,8 +351,6 @@ void SymbolTableWriter::writeSymbol(uint32_t name, uint8_t info, uint64_t value, uint16_t Index = LargeIndex ? uint16_t(ELF::SHN_XINDEX) : shndx; - raw_svector_ostream OS(SymtabF->getContents()); - if (Is64Bit) { write(*SymtabF, name); // st_name write(*SymtabF, info); // st_info @@ -438,7 +403,6 @@ ELFObjectWriter::~ELFObjectWriter() // Emit the ELF header. void ELFObjectWriter::WriteHeader(const MCAssembler &Asm, - uint64_t SectionDataSize, unsigned NumberOfSections) { // ELF Header // ---------- @@ -472,8 +436,7 @@ void ELFObjectWriter::WriteHeader(const MCAssembler &Asm, Write32(ELF::EV_CURRENT); // e_version WriteWord(0); // e_entry, no entry point in .o file WriteWord(0); // e_phoff, no program header for .o - WriteWord(SectionDataSize + (is64Bit() ? sizeof(ELF::Elf64_Ehdr) : - sizeof(ELF::Elf32_Ehdr))); // e_shoff = sec hdr table off in bytes + WriteWord(0); // e_shoff = sec hdr table off in bytes // e_flags = whatever the target wants Write32(Asm.getELFHeaderEFlags()); @@ -628,7 +591,7 @@ void ELFObjectWriter::WriteSymbol(SymbolTableWriter &Writer, ELFSymbolData &MSD, if (ESize) { int64_t Res; - if (!ESize->EvaluateAsAbsolute(Res, Layout)) + if (!ESize->evaluateKnownAbsolute(Res, Layout)) report_fatal_error("Size expression must be absolute."); Size = Res; } @@ -969,8 +932,7 @@ bool ELFObjectWriter::isLocal(const MCSymbolData &Data, bool isUsedInReloc) { } void ELFObjectWriter::computeIndexMap(MCAssembler &Asm, - SectionIndexMapTy &SectionIndexMap, - RelMapTy &RelMap) { + SectionIndexMapTy &SectionIndexMap) { unsigned Index = 1; for (MCAssembler::iterator it = Asm.begin(), ie = Asm.end(); it != ie; ++it) { @@ -994,17 +956,15 @@ void ELFObjectWriter::computeIndexMap(MCAssembler &Asm, if (MCSectionData *RelSD = createRelocationSection(Asm, SD)) { const MCSectionELF *RelSection = static_cast(&RelSD->getSection()); - RelMap[RelSection] = &Section; SectionIndexMap[RelSection] = Index++; } } } -void -ELFObjectWriter::computeSymbolTable(MCAssembler &Asm, const MCAsmLayout &Layout, - const SectionIndexMapTy &SectionIndexMap, - const RevGroupMapTy &RevGroupMap, - unsigned NumRegularSections) { +void ELFObjectWriter::computeSymbolTable( + MCAssembler &Asm, const MCAsmLayout &Layout, + const SectionIndexMapTy &SectionIndexMap, + const RevGroupMapTy &RevGroupMap) { // FIXME: Is this the correct place to do this? // FIXME: Why is an undefined reference to _GLOBAL_OFFSET_TABLE_ needed? if (NeedsGOT) { @@ -1167,15 +1127,12 @@ ELFObjectWriter::createRelocationSection(MCAssembler &Asm, EntrySize = is64Bit() ? sizeof(ELF::Elf64_Rel) : sizeof(ELF::Elf32_Rel); unsigned Flags = 0; - StringRef Group = ""; - if (Section.getFlags() & ELF::SHF_GROUP) { + if (Section.getFlags() & ELF::SHF_GROUP) Flags = ELF::SHF_GROUP; - Group = Section.getGroup()->getName(); - } - const MCSectionELF *RelaSection = Ctx.getELFSection( + const MCSectionELF *RelaSection = Ctx.createELFRelSection( RelaSectionName, hasRelocationAddend() ? ELF::SHT_RELA : ELF::SHT_REL, - Flags, EntrySize, Group, true); + Flags, EntrySize, Section.getGroup(), &Section); return &Asm.getOrCreateSectionData(*RelaSection); } @@ -1324,8 +1281,7 @@ void ELFObjectWriter::CompressDebugSections(MCAssembler &Asm, } } -void ELFObjectWriter::WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout, - const RelMapTy &RelMap) { +void ELFObjectWriter::WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout) { for (MCAssembler::iterator it = Asm.begin(), ie = Asm.end(); it != ie; ++it) { MCSectionData &RelSD = *it; const MCSectionELF &RelSection = @@ -1335,7 +1291,7 @@ void ELFObjectWriter::WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout, if (Type != ELF::SHT_REL && Type != ELF::SHT_RELA) continue; - const MCSectionELF *Section = RelMap.lookup(&RelSection); + const MCSectionELF *Section = RelSection.getAssociatedSection(); MCSectionData &SD = Asm.getOrCreateSectionData(*Section); RelSD.setAlignment(is64Bit() ? 8 : 4); @@ -1362,31 +1318,14 @@ void ELFObjectWriter::WriteSecHdrEntry(uint32_t Name, uint32_t Type, WriteWord(EntrySize); // sh_entsize } -// ELF doesn't require relocations to be in any order. We sort by the r_offset, -// just to match gnu as for easier comparison. The use type is an arbitrary way -// of making the sort deterministic. -static int cmpRel(const ELFRelocationEntry *AP, const ELFRelocationEntry *BP) { - const ELFRelocationEntry &A = *AP; - const ELFRelocationEntry &B = *BP; - if (A.Offset != B.Offset) - return B.Offset - A.Offset; - if (B.Type != A.Type) - return A.Type - B.Type; - //llvm_unreachable("ELFRelocs might be unstable!"); - return 0; -} - -static void sortRelocs(const MCAssembler &Asm, - std::vector &Relocs) { - array_pod_sort(Relocs.begin(), Relocs.end(), cmpRel); -} - void ELFObjectWriter::WriteRelocationsFragment(const MCAssembler &Asm, MCDataFragment *F, const MCSectionData *SD) { std::vector &Relocs = Relocations[SD]; - sortRelocs(Asm, Relocs); + // Sort the relocation entries. Most targets just sort by Offset, but some + // (e.g., MIPS) have additional constraints. + TargetObjectWriter->sortRelocs(Asm, Relocs); for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { const ELFRelocationEntry &Entry = Relocs[e - i - 1]; @@ -1473,12 +1412,9 @@ void ELFObjectWriter::CreateMetadataSections( ShStrTabBuilder.data().end()); } -void ELFObjectWriter::createIndexedSections(MCAssembler &Asm, - MCAsmLayout &Layout, - GroupMapTy &GroupMap, - RevGroupMapTy &RevGroupMap, - SectionIndexMapTy &SectionIndexMap, - RelMapTy &RelMap) { +void ELFObjectWriter::createIndexedSections( + MCAssembler &Asm, MCAsmLayout &Layout, GroupMapTy &GroupMap, + RevGroupMapTy &RevGroupMap, SectionIndexMapTy &SectionIndexMap) { MCContext &Ctx = Asm.getContext(); // Build the groups @@ -1502,7 +1438,7 @@ void ELFObjectWriter::createIndexedSections(MCAssembler &Asm, GroupMap[Group] = SignatureSymbol; } - computeIndexMap(Asm, SectionIndexMap, RelMap); + computeIndexMap(Asm, SectionIndexMap); // Add sections to the groups for (MCAssembler::const_iterator it = Asm.begin(), ie = Asm.end(); @@ -1522,7 +1458,6 @@ void ELFObjectWriter::createIndexedSections(MCAssembler &Asm, void ELFObjectWriter::writeSection(MCAssembler &Asm, const SectionIndexMapTy &SectionIndexMap, - const RelMapTy &RelMap, uint32_t GroupSymbolIndex, uint64_t Offset, uint64_t Size, uint64_t Alignment, @@ -1531,16 +1466,19 @@ void ELFObjectWriter::writeSection(MCAssembler &Asm, uint64_t sh_info = 0; switch(Section.getType()) { + default: + // Nothing to do. + break; + case ELF::SHT_DYNAMIC: sh_link = ShStrTabBuilder.getOffset(Section.getSectionName()); - sh_info = 0; break; case ELF::SHT_REL: case ELF::SHT_RELA: { sh_link = SymbolTableIndex; assert(sh_link && ".symtab not found"); - const MCSectionELF *InfoSection = RelMap.find(&Section)->second; + const MCSectionELF *InfoSection = Section.getAssociatedSection(); sh_info = SectionIndexMap.lookup(InfoSection); break; } @@ -1555,45 +1493,15 @@ void ELFObjectWriter::writeSection(MCAssembler &Asm, sh_link = SymbolTableIndex; break; - case ELF::SHT_PROGBITS: - case ELF::SHT_STRTAB: - case ELF::SHT_NOBITS: - case ELF::SHT_NOTE: - case ELF::SHT_NULL: - case ELF::SHT_ARM_ATTRIBUTES: - case ELF::SHT_INIT_ARRAY: - case ELF::SHT_FINI_ARRAY: - case ELF::SHT_PREINIT_ARRAY: - case ELF::SHT_X86_64_UNWIND: - case ELF::SHT_MIPS_REGINFO: - case ELF::SHT_MIPS_OPTIONS: - case ELF::SHT_MIPS_ABIFLAGS: - // Nothing to do. - break; - case ELF::SHT_GROUP: sh_link = SymbolTableIndex; sh_info = GroupSymbolIndex; break; - - default: - llvm_unreachable("FIXME: sh_type value not supported!"); } if (TargetObjectWriter->getEMachine() == ELF::EM_ARM && - Section.getType() == ELF::SHT_ARM_EXIDX) { - StringRef SecName(Section.getSectionName()); - if (SecName == ".ARM.exidx") { - sh_link = SectionIndexMap.lookup(Asm.getContext().getELFSection( - ".text", ELF::SHT_PROGBITS, ELF::SHF_EXECINSTR | ELF::SHF_ALLOC)); - } else if (SecName.startswith(".ARM.exidx")) { - StringRef GroupName = - Section.getGroup() ? Section.getGroup()->getName() : ""; - sh_link = SectionIndexMap.lookup(Asm.getContext().getELFSection( - SecName.substr(sizeof(".ARM.exidx") - 1), ELF::SHT_PROGBITS, - ELF::SHF_EXECINSTR | ELF::SHF_ALLOC, 0, GroupName)); - } - } + Section.getType() == ELF::SHT_ARM_EXIDX) + sh_link = SectionIndexMap.lookup(Section.getAssociatedSection()); WriteSecHdrEntry(ShStrTabBuilder.getOffset(Section.getSectionName()), Section.getType(), @@ -1617,13 +1525,6 @@ uint64_t ELFObjectWriter::DataSectionSize(const MCSectionData &SD) { return Ret; } -uint64_t ELFObjectWriter::GetSectionFileSize(const MCAsmLayout &Layout, - const MCSectionData &SD) { - if (IsELFMetaDataSection(SD)) - return DataSectionSize(SD); - return Layout.getSectionFileSize(&SD); -} - uint64_t ELFObjectWriter::GetSectionAddressSize(const MCAsmLayout &Layout, const MCSectionData &SD) { if (IsELFMetaDataSection(SD)) @@ -1631,14 +1532,9 @@ uint64_t ELFObjectWriter::GetSectionAddressSize(const MCAsmLayout &Layout, return Layout.getSectionAddressSize(&SD); } -void ELFObjectWriter::WriteDataSectionData(MCAssembler &Asm, +void ELFObjectWriter::writeDataSectionData(MCAssembler &Asm, const MCAsmLayout &Layout, - const MCSectionELF &Section) { - const MCSectionData &SD = Asm.getOrCreateSectionData(Section); - - uint64_t Padding = OffsetToAlignment(OS.tell(), SD.getAlignment()); - WriteZeros(Padding); - + const MCSectionData &SD) { if (IsELFMetaDataSection(SD)) { for (MCSectionData::const_iterator i = SD.begin(), e = SD.end(); i != e; ++i) { @@ -1652,28 +1548,20 @@ void ELFObjectWriter::WriteDataSectionData(MCAssembler &Asm, } void ELFObjectWriter::writeSectionHeader( - MCAssembler &Asm, const GroupMapTy &GroupMap, const MCAsmLayout &Layout, - const SectionIndexMapTy &SectionIndexMap, const RelMapTy &RelMap, + ArrayRef Sections, MCAssembler &Asm, + const GroupMapTy &GroupMap, const MCAsmLayout &Layout, + const SectionIndexMapTy &SectionIndexMap, const SectionOffsetMapTy &SectionOffsetMap) { - const unsigned NumSections = Asm.size() + 1; - - std::vector Sections; - Sections.resize(NumSections - 1); - - for (SectionIndexMapTy::const_iterator i= - SectionIndexMap.begin(), e = SectionIndexMap.end(); i != e; ++i) { - const std::pair &p = *i; - Sections[p.second - 1] = p.first; - } + const unsigned NumSections = Asm.size(); // Null section first. uint64_t FirstSectionSize = - NumSections >= ELF::SHN_LORESERVE ? NumSections : 0; + (NumSections + 1) >= ELF::SHN_LORESERVE ? NumSections + 1 : 0; uint32_t FirstSectionLink = ShstrtabIndex >= ELF::SHN_LORESERVE ? ShstrtabIndex : 0; WriteSecHdrEntry(0, 0, 0, 0, 0, FirstSectionSize, FirstSectionLink, 0, 0, 0); - for (unsigned i = 0; i < NumSections - 1; ++i) { + for (unsigned i = 0; i < NumSections; ++i) { const MCSectionELF &Section = *Sections[i]; const MCSectionData &SD = Asm.getOrCreateSectionData(Section); uint32_t GroupSymbolIndex; @@ -1685,39 +1573,9 @@ void ELFObjectWriter::writeSectionHeader( uint64_t Size = GetSectionAddressSize(Layout, SD); - writeSection(Asm, SectionIndexMap, RelMap, GroupSymbolIndex, - SectionOffsetMap.lookup(&Section), Size, - SD.getAlignment(), Section); - } -} - -void ELFObjectWriter::ComputeSectionOrder(MCAssembler &Asm, - std::vector &Sections) { - for (MCAssembler::iterator it = Asm.begin(), - ie = Asm.end(); it != ie; ++it) { - const MCSectionELF &Section = - static_cast(it->getSection()); - if (Section.getType() == ELF::SHT_GROUP) - Sections.push_back(&Section); - } - - for (MCAssembler::iterator it = Asm.begin(), - ie = Asm.end(); it != ie; ++it) { - const MCSectionELF &Section = - static_cast(it->getSection()); - if (Section.getType() != ELF::SHT_GROUP && - Section.getType() != ELF::SHT_REL && - Section.getType() != ELF::SHT_RELA) - Sections.push_back(&Section); - } - - for (MCAssembler::iterator it = Asm.begin(), - ie = Asm.end(); it != ie; ++it) { - const MCSectionELF &Section = - static_cast(it->getSection()); - if (Section.getType() == ELF::SHT_REL || - Section.getType() == ELF::SHT_RELA) - Sections.push_back(&Section); + writeSection(Asm, SectionIndexMap, GroupSymbolIndex, + SectionOffsetMap.lookup(&Section), Size, SD.getAlignment(), + Section); } } @@ -1727,102 +1585,77 @@ void ELFObjectWriter::WriteObject(MCAssembler &Asm, RevGroupMapTy RevGroupMap; SectionIndexMapTy SectionIndexMap; - unsigned NumUserSections = Asm.size(); - CompressDebugSections(Asm, const_cast(Layout)); - - DenseMap RelMap; - const unsigned NumUserAndRelocSections = Asm.size(); - createIndexedSections(Asm, const_cast(Layout), GroupMap, - RevGroupMap, SectionIndexMap, RelMap); - const unsigned AllSections = Asm.size(); - const unsigned NumIndexedSections = AllSections - NumUserAndRelocSections; - - unsigned NumRegularSections = NumUserSections + NumIndexedSections; + createIndexedSections(Asm, const_cast(Layout), GroupMap, + RevGroupMap, SectionIndexMap); // Compute symbol table information. - computeSymbolTable(Asm, Layout, SectionIndexMap, RevGroupMap, - NumRegularSections); + computeSymbolTable(Asm, Layout, SectionIndexMap, RevGroupMap); - WriteRelocations(Asm, const_cast(Layout), RelMap); + WriteRelocations(Asm, const_cast(Layout)); CreateMetadataSections(const_cast(Asm), const_cast(Layout), SectionIndexMap); - uint64_t NaturalAlignment = is64Bit() ? 8 : 4; - uint64_t HeaderSize = is64Bit() ? sizeof(ELF::Elf64_Ehdr) : - sizeof(ELF::Elf32_Ehdr); - uint64_t FileOff = HeaderSize; - + unsigned NumSections = Asm.size(); std::vector Sections; - ComputeSectionOrder(Asm, Sections); - unsigned NumSections = Sections.size(); - SectionOffsetMapTy SectionOffsetMap; - for (unsigned i = 0; i < NumRegularSections + 1; ++i) { - const MCSectionELF &Section = *Sections[i]; - const MCSectionData &SD = Asm.getOrCreateSectionData(Section); + Sections.resize(NumSections); - FileOff = RoundUpToAlignment(FileOff, SD.getAlignment()); - - // Remember the offset into the file for this section. - SectionOffsetMap[&Section] = FileOff; + for (auto &Pair : SectionIndexMap) + Sections[Pair.second - 1] = Pair.first; - // Get the size of the section in the output file (including padding). - FileOff += GetSectionFileSize(Layout, SD); - } - - FileOff = RoundUpToAlignment(FileOff, NaturalAlignment); - - const unsigned SectionHeaderOffset = FileOff - HeaderSize; + SectionOffsetMapTy SectionOffsetMap; - uint64_t SectionHeaderEntrySize = is64Bit() ? - sizeof(ELF::Elf64_Shdr) : sizeof(ELF::Elf32_Shdr); - FileOff += (NumSections + 1) * SectionHeaderEntrySize; + // Write out the ELF header ... + WriteHeader(Asm, NumSections + 1); - for (unsigned i = NumRegularSections + 1; i < NumSections; ++i) { + // ... then the sections ... + for (unsigned i = 0; i < NumSections; ++i) { const MCSectionELF &Section = *Sections[i]; const MCSectionData &SD = Asm.getOrCreateSectionData(Section); - - FileOff = RoundUpToAlignment(FileOff, SD.getAlignment()); + uint64_t Padding = OffsetToAlignment(OS.tell(), SD.getAlignment()); + WriteZeros(Padding); // Remember the offset into the file for this section. - SectionOffsetMap[&Section] = FileOff; + SectionOffsetMap[&Section] = OS.tell(); - // Get the size of the section in the output file (including padding). - FileOff += GetSectionFileSize(Layout, SD); + writeDataSectionData(Asm, Layout, SD); } - // Write out the ELF header ... - WriteHeader(Asm, SectionHeaderOffset, NumSections + 1); - - // ... then the regular sections ... - // + because of .shstrtab - for (unsigned i = 0; i < NumRegularSections + 1; ++i) - WriteDataSectionData(Asm, Layout, *Sections[i]); - + uint64_t NaturalAlignment = is64Bit() ? 8 : 4; uint64_t Padding = OffsetToAlignment(OS.tell(), NaturalAlignment); WriteZeros(Padding); + const unsigned SectionHeaderOffset = OS.tell(); + // ... then the section header table ... - writeSectionHeader(Asm, GroupMap, Layout, SectionIndexMap, RelMap, + writeSectionHeader(Sections, Asm, GroupMap, Layout, SectionIndexMap, SectionOffsetMap); - // ... and then the remaining sections ... - for (unsigned i = NumRegularSections + 1; i < NumSections; ++i) - WriteDataSectionData(Asm, Layout, *Sections[i]); + if (is64Bit()) { + uint64_t Val = SectionHeaderOffset; + if (sys::IsLittleEndianHost != IsLittleEndian) + sys::swapByteOrder(Val); + OS.pwrite(reinterpret_cast(&Val), sizeof(Val), + offsetof(ELF::Elf64_Ehdr, e_shoff)); + } else { + uint32_t Val = SectionHeaderOffset; + if (sys::IsLittleEndianHost != IsLittleEndian) + sys::swapByteOrder(Val); + OS.pwrite(reinterpret_cast(&Val), sizeof(Val), + offsetof(ELF::Elf32_Ehdr, e_shoff)); + } } -bool -ELFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, - const MCSymbolData &DataA, - const MCFragment &FB, - bool InSet, - bool IsPCRel) const { - if (::isWeak(DataA)) +bool ELFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( + const MCAssembler &Asm, const MCSymbolData &DataA, + const MCSymbolData *DataB, const MCFragment &FB, bool InSet, + bool IsPCRel) const { + if (!InSet && (::isWeak(DataA) || (DataB && ::isWeak(*DataB)))) return false; return MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( - Asm, DataA, FB,InSet, IsPCRel); + Asm, DataA, DataB, FB, InSet, IsPCRel); } bool ELFObjectWriter::isWeak(const MCSymbolData &SD) const { @@ -1830,7 +1663,7 @@ bool ELFObjectWriter::isWeak(const MCSymbolData &SD) const { } MCObjectWriter *llvm::createELFObjectWriter(MCELFObjectTargetWriter *MOTW, - raw_ostream &OS, + raw_pwrite_stream &OS, bool IsLittleEndian) { return new ELFObjectWriter(MOTW, OS, IsLittleEndian); } diff --git a/lib/MC/MCAsmStreamer.cpp b/lib/MC/MCAsmStreamer.cpp index 62f5279..144d355 100644 --- a/lib/MC/MCAsmStreamer.cpp +++ b/lib/MC/MCAsmStreamer.cpp @@ -36,11 +36,10 @@ using namespace llvm; namespace { -class MCAsmStreamer : public MCStreamer { -protected: +class MCAsmStreamer final : public MCStreamer { + std::unique_ptr OSOwner; formatted_raw_ostream &OS; const MCAsmInfo *MAI; -private: std::unique_ptr InstPrinter; std::unique_ptr Emitter; std::unique_ptr AsmBackend; @@ -57,14 +56,15 @@ private: void EmitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override; public: - MCAsmStreamer(MCContext &Context, formatted_raw_ostream &os, + MCAsmStreamer(MCContext &Context, std::unique_ptr os, bool isVerboseAsm, bool useDwarfDirectory, MCInstPrinter *printer, MCCodeEmitter *emitter, MCAsmBackend *asmbackend, bool showInst) - : MCStreamer(Context), OS(os), MAI(Context.getAsmInfo()), - InstPrinter(printer), Emitter(emitter), AsmBackend(asmbackend), - CommentStream(CommentToEmit), IsVerboseAsm(isVerboseAsm), - ShowInst(showInst), UseDwarfDirectory(useDwarfDirectory) { + : MCStreamer(Context), OSOwner(std::move(os)), OS(*OSOwner), + MAI(Context.getAsmInfo()), InstPrinter(printer), Emitter(emitter), + AsmBackend(asmbackend), CommentStream(CommentToEmit), + IsVerboseAsm(isVerboseAsm), ShowInst(showInst), + UseDwarfDirectory(useDwarfDirectory) { if (InstPrinter && IsVerboseAsm) InstPrinter->setCommentStream(CommentStream); } @@ -1262,7 +1262,7 @@ void MCAsmStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &S // If we have an AsmPrinter, use that to print, otherwise print the MCInst. if (InstPrinter) - InstPrinter->printInst(&Inst, OS, ""); + InstPrinter->printInst(&Inst, OS, "", STI); else Inst.print(OS); EmitEOL(); @@ -1314,10 +1314,10 @@ void MCAsmStreamer::FinishImpl() { } MCStreamer *llvm::createAsmStreamer(MCContext &Context, - formatted_raw_ostream &OS, + std::unique_ptr OS, bool isVerboseAsm, bool useDwarfDirectory, MCInstPrinter *IP, MCCodeEmitter *CE, MCAsmBackend *MAB, bool ShowInst) { - return new MCAsmStreamer(Context, OS, isVerboseAsm, useDwarfDirectory, IP, CE, - MAB, ShowInst); + return new MCAsmStreamer(Context, std::move(OS), isVerboseAsm, + useDwarfDirectory, IP, CE, MAB, ShowInst); } diff --git a/lib/MC/MCAssembler.cpp b/lib/MC/MCAssembler.cpp index 857eafc..d09e383 100644 --- a/lib/MC/MCAssembler.cpp +++ b/lib/MC/MCAssembler.cpp @@ -229,8 +229,9 @@ uint64_t MCAsmLayout::getSectionFileSize(const MCSectionData *SD) const { return getSectionAddressSize(SD); } -uint64_t MCAsmLayout::computeBundlePadding(const MCFragment *F, - uint64_t FOffset, uint64_t FSize) { +uint64_t llvm::computeBundlePadding(const MCAssembler &Assembler, + const MCFragment *F, + uint64_t FOffset, uint64_t FSize) { uint64_t BundleSize = Assembler.getBundleAlignSize(); assert(BundleSize > 0 && "computeBundlePadding should only be called if bundling is enabled"); @@ -332,6 +333,7 @@ MCSectionData::getSubsectionInsertionPoint(unsigned Subsection) { getFragmentList().insert(IP, F); F->setParent(this); } + return IP; } @@ -497,14 +499,12 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, } else { const MCSymbolRefExpr *A = Target.getSymA(); const MCSymbol &SA = A->getSymbol(); - if (A->getKind() != MCSymbolRefExpr::VK_None || - SA.AliasedSymbol().isUndefined()) { + if (A->getKind() != MCSymbolRefExpr::VK_None || SA.isUndefined()) { IsResolved = false; } else { const MCSymbolData &DataA = getSymbolData(SA); - IsResolved = - getWriter().IsSymbolRefDifferenceFullyResolvedImpl(*this, DataA, - *DF, false, true); + IsResolved = getWriter().IsSymbolRefDifferenceFullyResolvedImpl( + *this, DataA, nullptr, *DF, false, true); } } } else { @@ -514,12 +514,12 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, Value = Target.getConstant(); if (const MCSymbolRefExpr *A = Target.getSymA()) { - const MCSymbol &Sym = A->getSymbol().AliasedSymbol(); + const MCSymbol &Sym = A->getSymbol(); if (Sym.isDefined()) Value += Layout.getSymbolOffset(&getSymbolData(Sym)); } if (const MCSymbolRefExpr *B = Target.getSymB()) { - const MCSymbol &Sym = B->getSymbol().AliasedSymbol(); + const MCSymbol &Sym = B->getSymbol(); if (Sym.isDefined()) Value -= Layout.getSymbolOffset(&getSymbolData(Sym)); } @@ -634,7 +634,12 @@ void MCAsmLayout::layoutFragment(MCFragment *F) { // The fragment's offset will point to after the padding, and its computed // size won't include the padding. // - if (Assembler.isBundlingEnabled() && F->hasInstructions()) { + // When the -mc-relax-all flag is used, we optimize bundling by writting the + // bundle padding directly into fragments when the instructions are emitted + // inside the streamer. + // + if (Assembler.isBundlingEnabled() && !Assembler.getRelaxAll() && + F->hasInstructions()) { assert(isa(F) && "Only MCEncodedFragment implementations have instructions"); uint64_t FSize = Assembler.computeFragmentSize(*this, *F); @@ -642,7 +647,8 @@ void MCAsmLayout::layoutFragment(MCFragment *F) { if (FSize > Assembler.getBundleAlignSize()) report_fatal_error("Fragment can't be larger than a bundle size"); - uint64_t RequiredBundlePadding = computeBundlePadding(F, F->Offset, FSize); + uint64_t RequiredBundlePadding = computeBundlePadding(Assembler, F, + F->Offset, FSize); if (RequiredBundlePadding > UINT8_MAX) report_fatal_error("Padding cannot exceed 255 bytes"); F->setBundlePadding(static_cast(RequiredBundlePadding)); @@ -657,24 +663,18 @@ static void writeFragmentContents(const MCFragment &F, MCObjectWriter *OW) { OW->WriteBytes(EF.getContents()); } -/// \brief Write the fragment \p F to the output file. -static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout, - const MCFragment &F) { - MCObjectWriter *OW = &Asm.getWriter(); - - // FIXME: Embed in fragments instead? - uint64_t FragmentSize = Asm.computeFragmentSize(Layout, F); - +void MCAssembler::writeFragmentPadding(const MCFragment &F, uint64_t FSize, + MCObjectWriter *OW) const { // Should NOP padding be written out before this fragment? unsigned BundlePadding = F.getBundlePadding(); if (BundlePadding > 0) { - assert(Asm.isBundlingEnabled() && + assert(isBundlingEnabled() && "Writing bundle padding with disabled bundling"); assert(F.hasInstructions() && "Writing bundle padding for a fragment without instructions"); - unsigned TotalLength = BundlePadding + static_cast(FragmentSize); - if (F.alignToBundleEnd() && TotalLength > Asm.getBundleAlignSize()) { + unsigned TotalLength = BundlePadding + static_cast(FSize); + if (F.alignToBundleEnd() && TotalLength > getBundleAlignSize()) { // If the padding itself crosses a bundle boundary, it must be emitted // in 2 pieces, since even nop instructions must not cross boundaries. // v--------------v <- BundleAlignSize @@ -683,16 +683,27 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout, // | Prev |####|####| F | // ---------------------------- // ^-------------------^ <- TotalLength - unsigned DistanceToBoundary = TotalLength - Asm.getBundleAlignSize(); - if (!Asm.getBackend().writeNopData(DistanceToBoundary, OW)) + unsigned DistanceToBoundary = TotalLength - getBundleAlignSize(); + if (!getBackend().writeNopData(DistanceToBoundary, OW)) report_fatal_error("unable to write NOP sequence of " + Twine(DistanceToBoundary) + " bytes"); BundlePadding -= DistanceToBoundary; } - if (!Asm.getBackend().writeNopData(BundlePadding, OW)) + if (!getBackend().writeNopData(BundlePadding, OW)) report_fatal_error("unable to write NOP sequence of " + Twine(BundlePadding) + " bytes"); } +} + +/// \brief Write the fragment \p F to the output file. +static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout, + const MCFragment &F) { + MCObjectWriter *OW = &Asm.getWriter(); + + // FIXME: Embed in fragments instead? + uint64_t FragmentSize = Asm.computeFragmentSize(Layout, F); + + Asm.writeFragmentPadding(F, FragmentSize, OW); // This variable (and its dummy usage) is to participate in the assert at // the end of the function. diff --git a/lib/MC/MCContext.cpp b/lib/MC/MCContext.cpp index 3cb3ea1..5f8e3c1 100644 --- a/lib/MC/MCContext.cpp +++ b/lib/MC/MCContext.cpp @@ -139,6 +139,11 @@ MCSymbol *MCContext::getOrCreateFrameAllocSymbol(StringRef FuncName, "$frame_escape_" + Twine(Idx)); } +MCSymbol *MCContext::getOrCreateParentFrameOffsetSymbol(StringRef FuncName) { + return GetOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + FuncName + + "$parent_frame_offset"); +} + MCSymbol *MCContext::CreateSymbol(StringRef Name, bool AlwaysAddSuffix) { // Determine whether this is an assembler temporary or normal label, if used. bool IsTemporary = false; @@ -257,41 +262,63 @@ MCContext::getMachOSection(StringRef Segment, StringRef Section, Reserved2, Kind, Begin); } -const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, - unsigned Flags, - const char *BeginSymName) { - return getELFSection(Section, Type, Flags, 0, "", BeginSymName); -} - void MCContext::renameELFSection(const MCSectionELF *Section, StringRef Name) { StringRef GroupName; if (const MCSymbol *Group = Section->getGroup()) GroupName = Group->getName(); - ELFUniquingMap.erase(SectionGroupPair(Section->getSectionName(), GroupName)); - auto I = - ELFUniquingMap.insert(std::make_pair(SectionGroupPair(Name, GroupName), - Section)).first; - StringRef CachedName = I->first.first; + unsigned UniqueID = Section->getUniqueID(); + ELFUniquingMap.erase( + ELFSectionKey{Section->getSectionName(), GroupName, UniqueID}); + auto I = ELFUniquingMap.insert(std::make_pair( + ELFSectionKey{Name, GroupName, UniqueID}, + Section)).first; + StringRef CachedName = I->first.SectionName; const_cast(Section)->setSectionName(CachedName); } +const MCSectionELF * +MCContext::createELFRelSection(StringRef Name, unsigned Type, unsigned Flags, + unsigned EntrySize, const MCSymbol *Group, + const MCSectionELF *Associated) { + StringMap::iterator I; + bool Inserted; + std::tie(I, Inserted) = ELFRelSecNames.insert(std::make_pair(Name, true)); + + return new (*this) + MCSectionELF(I->getKey(), Type, Flags, SectionKind::getReadOnly(), + EntrySize, Group, true, nullptr, Associated); +} + const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, unsigned Flags, unsigned EntrySize, - StringRef Group, bool Unique, + StringRef Group, unsigned UniqueID, const char *BeginSymName) { + MCSymbol *GroupSym = nullptr; + if (!Group.empty()) + GroupSym = GetOrCreateSymbol(Group); + + return getELFSection(Section, Type, Flags, EntrySize, GroupSym, UniqueID, + BeginSymName, nullptr); +} + +const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, + unsigned Flags, unsigned EntrySize, + const MCSymbol *GroupSym, + unsigned UniqueID, + const char *BeginSymName, + const MCSectionELF *Associated) { + StringRef Group = ""; + if (GroupSym) + Group = GroupSym->getName(); // Do the lookup, if we have a hit, return it. auto IterBool = ELFUniquingMap.insert( - std::make_pair(SectionGroupPair(Section, Group), nullptr)); + std::make_pair(ELFSectionKey{Section, Group, UniqueID}, nullptr)); auto &Entry = *IterBool.first; - if (!IterBool.second && !Unique) + if (!IterBool.second) return Entry.second; - MCSymbol *GroupSym = nullptr; - if (!Group.empty()) - GroupSym = GetOrCreateSymbol(Group); - - StringRef CachedName = Entry.first.first; + StringRef CachedName = Entry.first.SectionName; SectionKind Kind; if (Flags & ELF::SHF_EXECINSTR) @@ -303,25 +330,17 @@ const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, if (BeginSymName) Begin = createTempSymbol(BeginSymName, false); - MCSectionELF *Result = new (*this) MCSectionELF( - CachedName, Type, Flags, Kind, EntrySize, GroupSym, Unique, Begin); - if (!Unique) - Entry.second = Result; + MCSectionELF *Result = + new (*this) MCSectionELF(CachedName, Type, Flags, Kind, EntrySize, + GroupSym, UniqueID, Begin, Associated); + Entry.second = Result; return Result; } -const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type, - unsigned Flags, unsigned EntrySize, - StringRef Group, - const char *BeginSymName) { - return getELFSection(Section, Type, Flags, EntrySize, Group, false, - BeginSymName); -} - const MCSectionELF *MCContext::CreateELFGroupSection() { MCSectionELF *Result = new (*this) MCSectionELF(".group", ELF::SHT_GROUP, 0, SectionKind::getReadOnly(), 4, - nullptr, false, nullptr); + nullptr, ~0, nullptr, nullptr); return Result; } @@ -329,23 +348,24 @@ const MCSectionCOFF * MCContext::getCOFFSection(StringRef Section, unsigned Characteristics, SectionKind Kind, StringRef COMDATSymName, int Selection, const char *BeginSymName) { - // Do the lookup, if we have a hit, return it. + MCSymbol *COMDATSymbol = nullptr; + if (!COMDATSymName.empty()) { + COMDATSymbol = GetOrCreateSymbol(COMDATSymName); + COMDATSymName = COMDATSymbol->getName(); + } - SectionGroupTriple T(Section, COMDATSymName, Selection); + // Do the lookup, if we have a hit, return it. + COFFSectionKey T{Section, COMDATSymName, Selection}; auto IterBool = COFFUniquingMap.insert(std::make_pair(T, nullptr)); auto Iter = IterBool.first; if (!IterBool.second) return Iter->second; - MCSymbol *COMDATSymbol = nullptr; - if (!COMDATSymName.empty()) - COMDATSymbol = GetOrCreateSymbol(COMDATSymName); - MCSymbol *Begin = nullptr; if (BeginSymName) Begin = createTempSymbol(BeginSymName, false); - StringRef CachedName = std::get<0>(Iter->first); + StringRef CachedName = Iter->first.SectionName; MCSectionCOFF *Result = new (*this) MCSectionCOFF( CachedName, Characteristics, COMDATSymbol, Selection, Kind, Begin); @@ -361,7 +381,7 @@ const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section, } const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section) { - SectionGroupTriple T(Section, "", 0); + COFFSectionKey T{Section, "", 0}; auto Iter = COFFUniquingMap.find(T); if (Iter == COFFUniquingMap.end()) return nullptr; diff --git a/lib/MC/MCDisassembler/Disassembler.cpp b/lib/MC/MCDisassembler/Disassembler.cpp index d9f01d0..716d76a 100644 --- a/lib/MC/MCDisassembler/Disassembler.cpp +++ b/lib/MC/MCDisassembler/Disassembler.cpp @@ -33,22 +33,22 @@ using namespace llvm; // disassembler context. If not, it returns NULL. // LLVMDisasmContextRef -LLVMCreateDisasmCPUFeatures(const char *Triple, const char *CPU, +LLVMCreateDisasmCPUFeatures(const char *TT, const char *CPU, const char *Features, void *DisInfo, int TagType, LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp) { // Get the target. std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error); + const Target *TheTarget = TargetRegistry::lookupTarget(TT, Error); if (!TheTarget) return nullptr; - const MCRegisterInfo *MRI = TheTarget->createMCRegInfo(Triple); + const MCRegisterInfo *MRI = TheTarget->createMCRegInfo(TT); if (!MRI) return nullptr; // Get the assembler info needed to setup the MCContext. - const MCAsmInfo *MAI = TheTarget->createMCAsmInfo(*MRI, Triple); + const MCAsmInfo *MAI = TheTarget->createMCAsmInfo(*MRI, TT); if (!MAI) return nullptr; @@ -56,8 +56,8 @@ LLVMCreateDisasmCPUFeatures(const char *Triple, const char *CPU, if (!MII) return nullptr; - const MCSubtargetInfo *STI = TheTarget->createMCSubtargetInfo(Triple, CPU, - Features); + const MCSubtargetInfo *STI = + TheTarget->createMCSubtargetInfo(TT, CPU, Features); if (!STI) return nullptr; @@ -72,25 +72,24 @@ LLVMCreateDisasmCPUFeatures(const char *Triple, const char *CPU, return nullptr; std::unique_ptr RelInfo( - TheTarget->createMCRelocationInfo(Triple, *Ctx)); + TheTarget->createMCRelocationInfo(TT, *Ctx)); if (!RelInfo) return nullptr; std::unique_ptr Symbolizer(TheTarget->createMCSymbolizer( - Triple, GetOpInfo, SymbolLookUp, DisInfo, Ctx, std::move(RelInfo))); + TT, GetOpInfo, SymbolLookUp, DisInfo, Ctx, std::move(RelInfo))); DisAsm->setSymbolizer(std::move(Symbolizer)); // Set up the instruction printer. int AsmPrinterVariant = MAI->getAssemblerDialect(); - MCInstPrinter *IP = TheTarget->createMCInstPrinter(AsmPrinterVariant, - *MAI, *MII, *MRI, *STI); + MCInstPrinter *IP = TheTarget->createMCInstPrinter( + Triple(TT), AsmPrinterVariant, *MAI, *MII, *MRI); if (!IP) return nullptr; - LLVMDisasmContext *DC = new LLVMDisasmContext(Triple, DisInfo, TagType, - GetOpInfo, SymbolLookUp, - TheTarget, MAI, MRI, - STI, MII, Ctx, DisAsm, IP); + LLVMDisasmContext *DC = + new LLVMDisasmContext(TT, DisInfo, TagType, GetOpInfo, SymbolLookUp, + TheTarget, MAI, MRI, STI, MII, Ctx, DisAsm, IP); if (!DC) return nullptr; @@ -98,19 +97,19 @@ LLVMCreateDisasmCPUFeatures(const char *Triple, const char *CPU, return DC; } -LLVMDisasmContextRef LLVMCreateDisasmCPU(const char *Triple, const char *CPU, - void *DisInfo, int TagType, - LLVMOpInfoCallback GetOpInfo, - LLVMSymbolLookupCallback SymbolLookUp){ - return LLVMCreateDisasmCPUFeatures(Triple, CPU, "", DisInfo, TagType, - GetOpInfo, SymbolLookUp); +LLVMDisasmContextRef +LLVMCreateDisasmCPU(const char *TT, const char *CPU, void *DisInfo, int TagType, + LLVMOpInfoCallback GetOpInfo, + LLVMSymbolLookupCallback SymbolLookUp) { + return LLVMCreateDisasmCPUFeatures(TT, CPU, "", DisInfo, TagType, GetOpInfo, + SymbolLookUp); } -LLVMDisasmContextRef LLVMCreateDisasm(const char *Triple, void *DisInfo, +LLVMDisasmContextRef LLVMCreateDisasm(const char *TT, void *DisInfo, int TagType, LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp) { - return LLVMCreateDisasmCPUFeatures(Triple, "", "", DisInfo, TagType, - GetOpInfo, SymbolLookUp); + return LLVMCreateDisasmCPUFeatures(TT, "", "", DisInfo, TagType, GetOpInfo, + SymbolLookUp); } // @@ -268,7 +267,7 @@ size_t LLVMDisasmInstruction(LLVMDisasmContextRef DCR, uint8_t *Bytes, SmallVector InsnStr; raw_svector_ostream OS(InsnStr); formatted_raw_ostream FormattedOS(OS); - IP->printInst(&Inst, FormattedOS, AnnotationsStr); + IP->printInst(&Inst, FormattedOS, AnnotationsStr, *DC->getSubtargetInfo()); if (DC->getOptions() & LLVMDisassembler_Option_PrintLatency) emitLatency(DC, Inst); @@ -312,11 +311,10 @@ int LLVMSetDisasmOptions(LLVMDisasmContextRef DCR, uint64_t Options){ const MCAsmInfo *MAI = DC->getAsmInfo(); const MCInstrInfo *MII = DC->getInstrInfo(); const MCRegisterInfo *MRI = DC->getRegisterInfo(); - const MCSubtargetInfo *STI = DC->getSubtargetInfo(); int AsmPrinterVariant = MAI->getAssemblerDialect(); AsmPrinterVariant = AsmPrinterVariant == 0 ? 1 : 0; MCInstPrinter *IP = DC->getTarget()->createMCInstPrinter( - AsmPrinterVariant, *MAI, *MII, *MRI, *STI); + Triple(DC->getTripleName()), AsmPrinterVariant, *MAI, *MII, *MRI); if (IP) { DC->setIP(IP); DC->addOptions(LLVMDisassembler_Option_AsmPrinterVariant); diff --git a/lib/MC/MCDwarf.cpp b/lib/MC/MCDwarf.cpp index 87e7ed1..e9f685e 100644 --- a/lib/MC/MCDwarf.cpp +++ b/lib/MC/MCDwarf.cpp @@ -803,7 +803,7 @@ static void EmitGenDwarfRanges(MCStreamer *MCOS) { MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfRangesSection()); - for (const auto sec : Sections) { + for (const auto &sec : Sections) { MCSymbol *StartSymbol = sec.second.first; MCSymbol *EndSymbol = sec.second.second; diff --git a/lib/MC/MCELFObjectTargetWriter.cpp b/lib/MC/MCELFObjectTargetWriter.cpp index 84176dc..dc3d6c3 100644 --- a/lib/MC/MCELFObjectTargetWriter.cpp +++ b/lib/MC/MCELFObjectTargetWriter.cpp @@ -28,3 +28,24 @@ bool MCELFObjectTargetWriter::needsRelocateWithSymbol(const MCSymbolData &SD, unsigned Type) const { return false; } + +// ELF doesn't require relocations to be in any order. We sort by the Offset, +// just to match gnu as for easier comparison. The use type is an arbitrary way +// of making the sort deterministic. +static int cmpRel(const ELFRelocationEntry *AP, const ELFRelocationEntry *BP) { + const ELFRelocationEntry &A = *AP; + const ELFRelocationEntry &B = *BP; + if (A.Offset != B.Offset) + return B.Offset - A.Offset; + if (B.Type != A.Type) + return A.Type - B.Type; + //llvm_unreachable("ELFRelocs might be unstable!"); + return 0; +} + + +void +MCELFObjectTargetWriter::sortRelocs(const MCAssembler &Asm, + std::vector &Relocs) { + array_pod_sort(Relocs.begin(), Relocs.end(), cmpRel); +} diff --git a/lib/MC/MCELFStreamer.cpp b/lib/MC/MCELFStreamer.cpp index cdf5033..aa05390 100644 --- a/lib/MC/MCELFStreamer.cpp +++ b/lib/MC/MCELFStreamer.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCAsmLayout.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCCodeEmitter.h" @@ -40,6 +41,48 @@ using namespace llvm; MCELFStreamer::~MCELFStreamer() { } +void MCELFStreamer::mergeFragment(MCDataFragment *DF, + MCEncodedFragmentWithFixups *EF) { + MCAssembler &Assembler = getAssembler(); + + if (Assembler.isBundlingEnabled() && Assembler.getRelaxAll()) { + uint64_t FSize = EF->getContents().size(); + + if (FSize > Assembler.getBundleAlignSize()) + report_fatal_error("Fragment can't be larger than a bundle size"); + + uint64_t RequiredBundlePadding = computeBundlePadding( + Assembler, EF, DF->getContents().size(), FSize); + + if (RequiredBundlePadding > UINT8_MAX) + report_fatal_error("Padding cannot exceed 255 bytes"); + + if (RequiredBundlePadding > 0) { + SmallString<256> Code; + raw_svector_ostream VecOS(Code); + MCObjectWriter *OW = Assembler.getBackend().createObjectWriter(VecOS); + + EF->setBundlePadding(static_cast(RequiredBundlePadding)); + + Assembler.writeFragmentPadding(*EF, FSize, OW); + VecOS.flush(); + delete OW; + + DF->getContents().append(Code.begin(), Code.end()); + } + } + + flushPendingLabels(DF, DF->getContents().size()); + + for (unsigned i = 0, e = EF->getFixups().size(); i != e; ++i) { + EF->getFixups()[i].setOffset(EF->getFixups()[i].getOffset() + + DF->getContents().size()); + DF->getFixups().push_back(EF->getFixups()[i]); + } + DF->setHasInstructions(true); + DF->getContents().append(EF->getContents().begin(), EF->getContents().end()); +} + void MCELFStreamer::InitSections(bool NoExecStack) { // This emulates the same behavior of GNU as. This makes it easier // to compare the output as the major sections are in the same order. @@ -449,7 +492,16 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst, if (Assembler.isBundlingEnabled()) { MCSectionData *SD = getCurrentSectionData(); - if (SD->isBundleLocked() && !SD->isBundleGroupBeforeFirstInst()) + if (Assembler.getRelaxAll() && SD->isBundleLocked()) + // If the -mc-relax-all flag is used and we are bundle-locked, we re-use + // the current bundle group. + DF = BundleGroups.back(); + else if (Assembler.getRelaxAll() && !SD->isBundleLocked()) + // When not in a bundle-locked group and the -mc-relax-all flag is used, + // we create a new temporary fragment which will be later merged into + // the current fragment. + DF = new MCDataFragment(); + else if (SD->isBundleLocked() && !SD->isBundleGroupBeforeFirstInst()) // If we are bundle-locked, we re-use the current fragment. // The bundle-locking directive ensures this is a new data fragment. DF = cast(getCurrentFragment()); @@ -487,6 +539,14 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst, } DF->setHasInstructions(true); DF->getContents().append(Code.begin(), Code.end()); + + if (Assembler.isBundlingEnabled() && Assembler.getRelaxAll()) { + MCSectionData *SD = getCurrentSectionData(); + if (!SD->isBundleLocked()) { + mergeFragment(getOrCreateDataFragment(), DF); + delete DF; + } + } } void MCELFStreamer::EmitBundleAlignMode(unsigned AlignPow2) { @@ -510,6 +570,12 @@ void MCELFStreamer::EmitBundleLock(bool AlignToEnd) { if (!SD->isBundleLocked()) SD->setBundleGroupBeforeFirstInst(true); + if (getAssembler().getRelaxAll() && !SD->isBundleLocked()) { + // TODO: drop the lock state and set directly in the fragment + MCDataFragment *DF = new MCDataFragment(); + BundleGroups.push_back(DF); + } + SD->setBundleLockState(AlignToEnd ? MCSectionData::BundleLockedAlignToEnd : MCSectionData::BundleLocked); } @@ -525,7 +591,27 @@ void MCELFStreamer::EmitBundleUnlock() { else if (SD->isBundleGroupBeforeFirstInst()) report_fatal_error("Empty bundle-locked group is forbidden"); - SD->setBundleLockState(MCSectionData::NotBundleLocked); + // When the -mc-relax-all flag is used, we emit instructions to fragments + // stored on a stack. When the bundle unlock is emited, we pop a fragment + // from the stack a merge it to the one below. + if (getAssembler().getRelaxAll()) { + assert(!BundleGroups.empty() && "There are no bundle groups"); + MCDataFragment *DF = BundleGroups.back(); + + // FIXME: Use BundleGroups to track the lock state instead. + SD->setBundleLockState(MCSectionData::NotBundleLocked); + + // FIXME: Use more separate fragments for nested groups. + if (!SD->isBundleLocked()) { + mergeFragment(getOrCreateDataFragment(), DF); + BundleGroups.pop_back(); + delete DF; + } + + if (SD->getBundleLockState() != MCSectionData::BundleLockedAlignToEnd) + getOrCreateDataFragment()->setAlignToBundleEnd(false); + } else + SD->setBundleLockState(MCSectionData::NotBundleLocked); } void MCELFStreamer::Flush() { @@ -561,7 +647,7 @@ void MCELFStreamer::FinishImpl() { } MCStreamer *llvm::createELFStreamer(MCContext &Context, MCAsmBackend &MAB, - raw_ostream &OS, MCCodeEmitter *CE, + raw_pwrite_stream &OS, MCCodeEmitter *CE, bool RelaxAll) { MCELFStreamer *S = new MCELFStreamer(Context, MAB, OS, CE); if (RelaxAll) diff --git a/lib/MC/MCExpr.cpp b/lib/MC/MCExpr.cpp index 8a64403..0702539 100644 --- a/lib/MC/MCExpr.cpp +++ b/lib/MC/MCExpr.cpp @@ -775,6 +775,10 @@ const MCSection *MCExpr::FindAssociatedSection() const { if (RHS_S == MCSymbol::AbsolutePseudoSection) return LHS_S; + // Not always correct, but probably the best we can do without more context. + if (BE->getOpcode() == MCBinaryExpr::Sub) + return MCSymbol::AbsolutePseudoSection; + // Otherwise, return the first non-null section. return LHS_S ? LHS_S : RHS_S; } diff --git a/lib/MC/MCMachOStreamer.cpp b/lib/MC/MCMachOStreamer.cpp index d5c7101..5c78f5f 100644 --- a/lib/MC/MCMachOStreamer.cpp +++ b/lib/MC/MCMachOStreamer.cpp @@ -54,7 +54,7 @@ private: void EmitDataRegionEnd(); public: - MCMachOStreamer(MCContext &Context, MCAsmBackend &MAB, raw_ostream &OS, + MCMachOStreamer(MCContext &Context, MCAsmBackend &MAB, raw_pwrite_stream &OS, MCCodeEmitter *Emitter, bool DWARFMustBeAtTheEnd, bool label) : MCObjectStreamer(Context, MAB, OS, Emitter), LabelSections(label), DWARFMustBeAtTheEnd(DWARFMustBeAtTheEnd), CreatedADWARFSection(false) {} @@ -491,7 +491,7 @@ void MCMachOStreamer::FinishImpl() { } MCStreamer *llvm::createMachOStreamer(MCContext &Context, MCAsmBackend &MAB, - raw_ostream &OS, MCCodeEmitter *CE, + raw_pwrite_stream &OS, MCCodeEmitter *CE, bool RelaxAll, bool DWARFMustBeAtTheEnd, bool LabelSections) { MCMachOStreamer *S = new MCMachOStreamer(Context, MAB, OS, CE, diff --git a/lib/MC/MCObjectStreamer.cpp b/lib/MC/MCObjectStreamer.cpp index 6aa2de3..d254e95 100644 --- a/lib/MC/MCObjectStreamer.cpp +++ b/lib/MC/MCObjectStreamer.cpp @@ -24,18 +24,13 @@ using namespace llvm; MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, - raw_ostream &OS, MCCodeEmitter *Emitter_) + raw_pwrite_stream &OS, + MCCodeEmitter *Emitter_) : MCStreamer(Context), Assembler(new MCAssembler(Context, TAB, *Emitter_, *TAB.createObjectWriter(OS), OS)), CurSectionData(nullptr), EmitEHFrame(true), EmitDebugFrame(false) {} -MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, - raw_ostream &OS, MCCodeEmitter *Emitter_, - MCAssembler *Assembler) - : MCStreamer(Context), Assembler(Assembler), CurSectionData(nullptr), - EmitEHFrame(true), EmitDebugFrame(false) {} - MCObjectStreamer::~MCObjectStreamer() { delete &Assembler->getBackend(); delete &Assembler->getEmitter(); @@ -43,7 +38,7 @@ MCObjectStreamer::~MCObjectStreamer() { delete Assembler; } -void MCObjectStreamer::flushPendingLabels(MCFragment *F) { +void MCObjectStreamer::flushPendingLabels(MCFragment *F, uint64_t FOffset) { if (PendingLabels.size()) { if (!F) { F = new MCDataFragment(); @@ -52,7 +47,7 @@ void MCObjectStreamer::flushPendingLabels(MCFragment *F) { } for (MCSymbolData *SD : PendingLabels) { SD->setFragment(F); - SD->setOffset(0); + SD->setOffset(FOffset); } PendingLabels.clear(); } @@ -93,7 +88,8 @@ MCDataFragment *MCObjectStreamer::getOrCreateDataFragment() { MCDataFragment *F = dyn_cast_or_null(getCurrentFragment()); // When bundling is enabled, we don't want to add data to a fragment that // already has instructions (see MCELFStreamer::EmitInstToData for details) - if (!F || (Assembler->isBundlingEnabled() && F->hasInstructions())) { + if (!F || (Assembler->isBundlingEnabled() && !Assembler->getRelaxAll() && + F->hasInstructions())) { F = new MCDataFragment(); insert(F); } @@ -149,7 +145,9 @@ void MCObjectStreamer::EmitLabel(MCSymbol *Symbol) { // If there is a current fragment, mark the symbol as pointing into it. // Otherwise queue the label and set its fragment pointer when we emit the // next fragment. - if (auto *F = dyn_cast_or_null(getCurrentFragment())) { + auto *F = dyn_cast_or_null(getCurrentFragment()); + if (F && !(getAssembler().isBundlingEnabled() && + getAssembler().getRelaxAll())) { SD.setFragment(F); SD.setOffset(F->getContents().size()); } else { @@ -248,6 +246,9 @@ void MCObjectStreamer::EmitInstruction(const MCInst &Inst, void MCObjectStreamer::EmitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &STI) { + if (getAssembler().getRelaxAll() && getAssembler().isBundlingEnabled()) + llvm_unreachable("All instructions should have already been relaxed"); + // Always create a new, separate fragment here, because its size can change // during relaxation. MCRelaxableFragment *IF = new MCRelaxableFragment(Inst, STI); diff --git a/lib/MC/MCObjectWriter.cpp b/lib/MC/MCObjectWriter.cpp index 3c536ec..e40c07d 100644 --- a/lib/MC/MCObjectWriter.cpp +++ b/lib/MC/MCObjectWriter.cpp @@ -27,7 +27,7 @@ bool MCObjectWriter::IsSymbolRefDifferenceFullyResolved( const MCSymbol &SA = A->getSymbol(); const MCSymbol &SB = B->getSymbol(); - if (SA.AliasedSymbol().isUndefined() || SB.AliasedSymbol().isUndefined()) + if (SA.isUndefined() || SB.isUndefined()) return false; const MCSymbolData &DataA = Asm.getSymbolData(SA); @@ -35,19 +35,15 @@ bool MCObjectWriter::IsSymbolRefDifferenceFullyResolved( if(!DataA.getFragment() || !DataB.getFragment()) return false; - return IsSymbolRefDifferenceFullyResolvedImpl(Asm, DataA, - *DataB.getFragment(), - InSet, - false); + return IsSymbolRefDifferenceFullyResolvedImpl( + Asm, DataA, &DataB, *DataB.getFragment(), InSet, false); } -bool -MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, - const MCSymbolData &DataA, - const MCFragment &FB, - bool InSet, - bool IsPCRel) const { - const MCSection &SecA = DataA.getSymbol().AliasedSymbol().getSection(); +bool MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( + const MCAssembler &Asm, const MCSymbolData &DataA, + const MCSymbolData *DataB, const MCFragment &FB, bool InSet, + bool IsPCRel) const { + const MCSection &SecA = DataA.getSymbol().getSection(); const MCSection &SecB = FB.getParent()->getSection(); // On ELF and COFF A - B is absolute if A and B are in the same section. return &SecA == &SecB; diff --git a/lib/MC/MCParser/AsmParser.cpp b/lib/MC/MCParser/AsmParser.cpp index 2bf980b..92a7507 100644 --- a/lib/MC/MCParser/AsmParser.cpp +++ b/lib/MC/MCParser/AsmParser.cpp @@ -175,7 +175,7 @@ private: public: AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out, const MCAsmInfo &MAI); - virtual ~AsmParser(); + ~AsmParser() override; bool Run(bool NoInitialTextSection, bool NoFinalize = false) override; @@ -4606,7 +4606,7 @@ bool AsmParser::parseMSInlineAsm( ++InputIdx; OutputDecls.push_back(OpDecl); OutputDeclsAddressOf.push_back(Operand.needAddressOf()); - OutputConstraints.push_back('=' + Operand.getConstraint().str()); + OutputConstraints.push_back(("=" + Operand.getConstraint()).str()); AsmStrRewrites.push_back(AsmRewrite(AOK_Output, Start, SymName.size())); } else { InputDecls.push_back(OpDecl); diff --git a/lib/MC/MCParser/ELFAsmParser.cpp b/lib/MC/MCParser/ELFAsmParser.cpp index 7a120a1..a19339d 100644 --- a/lib/MC/MCParser/ELFAsmParser.cpp +++ b/lib/MC/MCParser/ELFAsmParser.cpp @@ -379,7 +379,7 @@ bool ELFAsmParser::ParseSectionArguments(bool IsPush, SMLoc loc) { const MCExpr *Subsection = nullptr; bool UseLastGroup = false; StringRef UniqueStr; - bool Unique = false; + int64_t UniqueID = ~0; // Set the defaults first. if (SectionName == ".fini" || SectionName == ".init" || @@ -470,7 +470,15 @@ bool ELFAsmParser::ParseSectionArguments(bool IsPush, SMLoc loc) { return TokError("expected identifier in directive"); if (UniqueStr != "unique") return TokError("expected 'unique'"); - Unique = true; + if (getLexer().isNot(AsmToken::Comma)) + return TokError("expected commma"); + Lex(); + if (getParser().parseAbsoluteExpression(UniqueID)) + return true; + if (UniqueID < 0) + return TokError("unique id must be positive"); + if (!isUInt<32>(UniqueID) || UniqueID == ~0U) + return TokError("unique id is too large"); } } } @@ -520,7 +528,7 @@ EndStmt: } const MCSection *ELFSection = getContext().getELFSection( - SectionName, Type, Flags, Size, GroupName, Unique); + SectionName, Type, Flags, Size, GroupName, UniqueID); getStreamer().SwitchSection(ELFSection, Subsection); if (getContext().getGenDwarfForAssembly()) { diff --git a/lib/MC/MCSectionELF.cpp b/lib/MC/MCSectionELF.cpp index da38682..3cd8453 100644 --- a/lib/MC/MCSectionELF.cpp +++ b/lib/MC/MCSectionELF.cpp @@ -24,7 +24,7 @@ MCSectionELF::~MCSectionELF() {} // anchor. bool MCSectionELF::ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const { - if (Unique) + if (isUnique()) return false; // FIXME: Does .section .bss/.data/.text work everywhere?? @@ -148,8 +148,8 @@ void MCSectionELF::PrintSwitchToSection(const MCAsmInfo &MAI, OS << ",comdat"; } - if (Unique) - OS << ",unique"; + if (isUnique()) + OS << ",unique," << UniqueID; OS << '\n'; diff --git a/lib/MC/MCSubtargetInfo.cpp b/lib/MC/MCSubtargetInfo.cpp index ca3894b..daba321 100644 --- a/lib/MC/MCSubtargetInfo.cpp +++ b/lib/MC/MCSubtargetInfo.cpp @@ -93,9 +93,10 @@ MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const { const SubtargetInfoKV *Found = std::lower_bound(ProcSchedModels, ProcSchedModels+NumProcs, CPU); if (Found == ProcSchedModels+NumProcs || StringRef(Found->Key) != CPU) { - errs() << "'" << CPU - << "' is not a recognized processor for this target" - << " (ignoring processor)\n"; + if (CPU != "help") // Don't error if the user asked for help. + errs() << "'" << CPU + << "' is not a recognized processor for this target" + << " (ignoring processor)\n"; return MCSchedModel::GetDefaultSchedModel(); } assert(Found->Value && "Missing processor SchedModel value"); diff --git a/lib/MC/MCSymbol.cpp b/lib/MC/MCSymbol.cpp index 2416525..6582574 100644 --- a/lib/MC/MCSymbol.cpp +++ b/lib/MC/MCSymbol.cpp @@ -55,13 +55,7 @@ void MCSymbol::setVariableValue(const MCExpr *Value) { assert(!IsUsed && "Cannot set a variable that has already been used."); assert(Value && "Invalid variable value!"); this->Value = Value; - - // Variables should always be marked as in the same "section" as the value. - const MCSection *Section = Value->FindAssociatedSection(); - if (Section) - setSection(*Section); - else - setUndefined(); + this->Section = nullptr; } void MCSymbol::print(raw_ostream &OS) const { diff --git a/lib/MC/MachObjectWriter.cpp b/lib/MC/MachObjectWriter.cpp index 5e9e86f..837f585 100644 --- a/lib/MC/MachObjectWriter.cpp +++ b/lib/MC/MachObjectWriter.cpp @@ -649,38 +649,18 @@ void MachObjectWriter::computeSectionAddresses(const MCAssembler &Asm, } } -void MachObjectWriter::markAbsoluteVariableSymbols(MCAssembler &Asm, - const MCAsmLayout &Layout) { - for (MCSymbolData &SD : Asm.symbols()) { - if (!SD.getSymbol().isVariable()) - continue; - - // Is the variable is a symbol difference (SA - SB + C) expression, - // and neither symbol is external, mark the variable as absolute. - const MCExpr *Expr = SD.getSymbol().getVariableValue(); - MCValue Value; - if (Expr->EvaluateAsRelocatable(Value, &Layout, nullptr)) { - if (Value.getSymA() && Value.getSymB()) - const_cast(&SD.getSymbol())->setAbsolute(); - } - } -} - void MachObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm, const MCAsmLayout &Layout) { computeSectionAddresses(Asm, Layout); // Create symbol data for any indirect symbols. BindIndirectSymbols(Asm); - - // Mark symbol difference expressions in variables (from .set or = directives) - // as absolute. - markAbsoluteVariableSymbols(Asm, Layout); } bool MachObjectWriter:: IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, const MCSymbolData &DataA, + const MCSymbolData *DataB, const MCFragment &FB, bool InSet, bool IsPCRel) const { @@ -1027,7 +1007,7 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm, } MCObjectWriter *llvm::createMachObjectWriter(MCMachObjectTargetWriter *MOTW, - raw_ostream &OS, + raw_pwrite_stream &OS, bool IsLittleEndian) { return new MachObjectWriter(MOTW, OS, IsLittleEndian); } diff --git a/lib/MC/SubtargetFeature.cpp b/lib/MC/SubtargetFeature.cpp index ec6c9cb..b600baf 100644 --- a/lib/MC/SubtargetFeature.cpp +++ b/lib/MC/SubtargetFeature.cpp @@ -81,11 +81,12 @@ static std::string Join(const std::vector &V) { } /// Adding features. -void SubtargetFeatures::AddFeature(StringRef String) { - // Don't add empty features or features we already have. +void SubtargetFeatures::AddFeature(StringRef String, bool Enable) { + // Don't add empty features. if (!String.empty()) // Convert to lowercase, prepend flag if we don't already have a flag. - Features.push_back(hasFlag(String) ? String.str() : "+" + String.lower()); + Features.push_back(hasFlag(String) ? String.lower() + : (Enable ? "+" : "-") + String.lower()); } /// Find KV in array using binary search. diff --git a/lib/MC/WinCOFFObjectWriter.cpp b/lib/MC/WinCOFFObjectWriter.cpp index c6bc81d..38bb883 100644 --- a/lib/MC/WinCOFFObjectWriter.cpp +++ b/lib/MC/WinCOFFObjectWriter.cpp @@ -126,8 +126,8 @@ public: bool UseBigObj; - WinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, raw_ostream &OS); - + WinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, raw_pwrite_stream &OS); + void reset() override { memset(&Header, 0, sizeof(Header)); Header.Machine = TargetObjectWriter->getMachine(); @@ -172,6 +172,7 @@ public: bool IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, const MCSymbolData &DataA, + const MCSymbolData *DataB, const MCFragment &FB, bool InSet, bool IsPCRel) const override; @@ -257,7 +258,7 @@ size_t COFFSection::size() { // WinCOFFObjectWriter class implementation WinCOFFObjectWriter::WinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, - raw_ostream &OS) + raw_pwrite_stream &OS) : MCObjectWriter(OS, true), TargetObjectWriter(MOTW) { memset(&Header, 0, sizeof(Header)); @@ -382,9 +383,7 @@ void WinCOFFObjectWriter::DefineSymbol(MCSymbolData const &SymbolData, coff_symbol->Other = GetOrCreateCOFFSymbol(&SymRef->getSymbol()); } else { - std::string WeakName = std::string(".weak.") - + Symbol.getName().str() - + ".default"; + std::string WeakName = (".weak." + Symbol.getName() + ".default").str(); COFFSymbol *WeakDefault = createSymbol(WeakName); WeakDefault->Data.SectionNumber = COFF::IMAGE_SYM_ABSOLUTE; WeakDefault->Data.StorageClass = COFF::IMAGE_SYM_CLASS_EXTERNAL; @@ -651,16 +650,17 @@ void WinCOFFObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm, } bool WinCOFFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( - const MCAssembler &Asm, const MCSymbolData &DataA, const MCFragment &FB, - bool InSet, bool IsPCRel) const { + const MCAssembler &Asm, const MCSymbolData &DataA, + const MCSymbolData *DataB, const MCFragment &FB, bool InSet, + bool IsPCRel) const { // MS LINK expects to be able to replace all references to a function with a // thunk to implement their /INCREMENTAL feature. Make sure we don't optimize // away any relocations to functions. if ((((DataA.getFlags() & COFF::SF_TypeMask) >> COFF::SF_TypeShift) >> COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION) return false; - return MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(Asm, DataA, FB, - InSet, IsPCRel); + return MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl( + Asm, DataA, DataB, FB, InSet, IsPCRel); } bool WinCOFFObjectWriter::isWeak(const MCSymbolData &SD) const { @@ -1073,9 +1073,8 @@ void MCWinCOFFObjectTargetWriter::anchor() {} //------------------------------------------------------------------------------ // WinCOFFObjectWriter factory function -namespace llvm { - MCObjectWriter *createWinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, - raw_ostream &OS) { - return new WinCOFFObjectWriter(MOTW, OS); - } +MCObjectWriter * +llvm::createWinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, + raw_pwrite_stream &OS) { + return new WinCOFFObjectWriter(MOTW, OS); } diff --git a/lib/MC/WinCOFFStreamer.cpp b/lib/MC/WinCOFFStreamer.cpp index f902d2b..8f9aacb 100644 --- a/lib/MC/WinCOFFStreamer.cpp +++ b/lib/MC/WinCOFFStreamer.cpp @@ -39,7 +39,7 @@ using namespace llvm; namespace llvm { MCWinCOFFStreamer::MCWinCOFFStreamer(MCContext &Context, MCAsmBackend &MAB, - MCCodeEmitter &CE, raw_ostream &OS) + MCCodeEmitter &CE, raw_pwrite_stream &OS) : MCObjectStreamer(Context, MAB, OS, &CE), CurSymbol(nullptr) {} void MCWinCOFFStreamer::EmitInstToData(const MCInst &Inst, diff --git a/lib/Object/COFFObjectFile.cpp b/lib/Object/COFFObjectFile.cpp index ad278a4..4c38b8f 100644 --- a/lib/Object/COFFObjectFile.cpp +++ b/lib/Object/COFFObjectFile.cpp @@ -262,7 +262,7 @@ std::error_code COFFObjectFile::getSymbolSize(DataRefImpl Ref, } const section_iterator SecEnd = section_end(); uint64_t AfterAddr = UnknownAddressOrSize; - for (const symbol_iterator &SymbI : symbols()) { + for (const symbol_iterator SymbI : symbols()) { section_iterator SecI = SecEnd; if (std::error_code EC = SymbI->getSection(SecI)) return EC; diff --git a/lib/Object/MachOObjectFile.cpp b/lib/Object/MachOObjectFile.cpp index 4a1c311..7129aa3 100644 --- a/lib/Object/MachOObjectFile.cpp +++ b/lib/Object/MachOObjectFile.cpp @@ -1537,7 +1537,7 @@ bool ExportEntry::operator==(const ExportEntry &Other) const { if (Stack.size() != Other.Stack.size()) return false; // Not equal if different cumulative strings. - if (!CumulativeString.str().equals(Other.CumulativeString.str())) + if (!CumulativeString.equals(Other.CumulativeString)) return false; // Equal if all nodes in both stacks match. for (unsigned i=0; i < Stack.size(); ++i) { @@ -1559,7 +1559,7 @@ uint64_t ExportEntry::readULEB128(const uint8_t *&Ptr) { } StringRef ExportEntry::name() const { - return CumulativeString.str(); + return CumulativeString; } uint64_t ExportEntry::flags() const { diff --git a/lib/Option/ArgList.cpp b/lib/Option/ArgList.cpp index 4bc8f92..b771a18 100644 --- a/lib/Option/ArgList.cpp +++ b/lib/Option/ArgList.cpp @@ -395,7 +395,7 @@ Arg *DerivedArgList::MakeSeparateArg(const Arg *BaseArg, const Option Opt, Arg *DerivedArgList::MakeJoinedArg(const Arg *BaseArg, const Option Opt, StringRef Value) const { - unsigned Index = BaseArgs.MakeIndex(Opt.getName().str() + Value.str()); + unsigned Index = BaseArgs.MakeIndex((Opt.getName() + Value).str()); SynthesizedArgs.push_back(make_unique( Opt, MakeArgString(Opt.getPrefix() + Opt.getName()), Index, BaseArgs.getArgString(Index) + Opt.getName().size(), BaseArg)); diff --git a/lib/Support/APInt.cpp b/lib/Support/APInt.cpp index 2533fa0..228f75e 100644 --- a/lib/Support/APInt.cpp +++ b/lib/Support/APInt.cpp @@ -162,7 +162,7 @@ APInt& APInt::operator=(uint64_t RHS) { return clearUnusedBits(); } -/// Profile - This method 'profiles' an APInt for use with FoldingSet. +/// This method 'profiles' an APInt for use with FoldingSet. void APInt::Profile(FoldingSetNodeID& ID) const { ID.AddInteger(BitWidth); @@ -176,7 +176,7 @@ void APInt::Profile(FoldingSetNodeID& ID) const { ID.AddInteger(pVal[i]); } -/// add_1 - This function adds a single "digit" integer, y, to the multiple +/// This function adds a single "digit" integer, y, to the multiple /// "digit" integer array, x[]. x[] is modified to reflect the addition and /// 1 is returned if there is a carry out, otherwise 0 is returned. /// @returns the carry of the addition. @@ -202,7 +202,7 @@ APInt& APInt::operator++() { return clearUnusedBits(); } -/// sub_1 - This function subtracts a single "digit" (64-bit word), y, from +/// This function subtracts a single "digit" (64-bit word), y, from /// the multi-digit integer array, x[], propagating the borrowed 1 value until /// no further borrowing is neeeded or it runs out of "digits" in x. The result /// is 1 if "borrowing" exhausted the digits in x, or 0 if x was not exhausted. @@ -231,7 +231,7 @@ APInt& APInt::operator--() { return clearUnusedBits(); } -/// add - This function adds the integer array x to the integer array Y and +/// This function adds the integer array x to the integer array Y and /// places the result in dest. /// @returns the carry out from the addition /// @brief General addition of 64-bit integer arrays @@ -680,12 +680,12 @@ bool APInt::isSplat(unsigned SplatSizeInBits) const { return *this == rotl(SplatSizeInBits); } -/// HiBits - This function returns the high "numBits" bits of this APInt. +/// This function returns the high "numBits" bits of this APInt. APInt APInt::getHiBits(unsigned numBits) const { return APIntOps::lshr(*this, BitWidth - numBits); } -/// LoBits - This function returns the low "numBits" bits of this APInt. +/// This function returns the low "numBits" bits of this APInt. APInt APInt::getLoBits(unsigned numBits) const { return APIntOps::lshr(APIntOps::shl(*this, BitWidth - numBits), BitWidth - numBits); @@ -861,7 +861,7 @@ APInt llvm::APIntOps::RoundDoubleToAPInt(double Double, unsigned width) { return isNeg ? -Tmp : Tmp; } -/// RoundToDouble - This function converts this APInt to a double. +/// This function converts this APInt to a double. /// The layout for double is as following (IEEE Standard 754): /// -------------------------------------- /// | Sign Exponent Fraction Bias | @@ -2269,9 +2269,8 @@ void APInt::toString(SmallVectorImpl &Str, unsigned Radix, std::reverse(Str.begin()+StartDig, Str.end()); } -/// toString - This returns the APInt as a std::string. Note that this is an -/// inefficient method. It is better to pass in a SmallVector/SmallString -/// to the methods above. +/// Returns the APInt as a std::string. Note that this is an inefficient method. +/// It is better to pass in a SmallVector/SmallString to the methods above. std::string APInt::toString(unsigned Radix = 10, bool Signed = true) const { SmallString<40> S; toString(S, Radix, Signed, /* formatAsCLiteral = */false); diff --git a/lib/Support/CommandLine.cpp b/lib/Support/CommandLine.cpp index af6c605..3cabc54 100644 --- a/lib/Support/CommandLine.cpp +++ b/lib/Support/CommandLine.cpp @@ -313,7 +313,7 @@ static Option *LookupNearestOption(StringRef Arg, if (RHS.empty() || !PermitValue) NearestString = OptionNames[i]; else - NearestString = std::string(OptionNames[i]) + "=" + RHS.str(); + NearestString = (Twine(OptionNames[i]) + "=" + RHS).str(); } } } @@ -784,7 +784,7 @@ class StrDupSaver : public StringSaver { std::vector Dups; public: - ~StrDupSaver() { + ~StrDupSaver() override { for (std::vector::iterator I = Dups.begin(), E = Dups.end(); I != E; ++I) { char *Dup = *I; diff --git a/lib/Support/DataStream.cpp b/lib/Support/DataStream.cpp index a44b958..c243155 100644 --- a/lib/Support/DataStream.cpp +++ b/lib/Support/DataStream.cpp @@ -54,9 +54,7 @@ class DataFileStreamer : public DataStreamer { int Fd; public: DataFileStreamer() : Fd(0) {} - virtual ~DataFileStreamer() { - close(Fd); - } + ~DataFileStreamer() override { close(Fd); } size_t GetBytes(unsigned char *buf, size_t len) override { NumStreamFetches++; return read(Fd, buf, len); diff --git a/lib/Support/Debug.cpp b/lib/Support/Debug.cpp index a88b18e..eb99242 100644 --- a/lib/Support/Debug.cpp +++ b/lib/Support/Debug.cpp @@ -114,9 +114,9 @@ static void debug_user_sig_handler(void *Cookie) { // know that debug mode is enabled and dbgs() really is a // circular_raw_ostream. If NDEBUG is defined, then dbgs() == // errs() but this will never be invoked. - llvm::circular_raw_ostream *dbgout = - static_cast(&llvm::dbgs()); - dbgout->flushBufferWithBanner(); + llvm::circular_raw_ostream &dbgout = + static_cast(llvm::dbgs()); + dbgout.flushBufferWithBanner(); } /// dbgs - Return a circular-buffered debug stream. diff --git a/lib/Support/FoldingSet.cpp b/lib/Support/FoldingSet.cpp index 80d2aef..b8538ff 100644 --- a/lib/Support/FoldingSet.cpp +++ b/lib/Support/FoldingSet.cpp @@ -51,8 +51,8 @@ bool FoldingSetNodeIDRef::operator<(FoldingSetNodeIDRef RHS) const { /// void FoldingSetNodeID::AddPointer(const void *Ptr) { // Note: this adds pointers to the hash using sizes and endianness that - // depend on the host. It doesn't matter however, because hashing on - // pointer values in inherently unstable. Nothing should depend on the + // depend on the host. It doesn't matter, however, because hashing on + // pointer values is inherently unstable. Nothing should depend on the // ordering of nodes in the folding set. Bits.append(reinterpret_cast(&Ptr), reinterpret_cast(&Ptr+1)); diff --git a/lib/Support/GraphWriter.cpp b/lib/Support/GraphWriter.cpp index fd4ce54..97aedc8 100644 --- a/lib/Support/GraphWriter.cpp +++ b/lib/Support/GraphWriter.cpp @@ -92,7 +92,7 @@ static bool ExecGraphViewer(StringRef ExecPath, std::vector &args, errs() << " done. \n"; } else { sys::ExecuteNoWait(ExecPath, args.data(), nullptr, nullptr, 0, &ErrMsg); - errs() << "Remember to erase graph file: " << Filename.str() << "\n"; + errs() << "Remember to erase graph file: " << Filename << "\n"; } return false; } @@ -140,6 +140,29 @@ bool llvm::DisplayGraph(StringRef FilenameRef, bool wait, std::string ViewerPath; GraphSession S; +#ifdef __APPLE__ + if (S.TryFindProgram("open", ViewerPath)) { + std::vector args; + args.push_back(ViewerPath.c_str()); + if (wait) + args.push_back("-W"); + args.push_back(Filename.c_str()); + args.push_back(nullptr); + errs() << "Trying 'open' program... "; + if (!ExecGraphViewer(ViewerPath, args, Filename, wait, ErrMsg)) + return false; + } +#endif + if (S.TryFindProgram("xdg-open", ViewerPath)) { + std::vector args; + args.push_back(ViewerPath.c_str()); + args.push_back(Filename.c_str()); + args.push_back(nullptr); + errs() << "Trying 'xdg-open' program... "; + if (!ExecGraphViewer(ViewerPath, args, Filename, wait, ErrMsg)) + return false; + } + // Graphviz if (S.TryFindProgram("Graphviz", ViewerPath)) { std::vector args; diff --git a/lib/Support/Host.cpp b/lib/Support/Host.cpp index 0e9a62e..726961a 100644 --- a/lib/Support/Host.cpp +++ b/lib/Support/Host.cpp @@ -182,19 +182,21 @@ static bool GetX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, #endif } -static bool OSHasAVXSupport() { +static bool GetX86XCR0(unsigned *rEAX, unsigned *rEDX) { #if defined(__GNUC__) // Check xgetbv; this uses a .byte sequence instead of the instruction // directly because older assemblers do not include support for xgetbv and // there is no easy way to conditionally compile based on the assembler used. - int rEAX, rEDX; - __asm__ (".byte 0x0f, 0x01, 0xd0" : "=a" (rEAX), "=d" (rEDX) : "c" (0)); + __asm__ (".byte 0x0f, 0x01, 0xd0" : "=a" (*rEAX), "=d" (*rEDX) : "c" (0)); + return false; #elif defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) - unsigned long long rEAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); + unsigned long long Result = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); + *rEAX = Result; + *rEDX = Result >> 32; + return false; #else - int rEAX = 0; // Ensures we return false + return true; #endif - return (rEAX & 6) == 6; } static void DetectX86FamilyModel(unsigned EAX, unsigned &Family, @@ -223,19 +225,30 @@ StringRef sys::getHostCPUName() { char c[12]; } text; - GetX86CpuIDAndInfo(0, &EAX, text.u+0, text.u+2, text.u+1); - - unsigned MaxLeaf = EAX; - bool HasSSE3 = (ECX & 0x1); - bool HasSSE41 = (ECX & 0x80000); - // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV + unsigned MaxLeaf; + GetX86CpuIDAndInfo(0, &MaxLeaf, text.u+0, text.u+2, text.u+1); + + bool HasMMX = (EDX >> 23) & 1; + bool HasSSE = (EDX >> 25) & 1; + bool HasSSE2 = (EDX >> 26) & 1; + bool HasSSE3 = (ECX >> 0) & 1; + bool HasSSSE3 = (ECX >> 9) & 1; + bool HasSSE41 = (ECX >> 19) & 1; + bool HasSSE42 = (ECX >> 20) & 1; + bool HasMOVBE = (ECX >> 22) & 1; + // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV // indicates that the AVX registers will be saved and restored on context // switch, then we have full AVX support. const unsigned AVXBits = (1 << 27) | (1 << 28); - bool HasAVX = ((ECX & AVXBits) == AVXBits) && OSHasAVXSupport(); - bool HasAVX2 = HasAVX && MaxLeaf >= 0x7 && - !GetX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX) && - (EBX & 0x20); + bool HasAVX = ((ECX & AVXBits) == AVXBits) && !GetX86XCR0(&EAX, &EDX) && + ((EAX & 0x6) == 0x6); + bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0); + bool HasLeaf7 = MaxLeaf >= 0x7 && + !GetX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); + bool HasADX = HasLeaf7 && ((EBX >> 19) & 1); + bool HasAVX2 = HasAVX && HasLeaf7 && (EBX & 0x20); + bool HasAVX512 = HasLeaf7 && HasAVX512Save && ((EBX >> 16) & 1); + GetX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); bool Em64T = (EDX >> 29) & 0x1; bool HasTBM = (ECX >> 21) & 0x1; @@ -298,6 +311,8 @@ StringRef sys::getHostCPUName() { case 9: // Intel Pentium M processor, Intel Celeron M processor model 09. case 13: // Intel Pentium M processor, Intel Celeron M processor, model // 0Dh. All processors are manufactured using the 90 nm process. + case 21: // Intel EP80579 Integrated Processor and Intel EP80579 + // Integrated Processor with Intel QuickAssist Technology return "pentium-m"; case 14: // Intel Core Duo processor, Intel Core Solo processor, model @@ -313,74 +328,85 @@ StringRef sys::getHostCPUName() { // manufactured using the 65 nm process return "core2"; - case 21: // Intel EP80579 Integrated Processor and Intel EP80579 - // Integrated Processor with Intel QuickAssist Technology - return "i686"; // FIXME: ??? - case 23: // Intel Core 2 Extreme processor, Intel Xeon processor, model // 17h. All processors are manufactured using the 45 nm process. // // 45nm: Penryn , Wolfdale, Yorkfield (XE) - // Not all Penryn processors support SSE 4.1 (such as the Pentium brand) - return HasSSE41 ? "penryn" : "core2"; + case 29: // Intel Xeon processor MP. All processors are manufactured using + // the 45 nm process. + return "penryn"; case 26: // Intel Core i7 processor and Intel Xeon processor. All // processors are manufactured using the 45 nm process. - case 29: // Intel Xeon processor MP. All processors are manufactured using - // the 45 nm process. case 30: // Intel(R) Core(TM) i7 CPU 870 @ 2.93GHz. // As found in a Summer 2010 model iMac. + case 46: // Nehalem EX + return "nehalem"; case 37: // Intel Core i7, laptop version. case 44: // Intel Core i7 processor and Intel Xeon processor. All // processors are manufactured using the 32 nm process. - case 46: // Nehalem EX case 47: // Westmere EX - return "corei7"; + return "westmere"; // SandyBridge: case 42: // Intel Core i7 processor. All processors are manufactured // using the 32 nm process. case 45: - // Not all Sandy Bridge processors support AVX (such as the Pentium - // versions instead of the i7 versions). - return HasAVX ? "corei7-avx" : "corei7"; + return "sandybridge"; // Ivy Bridge: case 58: case 62: // Ivy Bridge EP - // Not all Ivy Bridge processors support AVX (such as the Pentium - // versions instead of the i7 versions). - return HasAVX ? "core-avx-i" : "corei7"; + return "ivybridge"; // Haswell: case 60: case 63: case 69: case 70: - // Not all Haswell processors support AVX2 (such as the Pentium - // versions instead of the i7 versions). - return HasAVX2 ? "core-avx2" : "corei7"; + return "haswell"; // Broadwell: case 61: - // Not all Broadwell processors support AVX2 (such as the Pentium - // versions instead of the i7 versions). - return HasAVX2 ? "broadwell" : "corei7"; + return "broadwell"; case 28: // Most 45 nm Intel Atom processors case 38: // 45 nm Atom Lincroft case 39: // 32 nm Atom Medfield case 53: // 32 nm Atom Midview case 54: // 32 nm Atom Midview - return "atom"; + return "bonnell"; // Atom Silvermont codes from the Intel software optimization guide. case 55: case 74: case 77: - return "slm"; - - default: return (Em64T) ? "x86-64" : "i686"; + return "silvermont"; + + default: // Unknown family 6 CPU, try to guess. + if (HasAVX512) + return "knl"; + if (HasADX) + return "broadwell"; + if (HasAVX2) + return "haswell"; + if (HasAVX) + return "sandybridge"; + if (HasSSE42) + return HasMOVBE ? "silvermont" : "nehalem"; + if (HasSSE41) + return "penryn"; + if (HasSSSE3) + return HasMOVBE ? "bonnell" : "core2"; + if (Em64T) + return "x86-64"; + if (HasSSE2) + return "pentium-m"; + if (HasSSE) + return "pentium3"; + if (HasMMX) + return "pentium2"; + return "pentiumpro"; } case 15: { switch (Model) { @@ -681,7 +707,89 @@ StringRef sys::getHostCPUName() { } #endif -#if defined(__linux__) && (defined(__arm__) || defined(__aarch64__)) +#if defined(i386) || defined(__i386__) || defined(__x86__) || defined(_M_IX86)\ + || defined(__x86_64__) || defined(_M_AMD64) || defined (_M_X64) +bool sys::getHostCPUFeatures(StringMap &Features) { + unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0; + unsigned MaxLevel; + union { + unsigned u[3]; + char c[12]; + } text; + + if (GetX86CpuIDAndInfo(0, &MaxLevel, text.u+0, text.u+2, text.u+1) || + MaxLevel < 1) + return false; + + GetX86CpuIDAndInfo(1, &EAX, &EBX, &ECX, &EDX); + + Features["cmov"] = (EDX >> 15) & 1; + Features["mmx"] = (EDX >> 23) & 1; + Features["sse"] = (EDX >> 25) & 1; + Features["sse2"] = (EDX >> 26) & 1; + Features["sse3"] = (ECX >> 0) & 1; + Features["ssse3"] = (ECX >> 9) & 1; + Features["sse4.1"] = (ECX >> 19) & 1; + Features["sse4.2"] = (ECX >> 20) & 1; + + Features["pclmul"] = (ECX >> 1) & 1; + Features["cx16"] = (ECX >> 13) & 1; + Features["movbe"] = (ECX >> 22) & 1; + Features["popcnt"] = (ECX >> 23) & 1; + Features["aes"] = (ECX >> 25) & 1; + Features["rdrnd"] = (ECX >> 30) & 1; + + // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV + // indicates that the AVX registers will be saved and restored on context + // switch, then we have full AVX support. + bool HasAVX = ((ECX >> 27) & 1) && ((ECX >> 28) & 1) && + !GetX86XCR0(&EAX, &EDX) && ((EAX & 0x6) == 0x6); + Features["avx"] = HasAVX; + Features["fma"] = HasAVX && (ECX >> 12) & 1; + Features["f16c"] = HasAVX && (ECX >> 29) & 1; + + // AVX512 requires additional context to be saved by the OS. + bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0); + + unsigned MaxExtLevel; + GetX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX); + + bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 && + !GetX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); + Features["lzcnt"] = HasExtLeaf1 && ((ECX >> 5) & 1); + Features["sse4a"] = HasExtLeaf1 && ((ECX >> 6) & 1); + Features["prfchw"] = HasExtLeaf1 && ((ECX >> 8) & 1); + Features["xop"] = HasAVX && HasExtLeaf1 && ((ECX >> 11) & 1); + Features["fma4"] = HasAVX && HasExtLeaf1 && ((ECX >> 16) & 1); + Features["tbm"] = HasExtLeaf1 && ((ECX >> 21) & 1); + + bool HasLeaf7 = MaxLevel >= 7 && + !GetX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); + + // AVX2 is only supported if we have the OS save support from AVX. + Features["avx2"] = HasAVX && HasLeaf7 && (EBX >> 5) & 1; + + Features["fsgsbase"] = HasLeaf7 && ((EBX >> 0) & 1); + Features["bmi"] = HasLeaf7 && ((EBX >> 3) & 1); + Features["hle"] = HasLeaf7 && ((EBX >> 4) & 1); + Features["bmi2"] = HasLeaf7 && ((EBX >> 8) & 1); + Features["rtm"] = HasLeaf7 && ((EBX >> 11) & 1); + Features["rdseed"] = HasLeaf7 && ((EBX >> 18) & 1); + Features["adx"] = HasLeaf7 && ((EBX >> 19) & 1); + Features["sha"] = HasLeaf7 && ((EBX >> 29) & 1); + + // AVX512 is only supported if the OS supports the context save for it. + Features["avx512f"] = HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save; + Features["avx512dq"] = HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save; + Features["avx512pf"] = HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save; + Features["avx512er"] = HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save; + Features["avx512cd"] = HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save; + Features["avx512bw"] = HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save; + Features["avx512vl"] = HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save; + + return true; +} +#elif defined(__linux__) && (defined(__arm__) || defined(__aarch64__)) bool sys::getHostCPUFeatures(StringMap &Features) { // Read 1024 bytes from /proc/cpuinfo, which should contain the Features line // in all cases. diff --git a/lib/Support/Process.cpp b/lib/Support/Process.cpp index d0c1748..3571cd3 100644 --- a/lib/Support/Process.cpp +++ b/lib/Support/Process.cpp @@ -26,24 +26,6 @@ using namespace sys; //=== independent code. //===----------------------------------------------------------------------===// -/// \brief A helper function to compute the elapsed wall-time since the program -/// started. -/// -/// Note that this routine actually computes the elapsed wall time since the -/// first time it was called. However, we arrange to have it called during the -/// startup of the process to get approximately correct results. -static TimeValue getElapsedWallTime() { - static TimeValue &StartTime = *new TimeValue(TimeValue::now()); - return TimeValue::now() - StartTime; -} - -/// \brief A special global variable to ensure we call \c getElapsedWallTime -/// during global initialization of the program. -/// -/// Note that this variable is never referenced elsewhere. Doing so could -/// create race conditions during program startup or shutdown. -static volatile TimeValue DummyTimeValue = getElapsedWallTime(); - Optional Process::FindInEnvPath(const std::string& EnvName, const std::string& FileName) { diff --git a/lib/Support/Regex.cpp b/lib/Support/Regex.cpp index d3e29ac..e8344ef 100644 --- a/lib/Support/Regex.cpp +++ b/lib/Support/Regex.cpp @@ -15,6 +15,7 @@ #include "regex_impl.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" #include using namespace llvm; @@ -158,7 +159,7 @@ std::string Regex::sub(StringRef Repl, StringRef String, RefValue < Matches.size()) Res += Matches[RefValue]; else if (Error && Error->empty()) - *Error = "invalid backreference string '" + Ref.str() + "'"; + *Error = ("invalid backreference string '" + Twine(Ref) + "'").str(); break; } } diff --git a/lib/Support/Triple.cpp b/lib/Support/Triple.cpp index d4b150a..5b43ecc 100644 --- a/lib/Support/Triple.cpp +++ b/lib/Support/Triple.cpp @@ -92,7 +92,7 @@ const char *Triple::getArchTypePrefix(ArchType Kind) { case sparcv9: case sparc: return "sparc"; - case systemz: return "systemz"; + case systemz: return "s390"; case x86: case x86_64: return "x86"; @@ -1111,7 +1111,7 @@ const char *Triple::getARMCPUForArch(StringRef MArch) const { .Cases("v7m", "v7-m", "cortex-m3") .Cases("v7em", "v7e-m", "cortex-m4") .Cases("v8", "v8a", "v8-a", "cortex-a53") - .Cases("v8.1a", "v8.1-a", "generic-armv8.1-a") + .Cases("v8.1a", "v8.1-a", "generic") .Default(nullptr); else result = llvm::StringSwitch(MArch) diff --git a/lib/Support/Unix/Signals.inc b/lib/Support/Unix/Signals.inc index a9b48e0..057bcab1 100644 --- a/lib/Support/Unix/Signals.inc +++ b/lib/Support/Unix/Signals.inc @@ -486,12 +486,12 @@ void llvm::sys::DisableSystemDialogsOnCrash() {} /// PrintStackTraceOnErrorSignal - When an error signal (such as SIGABRT or /// SIGSEGV) is delivered to the process, print a stack trace and then exit. -void llvm::sys::PrintStackTraceOnErrorSignal() { +void llvm::sys::PrintStackTraceOnErrorSignal(bool DisableCrashReporting) { AddSignalHandler(PrintStackTraceSignalHandler, nullptr); #if defined(__APPLE__) && defined(ENABLE_CRASH_OVERRIDES) // Environment variable to disable any kind of crash dialog. - if (getenv("LLVM_DISABLE_CRASH_REPORT")) { + if (DisableCrashReporting || getenv("LLVM_DISABLE_CRASH_REPORT")) { mach_port_t self = mach_task_self(); exception_mask_t mask = EXC_MASK_CRASH; diff --git a/lib/Support/Windows/Path.inc b/lib/Support/Windows/Path.inc index d558ff5..b5523aa 100644 --- a/lib/Support/Windows/Path.inc +++ b/lib/Support/Windows/Path.inc @@ -261,6 +261,7 @@ std::error_code rename(const Twine &from, const Twine &to) { MOVEFILE_COPY_ALLOWED | MOVEFILE_REPLACE_EXISTING)) return std::error_code(); DWORD LastError = ::GetLastError(); + ec = windows_error(LastError); if (LastError != ERROR_ACCESS_DENIED) break; // Retry MoveFile() at ACCESS_DENIED. diff --git a/lib/Support/Windows/Signals.inc b/lib/Support/Windows/Signals.inc index de6bf1c..f070111 100644 --- a/lib/Support/Windows/Signals.inc +++ b/lib/Support/Windows/Signals.inc @@ -389,7 +389,7 @@ void sys::DisableSystemDialogsOnCrash() { /// PrintStackTraceOnErrorSignal - When an error signal (such as SIBABRT or /// SIGSEGV) is delivered to the process, print a stack trace and then exit. -void sys::PrintStackTraceOnErrorSignal() { +void sys::PrintStackTraceOnErrorSignal(bool DisableCrashReporting) { DisableSystemDialogsOnCrash(); RegisterHandler(); LeaveCriticalSection(&CriticalSection); diff --git a/lib/Support/Windows/TimeValue.inc b/lib/Support/Windows/TimeValue.inc index 0223ab4..b90b4f1 100644 --- a/lib/Support/Windows/TimeValue.inc +++ b/lib/Support/Windows/TimeValue.inc @@ -47,6 +47,7 @@ std::string TimeValue::str() const { __time64_t OurTime = this->toEpochTime(); int Error = ::_localtime64_s(&Storage, &OurTime); assert(!Error); + (void)Error; LT = &Storage; #endif diff --git a/lib/Support/raw_ostream.cpp b/lib/Support/raw_ostream.cpp index 051e2dd..6f9f910 100644 --- a/lib/Support/raw_ostream.cpp +++ b/lib/Support/raw_ostream.cpp @@ -487,51 +487,53 @@ void format_object_base::home() { // raw_fd_ostream //===----------------------------------------------------------------------===// -raw_fd_ostream::raw_fd_ostream(StringRef Filename, std::error_code &EC, - sys::fs::OpenFlags Flags) - : Error(false), UseAtomicWrites(false), pos(0) { - EC = std::error_code(); +static int getFD(StringRef Filename, std::error_code &EC, + sys::fs::OpenFlags Flags) { // Handle "-" as stdout. Note that when we do this, we consider ourself // the owner of stdout. This means that we can do things like close the // file descriptor when we're done and set the "binary" flag globally. if (Filename == "-") { - FD = STDOUT_FILENO; + EC = std::error_code(); // If user requested binary then put stdout into binary mode if // possible. if (!(Flags & sys::fs::F_Text)) sys::ChangeStdoutToBinary(); - // Close stdout when we're done, to detect any output errors. - ShouldClose = true; - return; + return STDOUT_FILENO; } + int FD; EC = sys::fs::openFileForWrite(Filename, FD, Flags); + if (EC) + return -1; - if (EC) { - ShouldClose = false; - return; - } - - // Ok, we successfully opened the file, so it'll need to be closed. - ShouldClose = true; + return FD; } -/// raw_fd_ostream ctor - FD is the file descriptor that this writes to. If -/// ShouldClose is true, this closes the file when the stream is destroyed. +raw_fd_ostream::raw_fd_ostream(StringRef Filename, std::error_code &EC, + sys::fs::OpenFlags Flags) + : raw_fd_ostream(getFD(Filename, EC, Flags), true) {} + +/// FD is the file descriptor that this writes to. If ShouldClose is true, this +/// closes the file when the stream is destroyed. raw_fd_ostream::raw_fd_ostream(int fd, bool shouldClose, bool unbuffered) - : raw_ostream(unbuffered), FD(fd), - ShouldClose(shouldClose), Error(false), UseAtomicWrites(false) { -#ifdef O_BINARY - // Setting STDOUT to binary mode is necessary in Win32 - // to avoid undesirable linefeed conversion. - // Don't touch STDERR, or w*printf() (in assert()) would barf wide chars. - if (fd == STDOUT_FILENO) - setmode(fd, O_BINARY); -#endif + : raw_pwrite_stream(unbuffered), FD(fd), ShouldClose(shouldClose), + Error(false), UseAtomicWrites(false) { + if (FD < 0 ) { + ShouldClose = false; + return; + } // Get the starting position. off_t loc = ::lseek(FD, 0, SEEK_CUR); - if (loc == (off_t)-1) +#ifdef LLVM_ON_WIN32 + // MSVCRT's _lseek(SEEK_CUR) doesn't return -1 for pipes. + sys::fs::file_status Status; + std::error_code EC = status(FD, Status); + SupportsSeeking = !EC && Status.type() == sys::fs::file_type::regular_file; +#else + SupportsSeeking = loc != (off_t)-1; +#endif + if (!SupportsSeeking) pos = 0; else pos = static_cast(loc); @@ -623,11 +625,18 @@ void raw_fd_ostream::close() { uint64_t raw_fd_ostream::seek(uint64_t off) { flush(); pos = ::lseek(FD, off, SEEK_SET); - if (pos != off) + if (pos == (uint64_t)-1) error_detected(); return pos; } +void raw_fd_ostream::pwrite(const char *Ptr, size_t Size, uint64_t Offset) { + uint64_t Pos = tell(); + seek(Offset); + write(Ptr, Size); + seek(Pos); +} + size_t raw_fd_ostream::preferred_buffer_size() const { #if !defined(_MSC_VER) && !defined(__MINGW32__) && !defined(__minix) // Windows and Minix have no st_blksize. @@ -708,7 +717,9 @@ raw_ostream &llvm::outs() { // Set buffer settings to model stdout behavior. // Delete the file descriptor when the program exits, forcing error // detection. If you don't want this behavior, don't use outs(). - static raw_fd_ostream S(STDOUT_FILENO, true); + std::error_code EC; + static raw_fd_ostream S("-", EC, sys::fs::F_None); + assert(!EC); return S; } @@ -749,7 +760,14 @@ void raw_string_ostream::write_impl(const char *Ptr, size_t Size) { // capacity. This allows raw_ostream to write directly into the correct place, // and we only need to set the vector size when the data is flushed. +raw_svector_ostream::raw_svector_ostream(SmallVectorImpl &O, unsigned) + : OS(O) {} + raw_svector_ostream::raw_svector_ostream(SmallVectorImpl &O) : OS(O) { + init(); +} + +void raw_svector_ostream::init() { // Set up the initial external buffer. We make sure that the buffer has at // least 128 bytes free; raw_ostream itself only requires 64, but we want to // make sure that we don't grow the buffer unnecessarily on destruction (when @@ -763,6 +781,17 @@ raw_svector_ostream::~raw_svector_ostream() { flush(); } +void raw_svector_ostream::pwrite(const char *Ptr, size_t Size, + uint64_t Offset) { + flush(); + + uint64_t End = Offset + Size; + if (End > OS.size()) + OS.resize(End); + + memcpy(OS.begin() + Offset, Ptr, Size); +} + /// resync - This is called when the SmallVector we're appending to is changed /// outside of the raw_svector_ostream's control. It is only safe to do this /// if the raw_svector_ostream has previously been flushed. @@ -817,3 +846,5 @@ void raw_null_ostream::write_impl(const char *Ptr, size_t Size) { uint64_t raw_null_ostream::current_pos() const { return 0; } + +void raw_null_ostream::pwrite(const char *Ptr, size_t Size, uint64_t Offset) {} diff --git a/lib/TableGen/Record.cpp b/lib/TableGen/Record.cpp index 4ae9903..8a8f0ee 100644 --- a/lib/TableGen/Record.cpp +++ b/lib/TableGen/Record.cpp @@ -2040,7 +2040,7 @@ RecordKeeper::getAllDerivedDefinitions(const std::string &ClassName) const { /// to CurRec's name. Init *llvm::QualifyName(Record &CurRec, MultiClass *CurMultiClass, Init *Name, const std::string &Scoper) { - RecTy *Type = dyn_cast(Name)->getType(); + RecTy *Type = cast(Name)->getType(); BinOpInit *NewName = BinOpInit::get(BinOpInit::STRCONCAT, diff --git a/lib/TableGen/TGLexer.h b/lib/TableGen/TGLexer.h index 1f750fc..cbc30be 100644 --- a/lib/TableGen/TGLexer.h +++ b/lib/TableGen/TGLexer.h @@ -87,8 +87,7 @@ private: public: TGLexer(SourceMgr &SrcMgr); - ~TGLexer() {} - + tgtok::TokKind Lex() { return CurCode = LexToken(); } diff --git a/lib/Target/AArch64/AArch64.td b/lib/Target/AArch64/AArch64.td index bb3db4b..9a7d6c8 100644 --- a/lib/Target/AArch64/AArch64.td +++ b/lib/Target/AArch64/AArch64.td @@ -32,9 +32,6 @@ def FeatureCrypto : SubtargetFeature<"crypto", "HasCrypto", "true", def FeatureCRC : SubtargetFeature<"crc", "HasCRC", "true", "Enable ARMv8 CRC-32 checksum instructions">; -def FeatureV8_1a : SubtargetFeature<"v8.1a", "HasV8_1a", "true", - "Enable ARMv8.1a extensions", [FeatureCRC]>; - /// Cyclone has register move instructions which are "free". def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true", "Has zero-cycle register moves">; @@ -44,6 +41,13 @@ def FeatureZCZeroing : SubtargetFeature<"zcz", "HasZeroCycleZeroing", "true", "Has zero-cycle zeroing instructions">; //===----------------------------------------------------------------------===// +// Architectures. +// + +def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true", + "Support ARM v8.1a instructions", [FeatureCRC]>; + +//===----------------------------------------------------------------------===// // Register File Description //===----------------------------------------------------------------------===// @@ -92,10 +96,6 @@ def : ProcessorModel<"generic", NoSchedModel, [FeatureFPARMv8, FeatureNEON, FeatureCRC]>; -def : ProcessorModel<"generic-armv8.1-a", NoSchedModel, [FeatureV8_1a, - FeatureNEON, - FeatureCrypto]>; - def : ProcessorModel<"cortex-a53", CortexA53Model, [ProcA53]>; def : ProcessorModel<"cortex-a57", CortexA57Model, [ProcA57]>; // FIXME: Cortex-A72 is currently modelled as an Cortex-A57. @@ -123,12 +123,14 @@ def AppleAsmParserVariant : AsmParserVariant { // AsmWriter bits get associated with the correct class. def GenericAsmWriter : AsmWriter { string AsmWriterClassName = "InstPrinter"; + int PassSubtarget = 1; int Variant = 0; bit isMCAsmWriter = 1; } def AppleAsmWriter : AsmWriter { let AsmWriterClassName = "AppleInstPrinter"; + int PassSubtarget = 1; int Variant = 1; int isMCAsmWriter = 1; } diff --git a/lib/Target/AArch64/AArch64AsmPrinter.cpp b/lib/Target/AArch64/AArch64AsmPrinter.cpp index 1b4483a..0821cff 100644 --- a/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -131,29 +131,6 @@ void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) { OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols); SM.serializeToStackMapSection(); } - - // Emit a .data.rel section containing any stubs that were created. - if (TT.isOSBinFormatELF()) { - const TargetLoweringObjectFileELF &TLOFELF = - static_cast(getObjFileLowering()); - - MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo(); - - // Output stubs for external and common global variables. - MachineModuleInfoELF::SymbolListTy Stubs = MMIELF.GetGVStubList(); - if (!Stubs.empty()) { - OutStreamer.SwitchSection(TLOFELF.getDataRelSection()); - const DataLayout *TD = TM.getDataLayout(); - - for (unsigned i = 0, e = Stubs.size(); i != e; ++i) { - OutStreamer.EmitLabel(Stubs[i].first); - OutStreamer.EmitSymbolValue(Stubs[i].second.getPointer(), - TD->getPointerSize(0)); - } - Stubs.clear(); - } - } - } MachineLocation @@ -371,8 +348,8 @@ void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI, assert(NOps == 4); OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: "; // cast away const; DIetc do not take const operands for some reason. - DIVariable V(const_cast(MI->getOperand(NOps - 1).getMetadata())); - OS << V.getName(); + OS << cast(MI->getOperand(NOps - 2).getMetadata()) + ->getName(); OS << " <- "; // Frame address. Currently handles register +- offset only. assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm()); diff --git a/lib/Target/AArch64/AArch64CollectLOH.cpp b/lib/Target/AArch64/AArch64CollectLOH.cpp index 568f258..efdb2e3 100644 --- a/lib/Target/AArch64/AArch64CollectLOH.cpp +++ b/lib/Target/AArch64/AArch64CollectLOH.cpp @@ -328,7 +328,7 @@ static void initReachingDef(const MachineFunction &MF, const uint32_t *PreservedRegs = MO.getRegMask(); // Set generated regs. - for (const auto Entry : RegToId) { + for (const auto &Entry : RegToId) { unsigned Reg = Entry.second; // Use the global register ID when querying APIs external to this // pass. diff --git a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index 41b1132..c2470f7 100644 --- a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -698,12 +698,15 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, return expandMOVImm(MBB, MBBI, 32); case AArch64::MOVi64imm: return expandMOVImm(MBB, MBBI, 64); - case AArch64::RET_ReallyLR: - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET)) - .addReg(AArch64::LR); + case AArch64::RET_ReallyLR: { + MachineInstrBuilder MIB = + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET)) + .addReg(AArch64::LR); + transferImpOps(MI, MIB, MIB); MI.eraseFromParent(); return true; } + } return false; } diff --git a/lib/Target/AArch64/AArch64FastISel.cpp b/lib/Target/AArch64/AArch64FastISel.cpp index 99cb641..c3f6859 100644 --- a/lib/Target/AArch64/AArch64FastISel.cpp +++ b/lib/Target/AArch64/AArch64FastISel.cpp @@ -1917,7 +1917,8 @@ bool AArch64FastISel::selectLoad(const Instruction *I) { // could select it. Emit a copy to subreg if necessary. FastISel will remove // it when it selects the integer extend. unsigned Reg = lookUpRegForValue(IntExtVal); - if (!Reg) { + auto *MI = MRI.getUniqueVRegDef(Reg); + if (!MI) { if (RetVT == MVT::i64 && VT <= MVT::i32) { if (WantZExt) { // Delete the last emitted instruction from emitLoad (SUBREG_TO_REG). @@ -1935,10 +1936,7 @@ bool AArch64FastISel::selectLoad(const Instruction *I) { // The integer extend has already been emitted - delete all the instructions // that have been emitted by the integer extend lowering code and use the // result from the load instruction directly. - while (Reg) { - auto *MI = MRI.getUniqueVRegDef(Reg); - if (!MI) - break; + while (MI) { Reg = 0; for (auto &Opnd : MI->uses()) { if (Opnd.isReg()) { @@ -1947,6 +1945,9 @@ bool AArch64FastISel::selectLoad(const Instruction *I) { } } MI->eraseFromParent(); + MI = nullptr; + if (Reg) + MI = MRI.getUniqueVRegDef(Reg); } updateValueMap(IntExtVal, ResultReg); return true; @@ -3034,6 +3035,11 @@ bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT, // Copy all of the result registers out of their specified physreg. MVT CopyVT = RVLocs[0].getValVT(); + + // TODO: Handle big-endian results + if (CopyVT.isVector() && !Subtarget->isLittleEndian()) + return false; + unsigned ResultReg = createResultReg(TLI.getRegClassFor(CopyVT)); BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), ResultReg) diff --git a/lib/Target/AArch64/AArch64FrameLowering.cpp b/lib/Target/AArch64/AArch64FrameLowering.cpp index 84bf317..bd2af16 100644 --- a/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -9,6 +9,82 @@ // // This file contains the AArch64 implementation of TargetFrameLowering class. // +// On AArch64, stack frames are structured as follows: +// +// The stack grows downward. +// +// All of the individual frame areas on the frame below are optional, i.e. it's +// possible to create a function so that the particular area isn't present +// in the frame. +// +// At function entry, the "frame" looks as follows: +// +// | | Higher address +// |-----------------------------------| +// | | +// | arguments passed on the stack | +// | | +// |-----------------------------------| <- sp +// | | Lower address +// +// +// After the prologue has run, the frame has the following general structure. +// Note that this doesn't depict the case where a red-zone is used. Also, +// technically the last frame area (VLAs) doesn't get created until in the +// main function body, after the prologue is run. However, it's depicted here +// for completeness. +// +// | | Higher address +// |-----------------------------------| +// | | +// | arguments passed on the stack | +// | | +// |-----------------------------------| +// | | +// | prev_fp, prev_lr | +// | (a.k.a. "frame record") | +// |-----------------------------------| <- fp(=x29) +// | | +// | other callee-saved registers | +// | | +// |-----------------------------------| +// |.empty.space.to.make.part.below....| +// |.aligned.in.case.it.needs.more.than| (size of this area is unknown at +// |.the.standard.16-byte.alignment....| compile time; if present) +// |-----------------------------------| +// | | +// | local variables of fixed size | +// | including spill slots | +// |-----------------------------------| <- bp(not defined by ABI, +// |.variable-sized.local.variables....| LLVM chooses X19) +// |.(VLAs)............................| (size of this area is unknown at +// |...................................| compile time) +// |-----------------------------------| <- sp +// | | Lower address +// +// +// To access the data in a frame, at-compile time, a constant offset must be +// computable from one of the pointers (fp, bp, sp) to access it. The size +// of the areas with a dotted background cannot be computed at compile-time +// if they are present, making it required to have all three of fp, bp and +// sp to be set up to be able to access all contents in the frame areas, +// assuming all of the frame areas are non-empty. +// +// For most functions, some of the frame areas are empty. For those functions, +// it may not be necessary to set up fp or bp: +// * A base pointer is definitly needed when there are both VLAs and local +// variables with more-than-default alignment requirements. +// * A frame pointer is definitly needed when there are local variables with +// more-than-default alignment requirements. +// +// In some cases when a base pointer is not strictly needed, it is generated +// anyway when offsets from the frame pointer to access local variables become +// so large that the offset can't be encoded in the immediate fields of loads +// or stores. +// +// FIXME: also explain the redzone concept. +// FIXME: also explain the concept of reserved call frames. +// //===----------------------------------------------------------------------===// #include "AArch64FrameLowering.h" @@ -39,26 +115,6 @@ static cl::opt EnableRedZone("aarch64-redzone", STATISTIC(NumRedZoneFunctions, "Number of functions using red zone"); -static unsigned estimateStackSize(MachineFunction &MF) { - const MachineFrameInfo *FFI = MF.getFrameInfo(); - int Offset = 0; - for (int i = FFI->getObjectIndexBegin(); i != 0; ++i) { - int FixedOff = -FFI->getObjectOffset(i); - if (FixedOff > Offset) - Offset = FixedOff; - } - for (unsigned i = 0, e = FFI->getObjectIndexEnd(); i != e; ++i) { - if (FFI->isDeadObjectIndex(i)) - continue; - Offset += FFI->getObjectSize(i); - unsigned Align = FFI->getObjectAlignment(i); - // Adjust to alignment boundary - Offset = (Offset + Align - 1) / Align * Align; - } - // This does not include the 16 bytes used for fp and lr. - return (unsigned)Offset; -} - bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const { if (!EnableRedZone) return false; @@ -83,16 +139,10 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const { /// pointer register. bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const { const MachineFrameInfo *MFI = MF.getFrameInfo(); - -#ifndef NDEBUG const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); - assert(!RegInfo->needsStackRealignment(MF) && - "No stack realignment on AArch64!"); -#endif - return (MFI->hasCalls() || MFI->hasVarSizedObjects() || MFI->isFrameAddressTaken() || MFI->hasStackMap() || - MFI->hasPatchPoint()); + MFI->hasPatchPoint() || RegInfo->needsStackRealignment(MF)); } /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is @@ -288,11 +338,48 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const { AFI->setLocalStackSize(NumBytes); // Allocate space for the rest of the frame. - if (NumBytes) { - // If we're a leaf function, try using the red zone. - if (!canUseRedZone(MF)) - emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII, - MachineInstr::FrameSetup); + + const unsigned Alignment = MFI->getMaxAlignment(); + const bool NeedsRealignment = (Alignment > 16); + unsigned scratchSPReg = AArch64::SP; + if (NeedsRealignment) { + // Use the first callee-saved register as a scratch register + assert(MF.getRegInfo().isPhysRegUsed(AArch64::X9) && + "No scratch register to align SP!"); + scratchSPReg = AArch64::X9; + } + + // If we're a leaf function, try using the red zone. + if (NumBytes && !canUseRedZone(MF)) + // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have + // the correct value here, as NumBytes also includes padding bytes, + // which shouldn't be counted here. + emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII, + MachineInstr::FrameSetup); + + assert(!(NeedsRealignment && NumBytes==0) && + "NumBytes should never be 0 when realignment is needed"); + + if (NumBytes && NeedsRealignment) { + const unsigned NrBitsToZero = countTrailingZeros(Alignment); + assert(NrBitsToZero > 1); + assert(scratchSPReg != AArch64::SP); + + // SUB X9, SP, NumBytes + // -- X9 is temporary register, so shouldn't contain any live data here, + // -- free to use. This is already produced by emitFrameOffset above. + // AND SP, X9, 0b11111...0000 + // The logical immediates have a non-trivial encoding. The following + // formula computes the encoded immediate with all ones but + // NrBitsToZero zero bits as least significant bits. + uint32_t andMaskEncoded = + (1 <<12) // = N + | ((64-NrBitsToZero) << 6) // immr + | ((64-NrBitsToZero-1) << 0) // imms + ; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) + .addReg(scratchSPReg, RegState::Kill) + .addImm(andMaskEncoded); } // If we need a base pointer, set it up here. It's whatever the value of the @@ -302,15 +389,15 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const { // FIXME: Clarify FrameSetup flags here. // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is // needed. - // - if (RegInfo->hasBasePointer(MF)) - TII->copyPhysReg(MBB, MBBI, DL, AArch64::X19, AArch64::SP, false); + if (RegInfo->hasBasePointer(MF)) { + TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP, + false); + } if (needsFrameMoves) { const DataLayout *TD = MF.getTarget().getDataLayout(); const int StackGrowth = -TD->getPointerSize(0); unsigned FramePtr = RegInfo->getFrameRegister(MF); - // An example of the prologue: // // .globl __foo @@ -460,7 +547,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, if (MF.getFunction()->getCallingConv() == CallingConv::GHC) return; - // Initial and residual are named for consitency with the prologue. Note that + // Initial and residual are named for consistency with the prologue. Note that // in the epilogue, the residual adjustment is executed first. uint64_t ArgumentPopSize = 0; if (RetOpcode == AArch64::TCRETURNdi || RetOpcode == AArch64::TCRETURNri) { @@ -571,9 +658,9 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, bool isFixed = MFI->isFixedObjectIndex(FI); // Use frame pointer to reference fixed objects. Use it for locals if - // there are VLAs (and thus the SP isn't reliable as a base). - // Make sure useFPForScavengingIndex() does the right thing for the emergency - // spill slot. + // there are VLAs or a dynamically realigned SP (and thus the SP isn't + // reliable as a base). Make sure useFPForScavengingIndex() does the + // right thing for the emergency spill slot. bool UseFP = false; if (AFI->hasStackFrame()) { // Note: Keeping the following as multiple 'if' statements rather than @@ -582,7 +669,8 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, // Argument access should always use the FP. if (isFixed) { UseFP = hasFP(MF); - } else if (hasFP(MF) && !RegInfo->hasBasePointer(MF)) { + } else if (hasFP(MF) && !RegInfo->hasBasePointer(MF) && + !RegInfo->needsStackRealignment(MF)) { // Use SP or FP, whichever gives us the best chance of the offset // being in range for direct access. If the FPOffset is positive, // that'll always be best, as the SP will be even further away. @@ -598,6 +686,10 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, } } + assert((isFixed || !RegInfo->needsStackRealignment(MF) || !UseFP) && + "In the presence of dynamic stack pointer realignment, " + "non-argument objects cannot be accessed through the frame pointer"); + if (UseFP) { FrameReg = RegInfo->getFrameRegister(MF); return FPOffset; @@ -695,6 +787,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters( if (StrOpc == AArch64::STPDpre || StrOpc == AArch64::STPXpre) MIB.addReg(AArch64::SP, RegState::Define); + MBB.addLiveIn(Reg1); + MBB.addLiveIn(Reg2); MIB.addReg(Reg2, getPrologueDeath(MF, Reg2)) .addReg(Reg1, getPrologueDeath(MF, Reg1)) .addReg(AArch64::SP) @@ -794,6 +888,9 @@ void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan( if (RegInfo->hasBasePointer(MF)) MRI->setPhysRegUsed(RegInfo->getBaseRegister()); + if (RegInfo->needsStackRealignment(MF) && !RegInfo->hasBasePointer(MF)) + MRI->setPhysRegUsed(AArch64::X9); + // If any callee-saved registers are used, the frame cannot be eliminated. unsigned NumGPRSpilled = 0; unsigned NumFPRSpilled = 0; @@ -867,7 +964,8 @@ void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan( // The CSR spill slots have not been allocated yet, so estimateStackSize // won't include them. MachineFrameInfo *MFI = MF.getFrameInfo(); - unsigned CFSize = estimateStackSize(MF) + 8 * (NumGPRSpilled + NumFPRSpilled); + unsigned CFSize = + MFI->estimateStackSize(MF) + 8 * (NumGPRSpilled + NumFPRSpilled); DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n"); bool BigStack = (CFSize >= 256); if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF)) diff --git a/lib/Target/AArch64/AArch64FrameLowering.h b/lib/Target/AArch64/AArch64FrameLowering.h index df3875f..1439bf3 100644 --- a/lib/Target/AArch64/AArch64FrameLowering.h +++ b/lib/Target/AArch64/AArch64FrameLowering.h @@ -22,7 +22,7 @@ class AArch64FrameLowering : public TargetFrameLowering { public: explicit AArch64FrameLowering() : TargetFrameLowering(StackGrowsDown, 16, 0, 16, - false /*StackRealignable*/) {} + true /*StackRealignable*/) {} void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, diff --git a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 0a47dcb..f75700d 100644 --- a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -848,7 +848,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size, // MOV X0, WideImmediate // LDR X2, [BaseReg, X0] if (isa(RHS)) { - int64_t ImmOff = (int64_t)dyn_cast(RHS)->getZExtValue(); + int64_t ImmOff = (int64_t)cast(RHS)->getZExtValue(); unsigned Scale = Log2_32(Size); // Skip the immediate can be seleced by load/store addressing mode. // Also skip the immediate can be encoded by a single ADD (SUB is also diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index 0c0e856..90a5e5e 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -281,14 +281,39 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); - // f16 is storage-only, so we promote operations to f32 if we know this is - // valid, and ignore them otherwise. The operations not mentioned here will - // fail to select, but this is not a major problem as no source language - // should be emitting native f16 operations yet. - setOperationAction(ISD::FADD, MVT::f16, Promote); - setOperationAction(ISD::FDIV, MVT::f16, Promote); - setOperationAction(ISD::FMUL, MVT::f16, Promote); - setOperationAction(ISD::FSUB, MVT::f16, Promote); + // f16 is a storage-only type, always promote it to f32. + setOperationAction(ISD::SETCC, MVT::f16, Promote); + setOperationAction(ISD::BR_CC, MVT::f16, Promote); + setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); + setOperationAction(ISD::SELECT, MVT::f16, Promote); + setOperationAction(ISD::FADD, MVT::f16, Promote); + setOperationAction(ISD::FSUB, MVT::f16, Promote); + setOperationAction(ISD::FMUL, MVT::f16, Promote); + setOperationAction(ISD::FDIV, MVT::f16, Promote); + setOperationAction(ISD::FREM, MVT::f16, Promote); + setOperationAction(ISD::FMA, MVT::f16, Promote); + setOperationAction(ISD::FNEG, MVT::f16, Promote); + setOperationAction(ISD::FABS, MVT::f16, Promote); + setOperationAction(ISD::FCEIL, MVT::f16, Promote); + setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); + setOperationAction(ISD::FCOS, MVT::f16, Promote); + setOperationAction(ISD::FFLOOR, MVT::f16, Promote); + setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); + setOperationAction(ISD::FPOW, MVT::f16, Promote); + setOperationAction(ISD::FPOWI, MVT::f16, Promote); + setOperationAction(ISD::FRINT, MVT::f16, Promote); + setOperationAction(ISD::FSIN, MVT::f16, Promote); + setOperationAction(ISD::FSINCOS, MVT::f16, Promote); + setOperationAction(ISD::FSQRT, MVT::f16, Promote); + setOperationAction(ISD::FEXP, MVT::f16, Promote); + setOperationAction(ISD::FEXP2, MVT::f16, Promote); + setOperationAction(ISD::FLOG, MVT::f16, Promote); + setOperationAction(ISD::FLOG2, MVT::f16, Promote); + setOperationAction(ISD::FLOG10, MVT::f16, Promote); + setOperationAction(ISD::FROUND, MVT::f16, Promote); + setOperationAction(ISD::FTRUNC, MVT::f16, Promote); + setOperationAction(ISD::FMINNUM, MVT::f16, Promote); + setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); // v4f16 is also a storage-only type, so promote it to v4f32 when that is // known to be safe. @@ -481,6 +506,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // Enable TBZ/TBNZ MaskAndBranchFoldingIsLegal = true; + EnableExtLdPromotion = true; setMinFunctionAlignment(2); @@ -1557,6 +1583,14 @@ SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op, if (Op.getOperand(0).getValueType().isVector()) return LowerVectorFP_TO_INT(Op, DAG); + // f16 conversions are promoted to f32. + if (Op.getOperand(0).getValueType() == MVT::f16) { + SDLoc dl(Op); + return DAG.getNode( + Op.getOpcode(), dl, Op.getValueType(), + DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0))); + } + if (Op.getOperand(0).getValueType() != MVT::f128) { // It's legal except when f128 is involved return Op; @@ -1606,6 +1640,15 @@ SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op, if (Op.getValueType().isVector()) return LowerVectorINT_TO_FP(Op, DAG); + // f16 conversions are promoted to f32. + if (Op.getValueType() == MVT::f16) { + SDLoc dl(Op); + return DAG.getNode( + ISD::FP_ROUND, dl, MVT::f16, + DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)), + DAG.getIntPtrConstant(0)); + } + // i128 conversions are libcalls. if (Op.getOperand(0).getValueType() == MVT::i128) return SDValue(); @@ -2701,8 +2744,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, DAG.getConstant(Outs[i].Flags.getByValSize(), MVT::i64); SDValue Cpy = DAG.getMemcpy( Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(), - /*isVol = */ false, - /*AlwaysInline = */ false, DstInfo, MachinePointerInfo()); + /*isVol = */ false, /*AlwaysInline = */ false, + /*isTailCall = */ false, + DstInfo, MachinePointerInfo()); MemOpChains.push_back(Cpy); } else { @@ -3514,49 +3558,10 @@ static bool selectCCOpsAreFMaxCompatible(SDValue Cmp, SDValue Result) { return Result->getOpcode() == ISD::FP_EXTEND && Result->getOperand(0) == Cmp; } -SDValue AArch64TargetLowering::LowerSELECT(SDValue Op, - SelectionDAG &DAG) const { - SDValue CC = Op->getOperand(0); - SDValue TVal = Op->getOperand(1); - SDValue FVal = Op->getOperand(2); - SDLoc DL(Op); - - unsigned Opc = CC.getOpcode(); - // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select - // instruction. - if (CC.getResNo() == 1 && - (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || - Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { - // Only lower legal XALUO ops. - if (!DAG.getTargetLoweringInfo().isTypeLegal(CC->getValueType(0))) - return SDValue(); - - AArch64CC::CondCode OFCC; - SDValue Value, Overflow; - std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CC.getValue(0), DAG); - SDValue CCVal = DAG.getConstant(OFCC, MVT::i32); - - return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal, - CCVal, Overflow); - } - - if (CC.getOpcode() == ISD::SETCC) - return DAG.getSelectCC(DL, CC.getOperand(0), CC.getOperand(1), TVal, FVal, - cast(CC.getOperand(2))->get()); - else - return DAG.getSelectCC(DL, CC, DAG.getConstant(0, CC.getValueType()), TVal, - FVal, ISD::SETNE); -} - -SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op, +SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, + SDValue RHS, SDValue TVal, + SDValue FVal, SDLoc dl, SelectionDAG &DAG) const { - ISD::CondCode CC = cast(Op.getOperand(4))->get(); - SDValue LHS = Op.getOperand(0); - SDValue RHS = Op.getOperand(1); - SDValue TVal = Op.getOperand(2); - SDValue FVal = Op.getOperand(3); - SDLoc dl(Op); - // Handle f128 first, because it will result in a comparison of some RTLIB // call result against zero. if (LHS.getValueType() == MVT::f128) { @@ -3664,14 +3669,14 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op, SDValue CCVal; SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); - EVT VT = Op.getValueType(); + EVT VT = TVal.getValueType(); return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); } // Now we know we're dealing with FP values. assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); assert(LHS.getValueType() == RHS.getValueType()); - EVT VT = Op.getValueType(); + EVT VT = TVal.getValueType(); // Try to match this select into a max/min operation, which have dedicated // opcode in the instruction set. @@ -3732,6 +3737,58 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op, return CS1; } +SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op, + SelectionDAG &DAG) const { + ISD::CondCode CC = cast(Op.getOperand(4))->get(); + SDValue LHS = Op.getOperand(0); + SDValue RHS = Op.getOperand(1); + SDValue TVal = Op.getOperand(2); + SDValue FVal = Op.getOperand(3); + SDLoc DL(Op); + return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG); +} + +SDValue AArch64TargetLowering::LowerSELECT(SDValue Op, + SelectionDAG &DAG) const { + SDValue CCVal = Op->getOperand(0); + SDValue TVal = Op->getOperand(1); + SDValue FVal = Op->getOperand(2); + SDLoc DL(Op); + + unsigned Opc = CCVal.getOpcode(); + // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select + // instruction. + if (CCVal.getResNo() == 1 && + (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || + Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { + // Only lower legal XALUO ops. + if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0))) + return SDValue(); + + AArch64CC::CondCode OFCC; + SDValue Value, Overflow; + std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG); + SDValue CCVal = DAG.getConstant(OFCC, MVT::i32); + + return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal, + CCVal, Overflow); + } + + // Lower it the same way as we would lower a SELECT_CC node. + ISD::CondCode CC; + SDValue LHS, RHS; + if (CCVal.getOpcode() == ISD::SETCC) { + LHS = CCVal.getOperand(0); + RHS = CCVal.getOperand(1); + CC = cast(CCVal->getOperand(2))->get(); + } else { + LHS = CCVal; + RHS = DAG.getConstant(0, CCVal.getValueType()); + CC = ISD::SETNE; + } + return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG); +} + SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { // Jump table entries as PC relative offsets. No additional tweaking @@ -3920,7 +3977,7 @@ SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op, return DAG.getMemcpy(Op.getOperand(0), SDLoc(Op), Op.getOperand(1), Op.getOperand(2), DAG.getConstant(VaListSize, MVT::i32), - 8, false, false, MachinePointerInfo(DestSV), + 8, false, false, false, MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV)); } @@ -4989,7 +5046,7 @@ static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, unsigned Opcode; if (EltTy == MVT::i8) Opcode = AArch64ISD::DUPLANE8; - else if (EltTy == MVT::i16) + else if (EltTy == MVT::i16 || EltTy == MVT::f16) Opcode = AArch64ISD::DUPLANE16; else if (EltTy == MVT::i32 || EltTy == MVT::f32) Opcode = AArch64ISD::DUPLANE32; @@ -6554,6 +6611,59 @@ bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const { VT1.getSizeInBits() <= 32); } +bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const { + if (isa(Ext)) + return false; + + // Vector types are next free. + if (Ext->getType()->isVectorTy()) + return false; + + for (const Use &U : Ext->uses()) { + // The extension is free if we can fold it with a left shift in an + // addressing mode or an arithmetic operation: add, sub, and cmp. + + // Is there a shift? + const Instruction *Instr = cast(U.getUser()); + + // Is this a constant shift? + switch (Instr->getOpcode()) { + case Instruction::Shl: + if (!isa(Instr->getOperand(1))) + return false; + break; + case Instruction::GetElementPtr: { + gep_type_iterator GTI = gep_type_begin(Instr); + std::advance(GTI, U.getOperandNo()); + Type *IdxTy = *GTI; + // This extension will end up with a shift because of the scaling factor. + // 8-bit sized types have a scaling factor of 1, thus a shift amount of 0. + // Get the shift amount based on the scaling factor: + // log2(sizeof(IdxTy)) - log2(8). + uint64_t ShiftAmt = + countTrailingZeros(getDataLayout()->getTypeStoreSizeInBits(IdxTy)) - 3; + // Is the constant foldable in the shift of the addressing mode? + // I.e., shift amount is between 1 and 4 inclusive. + if (ShiftAmt == 0 || ShiftAmt > 4) + return false; + break; + } + case Instruction::Trunc: + // Check if this is a noop. + // trunc(sext ty1 to ty2) to ty1. + if (Instr->getType() == Ext->getOperand(0)->getType()) + continue; + // FALL THROUGH. + default: + return false; + } + + // At this point we can use the bfm family, so this extension is free + // for that use. + } + return true; +} + bool AArch64TargetLowering::hasPairedLoad(Type *LoadedType, unsigned &RequiredAligment) const { if (!LoadedType->isIntegerTy() && !LoadedType->isFloatTy()) @@ -6597,7 +6707,17 @@ EVT AArch64TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign, (allowsMisalignedMemoryAccesses(MVT::f128, 0, 1, &Fast) && Fast))) return MVT::f128; - return Size >= 8 ? MVT::i64 : MVT::i32; + if (Size >= 8 && + (memOpAlign(SrcAlign, DstAlign, 8) || + (allowsMisalignedMemoryAccesses(MVT::i64, 0, 1, &Fast) && Fast))) + return MVT::i64; + + if (Size >= 4 && + (memOpAlign(SrcAlign, DstAlign, 4) || + (allowsMisalignedMemoryAccesses(MVT::i32, 0, 1, &Fast) && Fast))) + return MVT::i32; + + return MVT::Other; } // 12-bit optionally shifted immediates are legal for adds. diff --git a/lib/Target/AArch64/AArch64ISelLowering.h b/lib/Target/AArch64/AArch64ISelLowering.h index 5ff11e8..820613b 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.h +++ b/lib/Target/AArch64/AArch64ISelLowering.h @@ -355,6 +355,8 @@ public: getPreferredVectorAction(EVT VT) const override; private: + bool isExtFreeImpl(const Instruction *Ext) const override; + /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can /// make the right decision when generating code for different targets. const AArch64Subtarget *Subtarget; @@ -418,6 +420,9 @@ private: SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS, + SDValue TVal, SDValue FVal, SDLoc dl, + SelectionDAG &DAG) const; SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; diff --git a/lib/Target/AArch64/AArch64InstrFormats.td b/lib/Target/AArch64/AArch64InstrFormats.td index d295c02..0c0efaf 100644 --- a/lib/Target/AArch64/AArch64InstrFormats.td +++ b/lib/Target/AArch64/AArch64InstrFormats.td @@ -1637,10 +1637,16 @@ multiclass AddSub { let hasSideEffects = 0, isReMaterializable = 1, isAsCheapAsAMove = 1 in { // Add/Subtract immediate + // Increase the weight of the immediate variant to try to match it before + // the extended register variant. + // We used to match the register variant before the immediate when the + // register argument could be implicitly zero-extended. + let AddedComplexity = 6 in def Wri : BaseAddSubImm { let Inst{31} = 0; } + let AddedComplexity = 6 in def Xri : BaseAddSubImm { let Inst{31} = 1; @@ -3282,6 +3288,10 @@ class LoadStoreExclusiveSimple sz, bit o2, bit L, bit o1, bit o0, : BaseLoadStoreExclusive { bits<5> Rt; bits<5> Rn; + let Inst{20-16} = 0b11111; + let Unpredictable{20-16} = 0b11111; + let Inst{14-10} = 0b11111; + let Unpredictable{14-10} = 0b11111; let Inst{9-5} = Rn; let Inst{4-0} = Rt; @@ -5298,6 +5308,27 @@ class BaseSIMDThreeScalar size, bits<5> opcode, let Inst{4-0} = Rd; } +let mayStore = 0, mayLoad = 0, hasSideEffects = 0 in +class BaseSIMDThreeScalarTied size, bit R, bits<5> opcode, + dag oops, dag iops, string asm, + list pattern> + : I, + Sched<[WriteV]> { + bits<5> Rd; + bits<5> Rn; + bits<5> Rm; + let Inst{31-30} = 0b01; + let Inst{29} = U; + let Inst{28-24} = 0b11110; + let Inst{23-22} = size; + let Inst{21} = R; + let Inst{20-16} = Rm; + let Inst{15-11} = opcode; + let Inst{10} = 1; + let Inst{9-5} = Rn; + let Inst{4-0} = Rd; +} + multiclass SIMDThreeScalarD opc, string asm, SDPatternOperator OpNode> { def v1i64 : BaseSIMDThreeScalar opc, string asm, def v1i16 : BaseSIMDThreeScalar; } +multiclass SIMDThreeScalarHSTied opc, string asm, + SDPatternOperator OpNode = null_frag> { + def v1i32: BaseSIMDThreeScalarTied; + def v1i16: BaseSIMDThreeScalarTied; +} + multiclass SIMDThreeScalarSD opc, string asm, SDPatternOperator OpNode = null_frag> { let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { @@ -5885,7 +5926,7 @@ multiclass SIMDIns { let Inst{20-18} = idx; let Inst{17-16} = 0b10; let Inst{14-12} = idx2; - let Inst{11} = 0; + let Inst{11} = {?}; } def vi32lane : SIMDInsFromElement<".s", v4i32, i32, VectorIndexS> { bits<2> idx; @@ -5893,7 +5934,7 @@ multiclass SIMDIns { let Inst{20-19} = idx; let Inst{18-16} = 0b100; let Inst{14-13} = idx2; - let Inst{12-11} = 0; + let Inst{12-11} = {?,?}; } def vi64lane : SIMDInsFromElement<".d", v2i64, i64, VectorIndexD> { bits<1> idx; @@ -5901,7 +5942,7 @@ multiclass SIMDIns { let Inst{20} = idx; let Inst{19-16} = 0b1000; let Inst{14} = idx2; - let Inst{13-11} = 0; + let Inst{13-11} = {?,?,?}; } // For all forms of the INS instruction, the "mov" mnemonic is the @@ -8517,6 +8558,174 @@ multiclass SIMDLdSt4SingleAliases { } // end of 'let Predicates = [HasNEON]' //---------------------------------------------------------------------------- +// AdvSIMD v8.1 Rounding Double Multiply Add/Subtract +//---------------------------------------------------------------------------- + +let Predicates = [HasNEON, HasV8_1a] in { + +class BaseSIMDThreeSameVectorTiedR0 size, bits<5> opcode, + RegisterOperand regtype, string asm, + string kind, list pattern> + : BaseSIMDThreeSameVectorTied { + let Inst{21}=0; +} +multiclass SIMDThreeSameVectorSQRDMLxHTiedHS opc, string asm, + SDPatternOperator Accum> { + def v4i16 : BaseSIMDThreeSameVectorTiedR0<0, U, 0b01, opc, V64, asm, ".4h", + [(set (v4i16 V64:$dst), + (Accum (v4i16 V64:$Rd), + (v4i16 (int_aarch64_neon_sqrdmulh (v4i16 V64:$Rn), + (v4i16 V64:$Rm)))))]>; + def v8i16 : BaseSIMDThreeSameVectorTiedR0<1, U, 0b01, opc, V128, asm, ".8h", + [(set (v8i16 V128:$dst), + (Accum (v8i16 V128:$Rd), + (v8i16 (int_aarch64_neon_sqrdmulh (v8i16 V128:$Rn), + (v8i16 V128:$Rm)))))]>; + def v2i32 : BaseSIMDThreeSameVectorTiedR0<0, U, 0b10, opc, V64, asm, ".2s", + [(set (v2i32 V64:$dst), + (Accum (v2i32 V64:$Rd), + (v2i32 (int_aarch64_neon_sqrdmulh (v2i32 V64:$Rn), + (v2i32 V64:$Rm)))))]>; + def v4i32 : BaseSIMDThreeSameVectorTiedR0<1, U, 0b10, opc, V128, asm, ".4s", + [(set (v4i32 V128:$dst), + (Accum (v4i32 V128:$Rd), + (v4i32 (int_aarch64_neon_sqrdmulh (v4i32 V128:$Rn), + (v4i32 V128:$Rm)))))]>; +} + +multiclass SIMDIndexedSQRDMLxHSDTied opc, string asm, + SDPatternOperator Accum> { + def v4i16_indexed : BaseSIMDIndexedTied<0, U, 0, 0b01, opc, + V64, V64, V128_lo, VectorIndexH, + asm, ".4h", ".4h", ".4h", ".h", + [(set (v4i16 V64:$dst), + (Accum (v4i16 V64:$Rd), + (v4i16 (int_aarch64_neon_sqrdmulh + (v4i16 V64:$Rn), + (v4i16 (AArch64duplane16 (v8i16 V128_lo:$Rm), + VectorIndexH:$idx))))))]> { + bits<3> idx; + let Inst{11} = idx{2}; + let Inst{21} = idx{1}; + let Inst{20} = idx{0}; + } + + def v8i16_indexed : BaseSIMDIndexedTied<1, U, 0, 0b01, opc, + V128, V128, V128_lo, VectorIndexH, + asm, ".8h", ".8h", ".8h", ".h", + [(set (v8i16 V128:$dst), + (Accum (v8i16 V128:$Rd), + (v8i16 (int_aarch64_neon_sqrdmulh + (v8i16 V128:$Rn), + (v8i16 (AArch64duplane16 (v8i16 V128_lo:$Rm), + VectorIndexH:$idx))))))]> { + bits<3> idx; + let Inst{11} = idx{2}; + let Inst{21} = idx{1}; + let Inst{20} = idx{0}; + } + + def v2i32_indexed : BaseSIMDIndexedTied<0, U, 0, 0b10, opc, + V64, V64, V128, VectorIndexS, + asm, ".2s", ".2s", ".2s", ".s", + [(set (v2i32 V64:$dst), + (Accum (v2i32 V64:$Rd), + (v2i32 (int_aarch64_neon_sqrdmulh + (v2i32 V64:$Rn), + (v2i32 (AArch64duplane32 (v4i32 V128:$Rm), + VectorIndexS:$idx))))))]> { + bits<2> idx; + let Inst{11} = idx{1}; + let Inst{21} = idx{0}; + } + + // FIXME: it would be nice to use the scalar (v1i32) instruction here, but + // an intermediate EXTRACT_SUBREG would be untyped. + // FIXME: direct EXTRACT_SUBREG from v2i32 to i32 is illegal, that's why we + // got it lowered here as (i32 vector_extract (v4i32 insert_subvector(..))) + def : Pat<(i32 (Accum (i32 FPR32Op:$Rd), + (i32 (vector_extract + (v4i32 (insert_subvector + (undef), + (v2i32 (int_aarch64_neon_sqrdmulh + (v2i32 V64:$Rn), + (v2i32 (AArch64duplane32 + (v4i32 V128:$Rm), + VectorIndexS:$idx)))), + (i32 0))), + (i64 0))))), + (EXTRACT_SUBREG + (v2i32 (!cast(NAME # v2i32_indexed) + (v2i32 (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)), + FPR32Op:$Rd, + ssub)), + V64:$Rn, + V128:$Rm, + VectorIndexS:$idx)), + ssub)>; + + def v4i32_indexed : BaseSIMDIndexedTied<1, U, 0, 0b10, opc, + V128, V128, V128, VectorIndexS, + asm, ".4s", ".4s", ".4s", ".s", + [(set (v4i32 V128:$dst), + (Accum (v4i32 V128:$Rd), + (v4i32 (int_aarch64_neon_sqrdmulh + (v4i32 V128:$Rn), + (v4i32 (AArch64duplane32 (v4i32 V128:$Rm), + VectorIndexS:$idx))))))]> { + bits<2> idx; + let Inst{11} = idx{1}; + let Inst{21} = idx{0}; + } + + // FIXME: it would be nice to use the scalar (v1i32) instruction here, but + // an intermediate EXTRACT_SUBREG would be untyped. + def : Pat<(i32 (Accum (i32 FPR32Op:$Rd), + (i32 (vector_extract + (v4i32 (int_aarch64_neon_sqrdmulh + (v4i32 V128:$Rn), + (v4i32 (AArch64duplane32 + (v4i32 V128:$Rm), + VectorIndexS:$idx)))), + (i64 0))))), + (EXTRACT_SUBREG + (v4i32 (!cast(NAME # v4i32_indexed) + (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), + FPR32Op:$Rd, + ssub)), + V128:$Rn, + V128:$Rm, + VectorIndexS:$idx)), + ssub)>; + + def i16_indexed : BaseSIMDIndexedTied<1, U, 1, 0b01, opc, + FPR16Op, FPR16Op, V128_lo, + VectorIndexH, asm, ".h", "", "", ".h", + []> { + bits<3> idx; + let Inst{11} = idx{2}; + let Inst{21} = idx{1}; + let Inst{20} = idx{0}; + } + + def i32_indexed : BaseSIMDIndexedTied<1, U, 1, 0b10, opc, + FPR32Op, FPR32Op, V128, VectorIndexS, + asm, ".s", "", "", ".s", + [(set (i32 FPR32Op:$dst), + (Accum (i32 FPR32Op:$Rd), + (i32 (int_aarch64_neon_sqrdmulh + (i32 FPR32Op:$Rn), + (i32 (vector_extract (v4i32 V128:$Rm), + VectorIndexS:$idx))))))]> { + bits<2> idx; + let Inst{11} = idx{1}; + let Inst{21} = idx{0}; + } +} +} // let Predicates = [HasNeon, HasV8_1a] + +//---------------------------------------------------------------------------- // Crypto extensions //---------------------------------------------------------------------------- diff --git a/lib/Target/AArch64/AArch64InstrInfo.cpp b/lib/Target/AArch64/AArch64InstrInfo.cpp index 8e0af2d..db231c4 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -1526,7 +1526,7 @@ void AArch64InstrInfo::copyPhysRegTuple( } for (; SubReg != End; SubReg += Incr) { - const MachineInstrBuilder &MIB = BuildMI(MBB, I, DL, get(Opcode)); + const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode)); AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI); AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI); AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI); @@ -1904,7 +1904,7 @@ void AArch64InstrInfo::storeRegToStackSlot( } assert(Opc && "Unknown register class"); - const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc)) + const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DL, get(Opc)) .addReg(SrcReg, getKillRegState(isKill)) .addFrameIndex(FI); @@ -2002,7 +2002,7 @@ void AArch64InstrInfo::loadRegFromStackSlot( } assert(Opc && "Unknown register class"); - const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc)) + const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DL, get(Opc)) .addReg(DestReg, getDefRegState(true)) .addFrameIndex(FI); if (Offset) diff --git a/lib/Target/AArch64/AArch64InstrInfo.td b/lib/Target/AArch64/AArch64InstrInfo.td index ec6fa5c..f7db50a 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.td +++ b/lib/Target/AArch64/AArch64InstrInfo.td @@ -14,6 +14,8 @@ //===----------------------------------------------------------------------===// // ARM Instruction Predicate Definitions. // +def HasV8_1a : Predicate<"Subtarget->hasV8_1aOps()">, + AssemblerPredicate<"HasV8_1aOps", "armv8.1a">; def HasFPARMv8 : Predicate<"Subtarget->hasFPARMv8()">, AssemblerPredicate<"FeatureFPARMv8", "fp-armv8">; def HasNEON : Predicate<"Subtarget->hasNEON()">, @@ -22,8 +24,6 @@ def HasCrypto : Predicate<"Subtarget->hasCrypto()">, AssemblerPredicate<"FeatureCrypto", "crypto">; def HasCRC : Predicate<"Subtarget->hasCRC()">, AssemblerPredicate<"FeatureCRC", "crc">; -def HasV8_1a : Predicate<"Subtarget->hasV8_1a()">, - AssemblerPredicate<"FeatureV8_1a", "v8.1a">; def IsLE : Predicate<"Subtarget->isLittleEndian()">; def IsBE : Predicate<"!Subtarget->isLittleEndian()">; def IsCyclone : Predicate<"Subtarget->isCyclone()">; @@ -2314,6 +2314,20 @@ def STLXPX : StoreExclusivePair<0b11, 0, 0, 1, 1, GPR64, "stlxp">; def STXPW : StoreExclusivePair<0b10, 0, 0, 1, 0, GPR32, "stxp">; def STXPX : StoreExclusivePair<0b11, 0, 0, 1, 0, GPR64, "stxp">; +let Predicates = [HasV8_1a] in { + // v8.1a "Limited Order Region" extension load-acquire instructions + def LDLARW : LoadAcquire <0b10, 1, 1, 0, 0, GPR32, "ldlar">; + def LDLARX : LoadAcquire <0b11, 1, 1, 0, 0, GPR64, "ldlar">; + def LDLARB : LoadAcquire <0b00, 1, 1, 0, 0, GPR32, "ldlarb">; + def LDLARH : LoadAcquire <0b01, 1, 1, 0, 0, GPR32, "ldlarh">; + + // v8.1a "Limited Order Region" extension store-release instructions + def STLLRW : StoreRelease <0b10, 1, 0, 0, 0, GPR32, "stllr">; + def STLLRX : StoreRelease <0b11, 1, 0, 0, 0, GPR64, "stllr">; + def STLLRB : StoreRelease <0b00, 1, 0, 0, 0, GPR32, "stllrb">; + def STLLRH : StoreRelease <0b01, 1, 0, 0, 0, GPR32, "stllrh">; +} + //===----------------------------------------------------------------------===// // Scaled floating point to integer conversion instructions. //===----------------------------------------------------------------------===// @@ -2778,6 +2792,10 @@ defm UQSUB : SIMDThreeSameVector<1,0b00101,"uqsub", int_aarch64_neon_uqsub>; defm URHADD : SIMDThreeSameVectorBHS<1,0b00010,"urhadd", int_aarch64_neon_urhadd>; defm URSHL : SIMDThreeSameVector<1,0b01010,"urshl", int_aarch64_neon_urshl>; defm USHL : SIMDThreeSameVector<1,0b01000,"ushl", int_aarch64_neon_ushl>; +defm SQRDMLAH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10000,"sqrdmlah", + int_aarch64_neon_sqadd>; +defm SQRDMLSH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10001,"sqrdmlsh", + int_aarch64_neon_sqsub>; defm AND : SIMDLogicalThreeVector<0, 0b00, "and", and>; defm BIC : SIMDLogicalThreeVector<0, 0b01, "bic", @@ -2994,6 +3012,20 @@ defm UQSHL : SIMDThreeScalarBHSD<1, 0b01001, "uqshl", int_aarch64_neon_uqshl> defm UQSUB : SIMDThreeScalarBHSD<1, 0b00101, "uqsub", int_aarch64_neon_uqsub>; defm URSHL : SIMDThreeScalarD< 1, 0b01010, "urshl", int_aarch64_neon_urshl>; defm USHL : SIMDThreeScalarD< 1, 0b01000, "ushl", int_aarch64_neon_ushl>; +let Predicates = [HasV8_1a] in { + defm SQRDMLAH : SIMDThreeScalarHSTied<1, 0, 0b10000, "sqrdmlah">; + defm SQRDMLSH : SIMDThreeScalarHSTied<1, 0, 0b10001, "sqrdmlsh">; + def : Pat<(i32 (int_aarch64_neon_sqadd + (i32 FPR32:$Rd), + (i32 (int_aarch64_neon_sqrdmulh (i32 FPR32:$Rn), + (i32 FPR32:$Rm))))), + (SQRDMLAHv1i32 FPR32:$Rd, FPR32:$Rn, FPR32:$Rm)>; + def : Pat<(i32 (int_aarch64_neon_sqsub + (i32 FPR32:$Rd), + (i32 (int_aarch64_neon_sqrdmulh (i32 FPR32:$Rn), + (i32 FPR32:$Rm))))), + (SQRDMLSHv1i32 FPR32:$Rd, FPR32:$Rn, FPR32:$Rm)>; +} def : InstAlias<"cmls $dst, $src1, $src2", (CMHSv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>; @@ -3478,13 +3510,13 @@ def : Pat<(f64 (int_aarch64_neon_fminv (v2f64 V128:$Rn))), // AdvSIMD INS/DUP instructions //---------------------------------------------------------------------------- -def DUPv8i8gpr : SIMDDupFromMain<0, 0b00001, ".8b", v8i8, V64, GPR32>; -def DUPv16i8gpr : SIMDDupFromMain<1, 0b00001, ".16b", v16i8, V128, GPR32>; -def DUPv4i16gpr : SIMDDupFromMain<0, 0b00010, ".4h", v4i16, V64, GPR32>; -def DUPv8i16gpr : SIMDDupFromMain<1, 0b00010, ".8h", v8i16, V128, GPR32>; -def DUPv2i32gpr : SIMDDupFromMain<0, 0b00100, ".2s", v2i32, V64, GPR32>; -def DUPv4i32gpr : SIMDDupFromMain<1, 0b00100, ".4s", v4i32, V128, GPR32>; -def DUPv2i64gpr : SIMDDupFromMain<1, 0b01000, ".2d", v2i64, V128, GPR64>; +def DUPv8i8gpr : SIMDDupFromMain<0, {?,?,?,?,1}, ".8b", v8i8, V64, GPR32>; +def DUPv16i8gpr : SIMDDupFromMain<1, {?,?,?,?,1}, ".16b", v16i8, V128, GPR32>; +def DUPv4i16gpr : SIMDDupFromMain<0, {?,?,?,1,0}, ".4h", v4i16, V64, GPR32>; +def DUPv8i16gpr : SIMDDupFromMain<1, {?,?,?,1,0}, ".8h", v8i16, V128, GPR32>; +def DUPv2i32gpr : SIMDDupFromMain<0, {?,?,1,0,0}, ".2s", v2i32, V64, GPR32>; +def DUPv4i32gpr : SIMDDupFromMain<1, {?,?,1,0,0}, ".4s", v4i32, V128, GPR32>; +def DUPv2i64gpr : SIMDDupFromMain<1, {?,1,0,0,0}, ".2d", v2i64, V128, GPR64>; def DUPv2i64lane : SIMDDup64FromElement; def DUPv2i32lane : SIMDDup32FromElement<0, ".2s", v2i32, V64>; @@ -4324,6 +4356,10 @@ defm SQDMLAL : SIMDIndexedLongSQDMLXSDTied<0, 0b0011, "sqdmlal", int_aarch64_neon_sqadd>; defm SQDMLSL : SIMDIndexedLongSQDMLXSDTied<0, 0b0111, "sqdmlsl", int_aarch64_neon_sqsub>; +defm SQRDMLAH : SIMDIndexedSQRDMLxHSDTied<1, 0b1101, "sqrdmlah", + int_aarch64_neon_sqadd>; +defm SQRDMLSH : SIMDIndexedSQRDMLxHSDTied<1, 0b1111, "sqrdmlsh", + int_aarch64_neon_sqsub>; defm SQDMULL : SIMDIndexedLongSD<0, 0b1011, "sqdmull", int_aarch64_neon_sqdmull>; defm UMLAL : SIMDVectorIndexedLongSDTied<1, 0b0010, "umlal", TriOpFrag<(add node:$LHS, (int_aarch64_neon_umull node:$MHS, node:$RHS))>>; diff --git a/lib/Target/AArch64/AArch64RegisterInfo.cpp b/lib/Target/AArch64/AArch64RegisterInfo.cpp index 33c11fe..1836682 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -165,7 +165,12 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { // large enough that referencing from the FP won't result in things being // in range relatively often, we can use a base pointer to allow access // from the other direction like the SP normally works. + // Furthermore, if both variable sized objects are present, and the + // stack needs to be dynamically re-aligned, the base pointer is the only + // reliable way to reference the locals. if (MFI->hasVarSizedObjects()) { + if (needsStackRealignment(MF)) + return true; // Conservatively estimate whether the negative offset from the frame // pointer will be sufficient to reach. If a function has a smallish // frame, it's less likely to have lots of spills and callee saved @@ -181,6 +186,31 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { return false; } +bool AArch64RegisterInfo::canRealignStack(const MachineFunction &MF) const { + + if (MF.getFunction()->hasFnAttribute("no-realign-stack")) + return false; + + return true; +} + +// FIXME: share this with other backends with identical implementation? +bool +AArch64RegisterInfo::needsStackRealignment(const MachineFunction &MF) const { + const MachineFrameInfo *MFI = MF.getFrameInfo(); + const Function *F = MF.getFunction(); + unsigned StackAlign = MF.getTarget() + .getSubtargetImpl(*MF.getFunction()) + ->getFrameLowering() + ->getStackAlignment(); + bool requiresRealignment = + ((MFI->getMaxAlignment() > StackAlign) || + F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::StackAlignment)); + + return requiresRealignment && canRealignStack(MF); +} + unsigned AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const { const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); diff --git a/lib/Target/AArch64/AArch64RegisterInfo.h b/lib/Target/AArch64/AArch64RegisterInfo.h index c01bfa5..8c379d9 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.h +++ b/lib/Target/AArch64/AArch64RegisterInfo.h @@ -93,6 +93,9 @@ public: unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override; + // Base pointer (stack realignment) support. + bool canRealignStack(const MachineFunction &MF) const; + bool needsStackRealignment(const MachineFunction &MF) const override; }; } // end namespace llvm diff --git a/lib/Target/AArch64/AArch64SchedA57.td b/lib/Target/AArch64/AArch64SchedA57.td index 3ec4157..ca4457a 100644 --- a/lib/Target/AArch64/AArch64SchedA57.td +++ b/lib/Target/AArch64/AArch64SchedA57.td @@ -60,7 +60,12 @@ include "AArch64SchedA57WriteRes.td" // Cortex-A57. The Cortex-A57 types are directly associated with resources, so // defining the aliases precludes the need for mapping them using WriteRes. The // aliases are sufficient for creating a coarse, working model. As the model -// evolves, InstRWs will be used to override these SchedAliases. +// evolves, InstRWs will be used to override some of these SchedAliases. +// +// WARNING: Using SchedAliases is convenient and works well for latency and +// resource lookup for instructions. However, this creates an entry in +// AArch64WriteLatencyTable with a WriteResourceID of 0, breaking +// any SchedReadAdvance since the lookup will fail. def : SchedAlias; def : SchedAlias; @@ -70,8 +75,8 @@ def : SchedAlias; def : SchedAlias; def : SchedAlias; def : SchedAlias; -def : SchedAlias; -def : SchedAlias; +def : WriteRes { let Latency = 3; } +def : WriteRes { let Latency = 5; } def : SchedAlias; def : SchedAlias; def : SchedAlias; @@ -127,6 +132,15 @@ def : InstRW<[A57Write_1cyc_1B_1I], (instrs BL)>; def : InstRW<[A57Write_2cyc_1B_1I], (instrs BLR)>; +// Shifted Register with Shift == 0 +// ---------------------------------------------------------------------------- + +def A57WriteISReg : SchedWriteVariant<[ + SchedVar, + SchedVar]>; +def : InstRW<[A57WriteISReg], (instregex ".*rs$")>; + + // Divide and Multiply Instructions // ----------------------------------------------------------------------------- diff --git a/lib/Target/AArch64/AArch64Subtarget.cpp b/lib/Target/AArch64/AArch64Subtarget.cpp index 221d70d..0b97af8 100644 --- a/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/lib/Target/AArch64/AArch64Subtarget.cpp @@ -47,8 +47,9 @@ AArch64Subtarget::AArch64Subtarget(const std::string &TT, const std::string &FS, const TargetMachine &TM, bool LittleEndian) : AArch64GenSubtargetInfo(TT, CPU, FS), ARMProcFamily(Others), + HasV8_1aOps(false), HasFPARMv8(false), HasNEON(false), HasCrypto(false), HasCRC(false), - HasV8_1a(false), HasZeroCycleRegMove(false), HasZeroCycleZeroing(false), + HasZeroCycleRegMove(false), HasZeroCycleZeroing(false), IsLittle(LittleEndian), CPUString(CPU), TargetTriple(TT), FrameLowering(), InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(TM.getDataLayout()), TLInfo(TM, *this) {} diff --git a/lib/Target/AArch64/AArch64Subtarget.h b/lib/Target/AArch64/AArch64Subtarget.h index bcab97d..5454b20 100644 --- a/lib/Target/AArch64/AArch64Subtarget.h +++ b/lib/Target/AArch64/AArch64Subtarget.h @@ -37,11 +37,12 @@ protected: /// ARMProcFamily - ARM processor family: Cortex-A53, Cortex-A57, and others. ARMProcFamilyEnum ARMProcFamily; + bool HasV8_1aOps; + bool HasFPARMv8; bool HasNEON; bool HasCrypto; bool HasCRC; - bool HasV8_1a; // HasZeroCycleRegMove - Has zero-cycle register mov instructions. bool HasZeroCycleRegMove; @@ -93,6 +94,8 @@ public: return isCortexA53() || isCortexA57(); } + bool hasV8_1aOps() const { return HasV8_1aOps; } + bool hasZeroCycleRegMove() const { return HasZeroCycleRegMove; } bool hasZeroCycleZeroing() const { return HasZeroCycleZeroing; } @@ -101,7 +104,6 @@ public: bool hasNEON() const { return HasNEON; } bool hasCrypto() const { return HasCrypto; } bool hasCRC() const { return HasCRC; } - bool hasV8_1a() const { return HasV8_1a; } bool isLittleEndian() const { return IsLittle; } diff --git a/lib/Target/AArch64/AArch64TargetMachine.cpp b/lib/Target/AArch64/AArch64TargetMachine.cpp index f902f64..ab28a16 100644 --- a/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -87,6 +87,11 @@ EnableGEPOpt("aarch64-gep-opt", cl::Hidden, cl::desc("Enable optimizations on complex GEPs"), cl::init(true)); +// FIXME: Unify control over GlobalMerge. +static cl::opt +EnableGlobalMerge("aarch64-global-merge", cl::Hidden, + cl::desc("Enable the global merge pass")); + extern "C" void LLVMInitializeAArch64Target() { // Register the target. RegisterTargetMachine X(TheAArch64leTarget); @@ -245,7 +250,9 @@ bool AArch64PassConfig::addPreISel() { // FIXME: On AArch64, this depends on the type. // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes(). // and the offset has to be a multiple of the related size in bytes. - if (TM->getOptLevel() == CodeGenOpt::Aggressive) + if ((TM->getOptLevel() == CodeGenOpt::Aggressive && + EnableGlobalMerge == cl::BOU_UNSET) || + EnableGlobalMerge == cl::BOU_TRUE) addPass(createGlobalMergePass(TM, 4095)); if (TM->getOptLevel() != CodeGenOpt::None) addPass(createAArch64AddressTypePromotionPass()); diff --git a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index 1219ffc..063c714 100644 --- a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -1972,7 +1972,8 @@ AArch64AsmParser::tryParsePrefetch(OperandVector &Operands) { bool Valid; auto Mapper = AArch64PRFM::PRFMMapper(); - StringRef Name = Mapper.toString(MCE->getValue(), Valid); + StringRef Name = + Mapper.toString(MCE->getValue(), STI.getFeatureBits(), Valid); Operands.push_back(AArch64Operand::CreatePrefetch(prfop, Name, S, getContext())); return MatchOperand_Success; @@ -1985,7 +1986,8 @@ AArch64AsmParser::tryParsePrefetch(OperandVector &Operands) { bool Valid; auto Mapper = AArch64PRFM::PRFMMapper(); - unsigned prfop = Mapper.fromString(Tok.getString(), Valid); + unsigned prfop = + Mapper.fromString(Tok.getString(), STI.getFeatureBits(), Valid); if (!Valid) { TokError("pre-fetch hint expected"); return MatchOperand_ParseFail; @@ -2090,15 +2092,16 @@ AArch64AsmParser::tryParseFPImm(OperandVector &Operands) { const AsmToken &Tok = Parser.getTok(); if (Tok.is(AsmToken::Real)) { APFloat RealVal(APFloat::IEEEdouble, Tok.getString()); + if (isNegative) + RealVal.changeSign(); + uint64_t IntVal = RealVal.bitcastToAPInt().getZExtValue(); - // If we had a '-' in front, toggle the sign bit. - IntVal ^= (uint64_t)isNegative << 63; int Val = AArch64_AM::getFP64Imm(APInt(64, IntVal)); Parser.Lex(); // Eat the token. // Check for out of range values. As an exception, we let Zero through, // as we handle that special case in post-processing before matching in // order to use the zero register for it. - if (Val == -1 && !RealVal.isZero()) { + if (Val == -1 && !RealVal.isPosZero()) { TokError("expected compatible register or floating-point constant"); return MatchOperand_ParseFail; } @@ -2597,7 +2600,8 @@ AArch64AsmParser::tryParseBarrierOperand(OperandVector &Operands) { } bool Valid; auto Mapper = AArch64DB::DBarrierMapper(); - StringRef Name = Mapper.toString(MCE->getValue(), Valid); + StringRef Name = + Mapper.toString(MCE->getValue(), STI.getFeatureBits(), Valid); Operands.push_back( AArch64Operand::CreateBarrier(MCE->getValue(), Name, ExprLoc, getContext())); return MatchOperand_Success; @@ -2610,7 +2614,8 @@ AArch64AsmParser::tryParseBarrierOperand(OperandVector &Operands) { bool Valid; auto Mapper = AArch64DB::DBarrierMapper(); - unsigned Opt = Mapper.fromString(Tok.getString(), Valid); + unsigned Opt = + Mapper.fromString(Tok.getString(), STI.getFeatureBits(), Valid); if (!Valid) { TokError("invalid barrier option name"); return MatchOperand_ParseFail; @@ -2638,18 +2643,21 @@ AArch64AsmParser::tryParseSysReg(OperandVector &Operands) { return MatchOperand_NoMatch; bool IsKnown; - auto MRSMapper = AArch64SysReg::MRSMapper(STI.getFeatureBits()); - uint32_t MRSReg = MRSMapper.fromString(Tok.getString(), IsKnown); + auto MRSMapper = AArch64SysReg::MRSMapper(); + uint32_t MRSReg = MRSMapper.fromString(Tok.getString(), STI.getFeatureBits(), + IsKnown); assert(IsKnown == (MRSReg != -1U) && "register should be -1 if and only if it's unknown"); - auto MSRMapper = AArch64SysReg::MSRMapper(STI.getFeatureBits()); - uint32_t MSRReg = MSRMapper.fromString(Tok.getString(), IsKnown); + auto MSRMapper = AArch64SysReg::MSRMapper(); + uint32_t MSRReg = MSRMapper.fromString(Tok.getString(), STI.getFeatureBits(), + IsKnown); assert(IsKnown == (MSRReg != -1U) && "register should be -1 if and only if it's unknown"); auto PStateMapper = AArch64PState::PStateMapper(); - uint32_t PStateField = PStateMapper.fromString(Tok.getString(), IsKnown); + uint32_t PStateField = + PStateMapper.fromString(Tok.getString(), STI.getFeatureBits(), IsKnown); assert(IsKnown == (PStateField != -1U) && "register should be -1 if and only if it's unknown"); diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index fb25089..1c8c0a66 100644 --- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -1102,6 +1102,12 @@ static DecodeStatus DecodeExclusiveLdStInstruction(llvm::MCInst &Inst, case AArch64::STLRW: case AArch64::STLRB: case AArch64::STLRH: + case AArch64::STLLRW: + case AArch64::STLLRB: + case AArch64::STLLRH: + case AArch64::LDLARW: + case AArch64::LDLARB: + case AArch64::LDLARH: DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder); break; case AArch64::STLXRX: @@ -1112,6 +1118,8 @@ static DecodeStatus DecodeExclusiveLdStInstruction(llvm::MCInst &Inst, case AArch64::LDAXRX: case AArch64::LDXRX: case AArch64::STLRX: + case AArch64::LDLARX: + case AArch64::STLLRX: DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder); break; case AArch64::STLXPW: @@ -1504,7 +1512,10 @@ static DecodeStatus DecodeSystemPStateInstruction(llvm::MCInst &Inst, Inst.addOperand(MCOperand::CreateImm(crm)); bool ValidNamed; - (void)AArch64PState::PStateMapper().toString(pstate_field, ValidNamed); + const AArch64Disassembler *Dis = + static_cast(Decoder); + (void)AArch64PState::PStateMapper().toString(pstate_field, + Dis->getSubtargetInfo().getFeatureBits(), ValidNamed); return ValidNamed ? Success : Fail; } diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp index 46a1d79..febd332 100644 --- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp +++ b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp @@ -34,18 +34,13 @@ using namespace llvm; AArch64InstPrinter::AArch64InstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII, - const MCRegisterInfo &MRI, - const MCSubtargetInfo &STI) - : MCInstPrinter(MAI, MII, MRI) { - // Initialize the set of available features. - setAvailableFeatures(STI.getFeatureBits()); -} + const MCRegisterInfo &MRI) + : MCInstPrinter(MAI, MII, MRI) {} AArch64AppleInstPrinter::AArch64AppleInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII, - const MCRegisterInfo &MRI, - const MCSubtargetInfo &STI) - : AArch64InstPrinter(MAI, MII, MRI, STI) {} + const MCRegisterInfo &MRI) + : AArch64InstPrinter(MAI, MII, MRI) {} void AArch64InstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const { // This is for .cfi directives. @@ -53,7 +48,8 @@ void AArch64InstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const { } void AArch64InstPrinter::printInst(const MCInst *MI, raw_ostream &O, - StringRef Annot) { + StringRef Annot, + const MCSubtargetInfo &STI) { // Check for special encodings and print the canonical alias instead. unsigned Opcode = MI->getOpcode(); @@ -210,8 +206,8 @@ void AArch64InstPrinter::printInst(const MCInst *MI, raw_ostream &O, return; } - if (!printAliasInstr(MI, O)) - printInstruction(MI, O); + if (!printAliasInstr(MI, STI, O)) + printInstruction(MI, STI, O); printAnnotation(O, Annot); } @@ -614,7 +610,8 @@ static LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) { } void AArch64AppleInstPrinter::printInst(const MCInst *MI, raw_ostream &O, - StringRef Annot) { + StringRef Annot, + const MCSubtargetInfo &STI) { unsigned Opcode = MI->getOpcode(); StringRef Layout, Mnemonic; @@ -624,7 +621,7 @@ void AArch64AppleInstPrinter::printInst(const MCInst *MI, raw_ostream &O, << getRegisterName(MI->getOperand(0).getReg(), AArch64::vreg) << ", "; unsigned ListOpNum = IsTbx ? 2 : 1; - printVectorList(MI, ListOpNum, O, ""); + printVectorList(MI, ListOpNum, STI, O, ""); O << ", " << getRegisterName(MI->getOperand(ListOpNum + 1).getReg(), AArch64::vreg); @@ -638,7 +635,7 @@ void AArch64AppleInstPrinter::printInst(const MCInst *MI, raw_ostream &O, // Now onto the operands: first a vector list with possible lane // specifier. E.g. { v0 }[2] int OpNum = LdStDesc->ListOperand; - printVectorList(MI, OpNum++, O, ""); + printVectorList(MI, OpNum++, STI, O, ""); if (LdStDesc->HasLane) O << '[' << MI->getOperand(OpNum++).getImm() << ']'; @@ -662,7 +659,7 @@ void AArch64AppleInstPrinter::printInst(const MCInst *MI, raw_ostream &O, return; } - AArch64InstPrinter::printInst(MI, O, Annot); + AArch64InstPrinter::printInst(MI, O, Annot, STI); } bool AArch64InstPrinter::printSysAlias(const MCInst *MI, raw_ostream &O) { @@ -889,6 +886,7 @@ bool AArch64InstPrinter::printSysAlias(const MCInst *MI, raw_ostream &O) { } void AArch64InstPrinter::printOperand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNo); if (Op.isReg()) { @@ -903,6 +901,7 @@ void AArch64InstPrinter::printOperand(const MCInst *MI, unsigned OpNo, } void AArch64InstPrinter::printHexImm(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNo); O << format("#%#llx", Op.getImm()); @@ -922,6 +921,7 @@ void AArch64InstPrinter::printPostIncOperand(const MCInst *MI, unsigned OpNo, } void AArch64InstPrinter::printVRegOperand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNo); assert(Op.isReg() && "Non-register vreg operand!"); @@ -930,6 +930,7 @@ void AArch64InstPrinter::printVRegOperand(const MCInst *MI, unsigned OpNo, } void AArch64InstPrinter::printSysCROperand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNo); assert(Op.isImm() && "System instruction C[nm] operands must be immediates!"); @@ -937,6 +938,7 @@ void AArch64InstPrinter::printSysCROperand(const MCInst *MI, unsigned OpNo, } void AArch64InstPrinter::printAddSubImm(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &MO = MI->getOperand(OpNum); if (MO.isImm()) { @@ -946,18 +948,19 @@ void AArch64InstPrinter::printAddSubImm(const MCInst *MI, unsigned OpNum, AArch64_AM::getShiftValue(MI->getOperand(OpNum + 1).getImm()); O << '#' << Val; if (Shift != 0) - printShifter(MI, OpNum + 1, O); + printShifter(MI, OpNum + 1, STI, O); if (CommentStream) *CommentStream << '=' << (Val << Shift) << '\n'; } else { assert(MO.isExpr() && "Unexpected operand type!"); O << *MO.getExpr(); - printShifter(MI, OpNum + 1, O); + printShifter(MI, OpNum + 1, STI, O); } } void AArch64InstPrinter::printLogicalImm32(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { uint64_t Val = MI->getOperand(OpNum).getImm(); O << "#0x"; @@ -965,6 +968,7 @@ void AArch64InstPrinter::printLogicalImm32(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printLogicalImm64(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { uint64_t Val = MI->getOperand(OpNum).getImm(); O << "#0x"; @@ -972,6 +976,7 @@ void AArch64InstPrinter::printLogicalImm64(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printShifter(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Val = MI->getOperand(OpNum).getImm(); // LSL #0 should not be printed. @@ -983,18 +988,21 @@ void AArch64InstPrinter::printShifter(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printShiftedRegister(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { O << getRegisterName(MI->getOperand(OpNum).getReg()); - printShifter(MI, OpNum + 1, O); + printShifter(MI, OpNum + 1, STI, O); } void AArch64InstPrinter::printExtendedRegister(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { O << getRegisterName(MI->getOperand(OpNum).getReg()); - printArithExtend(MI, OpNum + 1, O); + printArithExtend(MI, OpNum + 1, STI, O); } void AArch64InstPrinter::printArithExtend(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Val = MI->getOperand(OpNum).getImm(); AArch64_AM::ShiftExtendType ExtType = AArch64_AM::getArithExtendType(Val); @@ -1038,24 +1046,28 @@ void AArch64InstPrinter::printMemExtend(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printCondCode(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { AArch64CC::CondCode CC = (AArch64CC::CondCode)MI->getOperand(OpNum).getImm(); O << AArch64CC::getCondCodeName(CC); } void AArch64InstPrinter::printInverseCondCode(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { AArch64CC::CondCode CC = (AArch64CC::CondCode)MI->getOperand(OpNum).getImm(); O << AArch64CC::getCondCodeName(AArch64CC::getInvertedCondCode(CC)); } void AArch64InstPrinter::printAMNoIndex(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { O << '[' << getRegisterName(MI->getOperand(OpNum).getReg()) << ']'; } template void AArch64InstPrinter::printImmScale(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { O << '#' << Scale * MI->getOperand(OpNum).getImm(); } @@ -1085,10 +1097,12 @@ void AArch64InstPrinter::printAMIndexedWB(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printPrefetchOp(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned prfop = MI->getOperand(OpNum).getImm(); bool Valid; - StringRef Name = AArch64PRFM::PRFMMapper().toString(prfop, Valid); + StringRef Name = + AArch64PRFM::PRFMMapper().toString(prfop, STI.getFeatureBits(), Valid); if (Valid) O << Name; else @@ -1096,6 +1110,7 @@ void AArch64InstPrinter::printPrefetchOp(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printFPImmOperand(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &MO = MI->getOperand(OpNum); float FPImm = @@ -1151,6 +1166,7 @@ static unsigned getNextVectorRegister(unsigned Reg, unsigned Stride = 1) { } void AArch64InstPrinter::printVectorList(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O, StringRef LayoutSuffix) { unsigned Reg = MI->getOperand(OpNum).getReg(); @@ -1193,14 +1209,17 @@ void AArch64InstPrinter::printVectorList(const MCInst *MI, unsigned OpNum, O << " }"; } -void AArch64InstPrinter::printImplicitlyTypedVectorList(const MCInst *MI, - unsigned OpNum, - raw_ostream &O) { - printVectorList(MI, OpNum, O, ""); +void +AArch64InstPrinter::printImplicitlyTypedVectorList(const MCInst *MI, + unsigned OpNum, + const MCSubtargetInfo &STI, + raw_ostream &O) { + printVectorList(MI, OpNum, STI, O, ""); } template void AArch64InstPrinter::printTypedVectorList(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { std::string Suffix("."); if (NumLanes) @@ -1208,15 +1227,17 @@ void AArch64InstPrinter::printTypedVectorList(const MCInst *MI, unsigned OpNum, else Suffix += LaneKind; - printVectorList(MI, OpNum, O, Suffix); + printVectorList(MI, OpNum, STI, O, Suffix); } void AArch64InstPrinter::printVectorIndex(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { O << "[" << MI->getOperand(OpNum).getImm() << "]"; } void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNum); @@ -1241,6 +1262,7 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printAdrpLabel(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNum); @@ -1256,6 +1278,7 @@ void AArch64InstPrinter::printAdrpLabel(const MCInst *MI, unsigned OpNum, } void AArch64InstPrinter::printBarrierOption(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Val = MI->getOperand(OpNo).getImm(); unsigned Opcode = MI->getOpcode(); @@ -1263,9 +1286,11 @@ void AArch64InstPrinter::printBarrierOption(const MCInst *MI, unsigned OpNo, bool Valid; StringRef Name; if (Opcode == AArch64::ISB) - Name = AArch64ISB::ISBMapper().toString(Val, Valid); + Name = AArch64ISB::ISBMapper().toString(Val, STI.getFeatureBits(), + Valid); else - Name = AArch64DB::DBarrierMapper().toString(Val, Valid); + Name = AArch64DB::DBarrierMapper().toString(Val, STI.getFeatureBits(), + Valid); if (Valid) O << Name; else @@ -1273,31 +1298,35 @@ void AArch64InstPrinter::printBarrierOption(const MCInst *MI, unsigned OpNo, } void AArch64InstPrinter::printMRSSystemRegister(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Val = MI->getOperand(OpNo).getImm(); - auto Mapper = AArch64SysReg::MRSMapper(getAvailableFeatures()); - std::string Name = Mapper.toString(Val); + auto Mapper = AArch64SysReg::MRSMapper(); + std::string Name = Mapper.toString(Val, STI.getFeatureBits()); O << StringRef(Name).upper(); } void AArch64InstPrinter::printMSRSystemRegister(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Val = MI->getOperand(OpNo).getImm(); - auto Mapper = AArch64SysReg::MSRMapper(getAvailableFeatures()); - std::string Name = Mapper.toString(Val); + auto Mapper = AArch64SysReg::MSRMapper(); + std::string Name = Mapper.toString(Val, STI.getFeatureBits()); O << StringRef(Name).upper(); } void AArch64InstPrinter::printSystemPStateField(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Val = MI->getOperand(OpNo).getImm(); bool Valid; - StringRef Name = AArch64PState::PStateMapper().toString(Val, Valid); + StringRef Name = + AArch64PState::PStateMapper().toString(Val, STI.getFeatureBits(), Valid); if (Valid) O << StringRef(Name.str()).upper(); else @@ -1305,6 +1334,7 @@ void AArch64InstPrinter::printSystemPStateField(const MCInst *MI, unsigned OpNo, } void AArch64InstPrinter::printSIMDType10Operand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { unsigned RawVal = MI->getOperand(OpNo).getImm(); uint64_t Val = AArch64_AM::decodeAdvSIMDModImmType10(RawVal); diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h index 5f51621..c2077a0 100644 --- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h +++ b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h @@ -26,16 +26,21 @@ class MCOperand; class AArch64InstPrinter : public MCInstPrinter { public: AArch64InstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII, - const MCRegisterInfo &MRI, const MCSubtargetInfo &STI); + const MCRegisterInfo &MRI); - void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot) override; + void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot, + const MCSubtargetInfo &STI) override; void printRegName(raw_ostream &OS, unsigned RegNo) const override; // Autogenerated by tblgen. - virtual void printInstruction(const MCInst *MI, raw_ostream &O); - virtual bool printAliasInstr(const MCInst *MI, raw_ostream &O); + virtual void printInstruction(const MCInst *MI, const MCSubtargetInfo &STI, + raw_ostream &O); + virtual bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI, + raw_ostream &O); virtual void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx, - unsigned PrintMethodIdx, raw_ostream &O); + unsigned PrintMethodIdx, + const MCSubtargetInfo &STI, + raw_ostream &O); virtual StringRef getRegName(unsigned RegNo) const { return getRegisterName(RegNo); } @@ -45,90 +50,126 @@ public: protected: bool printSysAlias(const MCInst *MI, raw_ostream &O); // Operand printers - void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O); - void printHexImm(const MCInst *MI, unsigned OpNo, raw_ostream &O); + void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, + raw_ostream &O); + void printHexImm(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, + raw_ostream &O); void printPostIncOperand(const MCInst *MI, unsigned OpNo, unsigned Imm, raw_ostream &O); - template - void printPostIncOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + template + void printPostIncOperand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O) { printPostIncOperand(MI, OpNo, Amount, O); } - void printVRegOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O); - void printSysCROperand(const MCInst *MI, unsigned OpNo, raw_ostream &O); - void printAddSubImm(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printLogicalImm32(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printLogicalImm64(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printShifter(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printShiftedRegister(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printExtendedRegister(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printArithExtend(const MCInst *MI, unsigned OpNum, raw_ostream &O); + void printVRegOperand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); + void printSysCROperand(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); + void printAddSubImm(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printLogicalImm32(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printLogicalImm64(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printShifter(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printShiftedRegister(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printExtendedRegister(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printArithExtend(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); void printMemExtend(const MCInst *MI, unsigned OpNum, raw_ostream &O, char SrcRegKind, unsigned Width); template - void printMemExtend(const MCInst *MI, unsigned OpNum, raw_ostream &O) { + void printMemExtend(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { printMemExtend(MI, OpNum, O, SrcRegKind, Width); } - void printCondCode(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printInverseCondCode(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printAlignedLabel(const MCInst *MI, unsigned OpNum, raw_ostream &O); + void printCondCode(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printInverseCondCode(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printAlignedLabel(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); void printUImm12Offset(const MCInst *MI, unsigned OpNum, unsigned Scale, raw_ostream &O); void printAMIndexedWB(const MCInst *MI, unsigned OpNum, unsigned Scale, raw_ostream &O); - template - void printUImm12Offset(const MCInst *MI, unsigned OpNum, raw_ostream &O) { + template + void printUImm12Offset(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { printUImm12Offset(MI, OpNum, Scale, O); } - template - void printAMIndexedWB(const MCInst *MI, unsigned OpNum, raw_ostream &O) { + template + void printAMIndexedWB(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O) { printAMIndexedWB(MI, OpNum, BitWidth / 8, O); } - void printAMNoIndex(const MCInst *MI, unsigned OpNum, raw_ostream &O); + void printAMNoIndex(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); - template - void printImmScale(const MCInst *MI, unsigned OpNum, raw_ostream &O); + template + void printImmScale(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); - void printPrefetchOp(const MCInst *MI, unsigned OpNum, raw_ostream &O); + void printPrefetchOp(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); - void printFPImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O); + void printFPImmOperand(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); - void printVectorList(const MCInst *MI, unsigned OpNum, raw_ostream &O, + void printVectorList(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O, StringRef LayoutSuffix); /// Print a list of vector registers where the type suffix is implicit /// (i.e. attached to the instruction rather than the registers). void printImplicitlyTypedVectorList(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); template - void printTypedVectorList(const MCInst *MI, unsigned OpNum, raw_ostream &O); - - void printVectorIndex(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printAdrpLabel(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printBarrierOption(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printMSRSystemRegister(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printMRSSystemRegister(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printSystemPStateField(const MCInst *MI, unsigned OpNum, raw_ostream &O); - void printSIMDType10Operand(const MCInst *MI, unsigned OpNum, raw_ostream &O); + void printTypedVectorList(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + + void printVectorIndex(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printAdrpLabel(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printBarrierOption(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printMSRSystemRegister(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printMRSSystemRegister(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printSystemPStateField(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); + void printSIMDType10Operand(const MCInst *MI, unsigned OpNum, + const MCSubtargetInfo &STI, raw_ostream &O); }; class AArch64AppleInstPrinter : public AArch64InstPrinter { public: AArch64AppleInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII, - const MCRegisterInfo &MRI, const MCSubtargetInfo &STI); + const MCRegisterInfo &MRI); - void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot) override; + void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot, + const MCSubtargetInfo &STI) override; - void printInstruction(const MCInst *MI, raw_ostream &O) override; - bool printAliasInstr(const MCInst *MI, raw_ostream &O) override; + void printInstruction(const MCInst *MI, const MCSubtargetInfo &STI, + raw_ostream &O) override; + bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI, + raw_ostream &O) override; void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx, unsigned PrintMethodIdx, + const MCSubtargetInfo &STI, raw_ostream &O) override; StringRef getRegName(unsigned RegNo) const override { return getRegisterName(RegNo); diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index 84b63a0..e5eb90c 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -313,7 +313,7 @@ public: DarwinAArch64AsmBackend(const Target &T, const MCRegisterInfo &MRI) : AArch64AsmBackend(T), MRI(MRI) {} - MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { + MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override { return createAArch64MachObjectWriter(OS, MachO::CPU_TYPE_ARM64, MachO::CPU_SUBTYPE_ARM64_ALL); } @@ -461,7 +461,7 @@ public: ELFAArch64AsmBackend(const Target &T, uint8_t OSABI, bool IsLittleEndian) : AArch64AsmBackend(T), OSABI(OSABI), IsLittleEndian(IsLittleEndian) {} - MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { + MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override { return createAArch64ELFObjectWriter(OS, OSABI, IsLittleEndian); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp index 5ea49c3..1f516d1 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp @@ -26,7 +26,7 @@ class AArch64ELFObjectWriter : public MCELFObjectTargetWriter { public: AArch64ELFObjectWriter(uint8_t OSABI, bool IsLittleEndian); - virtual ~AArch64ELFObjectWriter(); + ~AArch64ELFObjectWriter() override; protected: unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup, @@ -248,9 +248,9 @@ unsigned AArch64ELFObjectWriter::GetRelocType(const MCValue &Target, llvm_unreachable("Unimplemented fixup -> relocation"); } -MCObjectWriter *llvm::createAArch64ELFObjectWriter(raw_ostream &OS, - uint8_t OSABI, - bool IsLittleEndian) { +MCObjectWriter *llvm::createAArch64ELFObjectWriter(raw_pwrite_stream &OS, + uint8_t OSABI, + bool IsLittleEndian) { MCELFObjectTargetWriter *MOTW = new AArch64ELFObjectWriter(OSABI, IsLittleEndian); return createELFObjectWriter(MOTW, OS, IsLittleEndian); diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp index 8f780d2..540d1fc 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp @@ -89,12 +89,12 @@ class AArch64ELFStreamer : public MCELFStreamer { public: friend class AArch64TargetELFStreamer; - AArch64ELFStreamer(MCContext &Context, MCAsmBackend &TAB, raw_ostream &OS, - MCCodeEmitter *Emitter) + AArch64ELFStreamer(MCContext &Context, MCAsmBackend &TAB, + raw_pwrite_stream &OS, MCCodeEmitter *Emitter) : MCELFStreamer(Context, TAB, OS, Emitter), MappingSymbolCounter(0), LastEMS(EMS_None) {} - ~AArch64ELFStreamer() {} + ~AArch64ELFStreamer() override {} void ChangeSection(const MCSection *Section, const MCExpr *Subsection) override { @@ -211,8 +211,8 @@ MCTargetStreamer *createAArch64AsmTargetStreamer(MCStreamer &S, } MCELFStreamer *createAArch64ELFStreamer(MCContext &Context, MCAsmBackend &TAB, - raw_ostream &OS, MCCodeEmitter *Emitter, - bool RelaxAll) { + raw_pwrite_stream &OS, + MCCodeEmitter *Emitter, bool RelaxAll) { AArch64ELFStreamer *S = new AArch64ELFStreamer(Context, TAB, OS, Emitter); if (RelaxAll) S->getAssembler().setRelaxAll(true); diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h index 71b05cc..ef48203 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h @@ -19,8 +19,8 @@ namespace llvm { MCELFStreamer *createAArch64ELFStreamer(MCContext &Context, MCAsmBackend &TAB, - raw_ostream &OS, MCCodeEmitter *Emitter, - bool RelaxAll); + raw_pwrite_stream &OS, + MCCodeEmitter *Emitter, bool RelaxAll); } #endif diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp index 9ea49f0..fd4dc47 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp @@ -40,7 +40,7 @@ class AArch64MCCodeEmitter : public MCCodeEmitter { public: AArch64MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) : Ctx(ctx) {} - ~AArch64MCCodeEmitter() {} + ~AArch64MCCodeEmitter() override {} // getBinaryCodeForInstr - TableGen'erated function for getting the // binary encoding for an instruction. diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp index 38b399d..afad674 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp @@ -109,29 +109,28 @@ static MCCodeGenInfo *createAArch64MCCodeGenInfo(StringRef TT, Reloc::Model RM, return X; } -static MCInstPrinter *createAArch64MCInstPrinter(const Target &T, +static MCInstPrinter *createAArch64MCInstPrinter(const Triple &T, unsigned SyntaxVariant, const MCAsmInfo &MAI, const MCInstrInfo &MII, - const MCRegisterInfo &MRI, - const MCSubtargetInfo &STI) { + const MCRegisterInfo &MRI) { if (SyntaxVariant == 0) - return new AArch64InstPrinter(MAI, MII, MRI, STI); + return new AArch64InstPrinter(MAI, MII, MRI); if (SyntaxVariant == 1) - return new AArch64AppleInstPrinter(MAI, MII, MRI, STI); + return new AArch64AppleInstPrinter(MAI, MII, MRI); return nullptr; } static MCStreamer *createELFStreamer(const Triple &T, MCContext &Ctx, - MCAsmBackend &TAB, raw_ostream &OS, + MCAsmBackend &TAB, raw_pwrite_stream &OS, MCCodeEmitter *Emitter, bool RelaxAll) { return createAArch64ELFStreamer(Ctx, TAB, OS, Emitter, RelaxAll); } static MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB, - raw_ostream &OS, MCCodeEmitter *Emitter, - bool RelaxAll, + raw_pwrite_stream &OS, + MCCodeEmitter *Emitter, bool RelaxAll, bool DWARFMustBeAtTheEnd) { return createMachOStreamer(Ctx, TAB, OS, Emitter, RelaxAll, DWARFMustBeAtTheEnd, diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h index 7ce303b..4705bdf 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h @@ -33,6 +33,7 @@ class StringRef; class Target; class Triple; class raw_ostream; +class raw_pwrite_stream; extern Target TheAArch64leTarget; extern Target TheAArch64beTarget; @@ -48,11 +49,13 @@ MCAsmBackend *createAArch64beAsmBackend(const Target &T, const MCRegisterInfo &MRI, StringRef TT, StringRef CPU); -MCObjectWriter *createAArch64ELFObjectWriter(raw_ostream &OS, uint8_t OSABI, +MCObjectWriter *createAArch64ELFObjectWriter(raw_pwrite_stream &OS, + uint8_t OSABI, bool IsLittleEndian); -MCObjectWriter *createAArch64MachObjectWriter(raw_ostream &OS, uint32_t CPUType, - uint32_t CPUSubtype); +MCObjectWriter *createAArch64MachObjectWriter(raw_pwrite_stream &OS, + uint32_t CPUType, + uint32_t CPUSubtype); MCTargetStreamer *createAArch64AsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS, diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp index 0d9385d..61649c4 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp @@ -413,9 +413,9 @@ void AArch64MachObjectWriter::RecordRelocation( Writer->addRelocation(RelSymbol, Fragment->getParent(), MRE); } -MCObjectWriter *llvm::createAArch64MachObjectWriter(raw_ostream &OS, - uint32_t CPUType, - uint32_t CPUSubtype) { +MCObjectWriter *llvm::createAArch64MachObjectWriter(raw_pwrite_stream &OS, + uint32_t CPUType, + uint32_t CPUSubtype) { return createMachObjectWriter( new AArch64MachObjectWriter(CPUType, CPUSubtype), OS, /*IsLittleEndian=*/true); diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp index 160c1c5..8696163 100644 --- a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp +++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp @@ -18,9 +18,10 @@ using namespace llvm; -StringRef AArch64NamedImmMapper::toString(uint32_t Value, bool &Valid) const { +StringRef AArch64NamedImmMapper::toString(uint32_t Value, uint64_t FeatureBits, + bool &Valid) const { for (unsigned i = 0; i < NumMappings; ++i) { - if (Mappings[i].Value == Value) { + if (Mappings[i].isValueEqual(Value, FeatureBits)) { Valid = true; return Mappings[i].Name; } @@ -30,10 +31,11 @@ StringRef AArch64NamedImmMapper::toString(uint32_t Value, bool &Valid) const { return StringRef(); } -uint32_t AArch64NamedImmMapper::fromString(StringRef Name, bool &Valid) const { +uint32_t AArch64NamedImmMapper::fromString(StringRef Name, uint64_t FeatureBits, + bool &Valid) const { std::string LowerCaseName = Name.lower(); for (unsigned i = 0; i < NumMappings; ++i) { - if (Mappings[i].Name == LowerCaseName) { + if (Mappings[i].isNameEqual(LowerCaseName, FeatureBits)) { Valid = true; return Mappings[i].Value; } @@ -48,744 +50,776 @@ bool AArch64NamedImmMapper::validImm(uint32_t Value) const { } const AArch64NamedImmMapper::Mapping AArch64AT::ATMapper::ATMappings[] = { - {"s1e1r", S1E1R}, - {"s1e2r", S1E2R}, - {"s1e3r", S1E3R}, - {"s1e1w", S1E1W}, - {"s1e2w", S1E2W}, - {"s1e3w", S1E3W}, - {"s1e0r", S1E0R}, - {"s1e0w", S1E0W}, - {"s12e1r", S12E1R}, - {"s12e1w", S12E1W}, - {"s12e0r", S12E0R}, - {"s12e0w", S12E0W}, + {"s1e1r", S1E1R, 0}, + {"s1e2r", S1E2R, 0}, + {"s1e3r", S1E3R, 0}, + {"s1e1w", S1E1W, 0}, + {"s1e2w", S1E2W, 0}, + {"s1e3w", S1E3W, 0}, + {"s1e0r", S1E0R, 0}, + {"s1e0w", S1E0W, 0}, + {"s12e1r", S12E1R, 0}, + {"s12e1w", S12E1W, 0}, + {"s12e0r", S12E0R, 0}, + {"s12e0w", S12E0W, 0}, }; AArch64AT::ATMapper::ATMapper() : AArch64NamedImmMapper(ATMappings, 0) {} const AArch64NamedImmMapper::Mapping AArch64DB::DBarrierMapper::DBarrierMappings[] = { - {"oshld", OSHLD}, - {"oshst", OSHST}, - {"osh", OSH}, - {"nshld", NSHLD}, - {"nshst", NSHST}, - {"nsh", NSH}, - {"ishld", ISHLD}, - {"ishst", ISHST}, - {"ish", ISH}, - {"ld", LD}, - {"st", ST}, - {"sy", SY} + {"oshld", OSHLD, 0}, + {"oshst", OSHST, 0}, + {"osh", OSH, 0}, + {"nshld", NSHLD, 0}, + {"nshst", NSHST, 0}, + {"nsh", NSH, 0}, + {"ishld", ISHLD, 0}, + {"ishst", ISHST, 0}, + {"ish", ISH, 0}, + {"ld", LD, 0}, + {"st", ST, 0}, + {"sy", SY, 0} }; AArch64DB::DBarrierMapper::DBarrierMapper() : AArch64NamedImmMapper(DBarrierMappings, 16u) {} const AArch64NamedImmMapper::Mapping AArch64DC::DCMapper::DCMappings[] = { - {"zva", ZVA}, - {"ivac", IVAC}, - {"isw", ISW}, - {"cvac", CVAC}, - {"csw", CSW}, - {"cvau", CVAU}, - {"civac", CIVAC}, - {"cisw", CISW} + {"zva", ZVA, 0}, + {"ivac", IVAC, 0}, + {"isw", ISW, 0}, + {"cvac", CVAC, 0}, + {"csw", CSW, 0}, + {"cvau", CVAU, 0}, + {"civac", CIVAC, 0}, + {"cisw", CISW, 0} }; AArch64DC::DCMapper::DCMapper() : AArch64NamedImmMapper(DCMappings, 0) {} const AArch64NamedImmMapper::Mapping AArch64IC::ICMapper::ICMappings[] = { - {"ialluis", IALLUIS}, - {"iallu", IALLU}, - {"ivau", IVAU} + {"ialluis", IALLUIS, 0}, + {"iallu", IALLU, 0}, + {"ivau", IVAU, 0} }; AArch64IC::ICMapper::ICMapper() : AArch64NamedImmMapper(ICMappings, 0) {} const AArch64NamedImmMapper::Mapping AArch64ISB::ISBMapper::ISBMappings[] = { - {"sy", SY}, + {"sy", SY, 0}, }; AArch64ISB::ISBMapper::ISBMapper() : AArch64NamedImmMapper(ISBMappings, 16) {} const AArch64NamedImmMapper::Mapping AArch64PRFM::PRFMMapper::PRFMMappings[] = { - {"pldl1keep", PLDL1KEEP}, - {"pldl1strm", PLDL1STRM}, - {"pldl2keep", PLDL2KEEP}, - {"pldl2strm", PLDL2STRM}, - {"pldl3keep", PLDL3KEEP}, - {"pldl3strm", PLDL3STRM}, - {"plil1keep", PLIL1KEEP}, - {"plil1strm", PLIL1STRM}, - {"plil2keep", PLIL2KEEP}, - {"plil2strm", PLIL2STRM}, - {"plil3keep", PLIL3KEEP}, - {"plil3strm", PLIL3STRM}, - {"pstl1keep", PSTL1KEEP}, - {"pstl1strm", PSTL1STRM}, - {"pstl2keep", PSTL2KEEP}, - {"pstl2strm", PSTL2STRM}, - {"pstl3keep", PSTL3KEEP}, - {"pstl3strm", PSTL3STRM} + {"pldl1keep", PLDL1KEEP, 0}, + {"pldl1strm", PLDL1STRM, 0}, + {"pldl2keep", PLDL2KEEP, 0}, + {"pldl2strm", PLDL2STRM, 0}, + {"pldl3keep", PLDL3KEEP, 0}, + {"pldl3strm", PLDL3STRM, 0}, + {"plil1keep", PLIL1KEEP, 0}, + {"plil1strm", PLIL1STRM, 0}, + {"plil2keep", PLIL2KEEP, 0}, + {"plil2strm", PLIL2STRM, 0}, + {"plil3keep", PLIL3KEEP, 0}, + {"plil3strm", PLIL3STRM, 0}, + {"pstl1keep", PSTL1KEEP, 0}, + {"pstl1strm", PSTL1STRM, 0}, + {"pstl2keep", PSTL2KEEP, 0}, + {"pstl2strm", PSTL2STRM, 0}, + {"pstl3keep", PSTL3KEEP, 0}, + {"pstl3strm", PSTL3STRM, 0} }; AArch64PRFM::PRFMMapper::PRFMMapper() : AArch64NamedImmMapper(PRFMMappings, 32) {} const AArch64NamedImmMapper::Mapping AArch64PState::PStateMapper::PStateMappings[] = { - {"spsel", SPSel}, - {"daifset", DAIFSet}, - {"daifclr", DAIFClr} + {"spsel", SPSel, 0}, + {"daifset", DAIFSet, 0}, + {"daifclr", DAIFClr, 0}, + + // v8.1a "Privileged Access Never" extension-specific PStates + {"pan", PAN, AArch64::HasV8_1aOps}, }; AArch64PState::PStateMapper::PStateMapper() : AArch64NamedImmMapper(PStateMappings, 0) {} const AArch64NamedImmMapper::Mapping AArch64SysReg::MRSMapper::MRSMappings[] = { - {"mdccsr_el0", MDCCSR_EL0}, - {"dbgdtrrx_el0", DBGDTRRX_EL0}, - {"mdrar_el1", MDRAR_EL1}, - {"oslsr_el1", OSLSR_EL1}, - {"dbgauthstatus_el1", DBGAUTHSTATUS_EL1}, - {"pmceid0_el0", PMCEID0_EL0}, - {"pmceid1_el0", PMCEID1_EL0}, - {"midr_el1", MIDR_EL1}, - {"ccsidr_el1", CCSIDR_EL1}, - {"clidr_el1", CLIDR_EL1}, - {"ctr_el0", CTR_EL0}, - {"mpidr_el1", MPIDR_EL1}, - {"revidr_el1", REVIDR_EL1}, - {"aidr_el1", AIDR_EL1}, - {"dczid_el0", DCZID_EL0}, - {"id_pfr0_el1", ID_PFR0_EL1}, - {"id_pfr1_el1", ID_PFR1_EL1}, - {"id_dfr0_el1", ID_DFR0_EL1}, - {"id_afr0_el1", ID_AFR0_EL1}, - {"id_mmfr0_el1", ID_MMFR0_EL1}, - {"id_mmfr1_el1", ID_MMFR1_EL1}, - {"id_mmfr2_el1", ID_MMFR2_EL1}, - {"id_mmfr3_el1", ID_MMFR3_EL1}, - {"id_isar0_el1", ID_ISAR0_EL1}, - {"id_isar1_el1", ID_ISAR1_EL1}, - {"id_isar2_el1", ID_ISAR2_EL1}, - {"id_isar3_el1", ID_ISAR3_EL1}, - {"id_isar4_el1", ID_ISAR4_EL1}, - {"id_isar5_el1", ID_ISAR5_EL1}, - {"id_aa64pfr0_el1", ID_A64PFR0_EL1}, - {"id_aa64pfr1_el1", ID_A64PFR1_EL1}, - {"id_aa64dfr0_el1", ID_A64DFR0_EL1}, - {"id_aa64dfr1_el1", ID_A64DFR1_EL1}, - {"id_aa64afr0_el1", ID_A64AFR0_EL1}, - {"id_aa64afr1_el1", ID_A64AFR1_EL1}, - {"id_aa64isar0_el1", ID_A64ISAR0_EL1}, - {"id_aa64isar1_el1", ID_A64ISAR1_EL1}, - {"id_aa64mmfr0_el1", ID_A64MMFR0_EL1}, - {"id_aa64mmfr1_el1", ID_A64MMFR1_EL1}, - {"mvfr0_el1", MVFR0_EL1}, - {"mvfr1_el1", MVFR1_EL1}, - {"mvfr2_el1", MVFR2_EL1}, - {"rvbar_el1", RVBAR_EL1}, - {"rvbar_el2", RVBAR_EL2}, - {"rvbar_el3", RVBAR_EL3}, - {"isr_el1", ISR_EL1}, - {"cntpct_el0", CNTPCT_EL0}, - {"cntvct_el0", CNTVCT_EL0}, + {"mdccsr_el0", MDCCSR_EL0, 0}, + {"dbgdtrrx_el0", DBGDTRRX_EL0, 0}, + {"mdrar_el1", MDRAR_EL1, 0}, + {"oslsr_el1", OSLSR_EL1, 0}, + {"dbgauthstatus_el1", DBGAUTHSTATUS_EL1, 0}, + {"pmceid0_el0", PMCEID0_EL0, 0}, + {"pmceid1_el0", PMCEID1_EL0, 0}, + {"midr_el1", MIDR_EL1, 0}, + {"ccsidr_el1", CCSIDR_EL1, 0}, + {"clidr_el1", CLIDR_EL1, 0}, + {"ctr_el0", CTR_EL0, 0}, + {"mpidr_el1", MPIDR_EL1, 0}, + {"revidr_el1", REVIDR_EL1, 0}, + {"aidr_el1", AIDR_EL1, 0}, + {"dczid_el0", DCZID_EL0, 0}, + {"id_pfr0_el1", ID_PFR0_EL1, 0}, + {"id_pfr1_el1", ID_PFR1_EL1, 0}, + {"id_dfr0_el1", ID_DFR0_EL1, 0}, + {"id_afr0_el1", ID_AFR0_EL1, 0}, + {"id_mmfr0_el1", ID_MMFR0_EL1, 0}, + {"id_mmfr1_el1", ID_MMFR1_EL1, 0}, + {"id_mmfr2_el1", ID_MMFR2_EL1, 0}, + {"id_mmfr3_el1", ID_MMFR3_EL1, 0}, + {"id_isar0_el1", ID_ISAR0_EL1, 0}, + {"id_isar1_el1", ID_ISAR1_EL1, 0}, + {"id_isar2_el1", ID_ISAR2_EL1, 0}, + {"id_isar3_el1", ID_ISAR3_EL1, 0}, + {"id_isar4_el1", ID_ISAR4_EL1, 0}, + {"id_isar5_el1", ID_ISAR5_EL1, 0}, + {"id_aa64pfr0_el1", ID_A64PFR0_EL1, 0}, + {"id_aa64pfr1_el1", ID_A64PFR1_EL1, 0}, + {"id_aa64dfr0_el1", ID_A64DFR0_EL1, 0}, + {"id_aa64dfr1_el1", ID_A64DFR1_EL1, 0}, + {"id_aa64afr0_el1", ID_A64AFR0_EL1, 0}, + {"id_aa64afr1_el1", ID_A64AFR1_EL1, 0}, + {"id_aa64isar0_el1", ID_A64ISAR0_EL1, 0}, + {"id_aa64isar1_el1", ID_A64ISAR1_EL1, 0}, + {"id_aa64mmfr0_el1", ID_A64MMFR0_EL1, 0}, + {"id_aa64mmfr1_el1", ID_A64MMFR1_EL1, 0}, + {"mvfr0_el1", MVFR0_EL1, 0}, + {"mvfr1_el1", MVFR1_EL1, 0}, + {"mvfr2_el1", MVFR2_EL1, 0}, + {"rvbar_el1", RVBAR_EL1, 0}, + {"rvbar_el2", RVBAR_EL2, 0}, + {"rvbar_el3", RVBAR_EL3, 0}, + {"isr_el1", ISR_EL1, 0}, + {"cntpct_el0", CNTPCT_EL0, 0}, + {"cntvct_el0", CNTVCT_EL0, 0}, // Trace registers - {"trcstatr", TRCSTATR}, - {"trcidr8", TRCIDR8}, - {"trcidr9", TRCIDR9}, - {"trcidr10", TRCIDR10}, - {"trcidr11", TRCIDR11}, - {"trcidr12", TRCIDR12}, - {"trcidr13", TRCIDR13}, - {"trcidr0", TRCIDR0}, - {"trcidr1", TRCIDR1}, - {"trcidr2", TRCIDR2}, - {"trcidr3", TRCIDR3}, - {"trcidr4", TRCIDR4}, - {"trcidr5", TRCIDR5}, - {"trcidr6", TRCIDR6}, - {"trcidr7", TRCIDR7}, - {"trcoslsr", TRCOSLSR}, - {"trcpdsr", TRCPDSR}, - {"trcdevaff0", TRCDEVAFF0}, - {"trcdevaff1", TRCDEVAFF1}, - {"trclsr", TRCLSR}, - {"trcauthstatus", TRCAUTHSTATUS}, - {"trcdevarch", TRCDEVARCH}, - {"trcdevid", TRCDEVID}, - {"trcdevtype", TRCDEVTYPE}, - {"trcpidr4", TRCPIDR4}, - {"trcpidr5", TRCPIDR5}, - {"trcpidr6", TRCPIDR6}, - {"trcpidr7", TRCPIDR7}, - {"trcpidr0", TRCPIDR0}, - {"trcpidr1", TRCPIDR1}, - {"trcpidr2", TRCPIDR2}, - {"trcpidr3", TRCPIDR3}, - {"trccidr0", TRCCIDR0}, - {"trccidr1", TRCCIDR1}, - {"trccidr2", TRCCIDR2}, - {"trccidr3", TRCCIDR3}, + {"trcstatr", TRCSTATR, 0}, + {"trcidr8", TRCIDR8, 0}, + {"trcidr9", TRCIDR9, 0}, + {"trcidr10", TRCIDR10, 0}, + {"trcidr11", TRCIDR11, 0}, + {"trcidr12", TRCIDR12, 0}, + {"trcidr13", TRCIDR13, 0}, + {"trcidr0", TRCIDR0, 0}, + {"trcidr1", TRCIDR1, 0}, + {"trcidr2", TRCIDR2, 0}, + {"trcidr3", TRCIDR3, 0}, + {"trcidr4", TRCIDR4, 0}, + {"trcidr5", TRCIDR5, 0}, + {"trcidr6", TRCIDR6, 0}, + {"trcidr7", TRCIDR7, 0}, + {"trcoslsr", TRCOSLSR, 0}, + {"trcpdsr", TRCPDSR, 0}, + {"trcdevaff0", TRCDEVAFF0, 0}, + {"trcdevaff1", TRCDEVAFF1, 0}, + {"trclsr", TRCLSR, 0}, + {"trcauthstatus", TRCAUTHSTATUS, 0}, + {"trcdevarch", TRCDEVARCH, 0}, + {"trcdevid", TRCDEVID, 0}, + {"trcdevtype", TRCDEVTYPE, 0}, + {"trcpidr4", TRCPIDR4, 0}, + {"trcpidr5", TRCPIDR5, 0}, + {"trcpidr6", TRCPIDR6, 0}, + {"trcpidr7", TRCPIDR7, 0}, + {"trcpidr0", TRCPIDR0, 0}, + {"trcpidr1", TRCPIDR1, 0}, + {"trcpidr2", TRCPIDR2, 0}, + {"trcpidr3", TRCPIDR3, 0}, + {"trccidr0", TRCCIDR0, 0}, + {"trccidr1", TRCCIDR1, 0}, + {"trccidr2", TRCCIDR2, 0}, + {"trccidr3", TRCCIDR3, 0}, // GICv3 registers - {"icc_iar1_el1", ICC_IAR1_EL1}, - {"icc_iar0_el1", ICC_IAR0_EL1}, - {"icc_hppir1_el1", ICC_HPPIR1_EL1}, - {"icc_hppir0_el1", ICC_HPPIR0_EL1}, - {"icc_rpr_el1", ICC_RPR_EL1}, - {"ich_vtr_el2", ICH_VTR_EL2}, - {"ich_eisr_el2", ICH_EISR_EL2}, - {"ich_elsr_el2", ICH_ELSR_EL2} + {"icc_iar1_el1", ICC_IAR1_EL1, 0}, + {"icc_iar0_el1", ICC_IAR0_EL1, 0}, + {"icc_hppir1_el1", ICC_HPPIR1_EL1, 0}, + {"icc_hppir0_el1", ICC_HPPIR0_EL1, 0}, + {"icc_rpr_el1", ICC_RPR_EL1, 0}, + {"ich_vtr_el2", ICH_VTR_EL2, 0}, + {"ich_eisr_el2", ICH_EISR_EL2, 0}, + {"ich_elsr_el2", ICH_ELSR_EL2, 0} }; -AArch64SysReg::MRSMapper::MRSMapper(uint64_t FeatureBits) - : SysRegMapper(FeatureBits) { +AArch64SysReg::MRSMapper::MRSMapper() { InstMappings = &MRSMappings[0]; NumInstMappings = llvm::array_lengthof(MRSMappings); } const AArch64NamedImmMapper::Mapping AArch64SysReg::MSRMapper::MSRMappings[] = { - {"dbgdtrtx_el0", DBGDTRTX_EL0}, - {"oslar_el1", OSLAR_EL1}, - {"pmswinc_el0", PMSWINC_EL0}, + {"dbgdtrtx_el0", DBGDTRTX_EL0, 0}, + {"oslar_el1", OSLAR_EL1, 0}, + {"pmswinc_el0", PMSWINC_EL0, 0}, // Trace registers - {"trcoslar", TRCOSLAR}, - {"trclar", TRCLAR}, + {"trcoslar", TRCOSLAR, 0}, + {"trclar", TRCLAR, 0}, // GICv3 registers - {"icc_eoir1_el1", ICC_EOIR1_EL1}, - {"icc_eoir0_el1", ICC_EOIR0_EL1}, - {"icc_dir_el1", ICC_DIR_EL1}, - {"icc_sgi1r_el1", ICC_SGI1R_EL1}, - {"icc_asgi1r_el1", ICC_ASGI1R_EL1}, - {"icc_sgi0r_el1", ICC_SGI0R_EL1} + {"icc_eoir1_el1", ICC_EOIR1_EL1, 0}, + {"icc_eoir0_el1", ICC_EOIR0_EL1, 0}, + {"icc_dir_el1", ICC_DIR_EL1, 0}, + {"icc_sgi1r_el1", ICC_SGI1R_EL1, 0}, + {"icc_asgi1r_el1", ICC_ASGI1R_EL1, 0}, + {"icc_sgi0r_el1", ICC_SGI0R_EL1, 0}, + + // v8.1a "Privileged Access Never" extension-specific system registers + {"pan", PAN, AArch64::HasV8_1aOps}, }; -AArch64SysReg::MSRMapper::MSRMapper(uint64_t FeatureBits) - : SysRegMapper(FeatureBits) { +AArch64SysReg::MSRMapper::MSRMapper() { InstMappings = &MSRMappings[0]; NumInstMappings = llvm::array_lengthof(MSRMappings); } const AArch64NamedImmMapper::Mapping AArch64SysReg::SysRegMapper::SysRegMappings[] = { - {"osdtrrx_el1", OSDTRRX_EL1}, - {"osdtrtx_el1", OSDTRTX_EL1}, - {"teecr32_el1", TEECR32_EL1}, - {"mdccint_el1", MDCCINT_EL1}, - {"mdscr_el1", MDSCR_EL1}, - {"dbgdtr_el0", DBGDTR_EL0}, - {"oseccr_el1", OSECCR_EL1}, - {"dbgvcr32_el2", DBGVCR32_EL2}, - {"dbgbvr0_el1", DBGBVR0_EL1}, - {"dbgbvr1_el1", DBGBVR1_EL1}, - {"dbgbvr2_el1", DBGBVR2_EL1}, - {"dbgbvr3_el1", DBGBVR3_EL1}, - {"dbgbvr4_el1", DBGBVR4_EL1}, - {"dbgbvr5_el1", DBGBVR5_EL1}, - {"dbgbvr6_el1", DBGBVR6_EL1}, - {"dbgbvr7_el1", DBGBVR7_EL1}, - {"dbgbvr8_el1", DBGBVR8_EL1}, - {"dbgbvr9_el1", DBGBVR9_EL1}, - {"dbgbvr10_el1", DBGBVR10_EL1}, - {"dbgbvr11_el1", DBGBVR11_EL1}, - {"dbgbvr12_el1", DBGBVR12_EL1}, - {"dbgbvr13_el1", DBGBVR13_EL1}, - {"dbgbvr14_el1", DBGBVR14_EL1}, - {"dbgbvr15_el1", DBGBVR15_EL1}, - {"dbgbcr0_el1", DBGBCR0_EL1}, - {"dbgbcr1_el1", DBGBCR1_EL1}, - {"dbgbcr2_el1", DBGBCR2_EL1}, - {"dbgbcr3_el1", DBGBCR3_EL1}, - {"dbgbcr4_el1", DBGBCR4_EL1}, - {"dbgbcr5_el1", DBGBCR5_EL1}, - {"dbgbcr6_el1", DBGBCR6_EL1}, - {"dbgbcr7_el1", DBGBCR7_EL1}, - {"dbgbcr8_el1", DBGBCR8_EL1}, - {"dbgbcr9_el1", DBGBCR9_EL1}, - {"dbgbcr10_el1", DBGBCR10_EL1}, - {"dbgbcr11_el1", DBGBCR11_EL1}, - {"dbgbcr12_el1", DBGBCR12_EL1}, - {"dbgbcr13_el1", DBGBCR13_EL1}, - {"dbgbcr14_el1", DBGBCR14_EL1}, - {"dbgbcr15_el1", DBGBCR15_EL1}, - {"dbgwvr0_el1", DBGWVR0_EL1}, - {"dbgwvr1_el1", DBGWVR1_EL1}, - {"dbgwvr2_el1", DBGWVR2_EL1}, - {"dbgwvr3_el1", DBGWVR3_EL1}, - {"dbgwvr4_el1", DBGWVR4_EL1}, - {"dbgwvr5_el1", DBGWVR5_EL1}, - {"dbgwvr6_el1", DBGWVR6_EL1}, - {"dbgwvr7_el1", DBGWVR7_EL1}, - {"dbgwvr8_el1", DBGWVR8_EL1}, - {"dbgwvr9_el1", DBGWVR9_EL1}, - {"dbgwvr10_el1", DBGWVR10_EL1}, - {"dbgwvr11_el1", DBGWVR11_EL1}, - {"dbgwvr12_el1", DBGWVR12_EL1}, - {"dbgwvr13_el1", DBGWVR13_EL1}, - {"dbgwvr14_el1", DBGWVR14_EL1}, - {"dbgwvr15_el1", DBGWVR15_EL1}, - {"dbgwcr0_el1", DBGWCR0_EL1}, - {"dbgwcr1_el1", DBGWCR1_EL1}, - {"dbgwcr2_el1", DBGWCR2_EL1}, - {"dbgwcr3_el1", DBGWCR3_EL1}, - {"dbgwcr4_el1", DBGWCR4_EL1}, - {"dbgwcr5_el1", DBGWCR5_EL1}, - {"dbgwcr6_el1", DBGWCR6_EL1}, - {"dbgwcr7_el1", DBGWCR7_EL1}, - {"dbgwcr8_el1", DBGWCR8_EL1}, - {"dbgwcr9_el1", DBGWCR9_EL1}, - {"dbgwcr10_el1", DBGWCR10_EL1}, - {"dbgwcr11_el1", DBGWCR11_EL1}, - {"dbgwcr12_el1", DBGWCR12_EL1}, - {"dbgwcr13_el1", DBGWCR13_EL1}, - {"dbgwcr14_el1", DBGWCR14_EL1}, - {"dbgwcr15_el1", DBGWCR15_EL1}, - {"teehbr32_el1", TEEHBR32_EL1}, - {"osdlr_el1", OSDLR_EL1}, - {"dbgprcr_el1", DBGPRCR_EL1}, - {"dbgclaimset_el1", DBGCLAIMSET_EL1}, - {"dbgclaimclr_el1", DBGCLAIMCLR_EL1}, - {"csselr_el1", CSSELR_EL1}, - {"vpidr_el2", VPIDR_EL2}, - {"vmpidr_el2", VMPIDR_EL2}, - {"sctlr_el1", SCTLR_EL1}, - {"sctlr_el2", SCTLR_EL2}, - {"sctlr_el3", SCTLR_EL3}, - {"actlr_el1", ACTLR_EL1}, - {"actlr_el2", ACTLR_EL2}, - {"actlr_el3", ACTLR_EL3}, - {"cpacr_el1", CPACR_EL1}, - {"hcr_el2", HCR_EL2}, - {"scr_el3", SCR_EL3}, - {"mdcr_el2", MDCR_EL2}, - {"sder32_el3", SDER32_EL3}, - {"cptr_el2", CPTR_EL2}, - {"cptr_el3", CPTR_EL3}, - {"hstr_el2", HSTR_EL2}, - {"hacr_el2", HACR_EL2}, - {"mdcr_el3", MDCR_EL3}, - {"ttbr0_el1", TTBR0_EL1}, - {"ttbr0_el2", TTBR0_EL2}, - {"ttbr0_el3", TTBR0_EL3}, - {"ttbr1_el1", TTBR1_EL1}, - {"tcr_el1", TCR_EL1}, - {"tcr_el2", TCR_EL2}, - {"tcr_el3", TCR_EL3}, - {"vttbr_el2", VTTBR_EL2}, - {"vtcr_el2", VTCR_EL2}, - {"dacr32_el2", DACR32_EL2}, - {"spsr_el1", SPSR_EL1}, - {"spsr_el2", SPSR_EL2}, - {"spsr_el3", SPSR_EL3}, - {"elr_el1", ELR_EL1}, - {"elr_el2", ELR_EL2}, - {"elr_el3", ELR_EL3}, - {"sp_el0", SP_EL0}, - {"sp_el1", SP_EL1}, - {"sp_el2", SP_EL2}, - {"spsel", SPSel}, - {"nzcv", NZCV}, - {"daif", DAIF}, - {"currentel", CurrentEL}, - {"spsr_irq", SPSR_irq}, - {"spsr_abt", SPSR_abt}, - {"spsr_und", SPSR_und}, - {"spsr_fiq", SPSR_fiq}, - {"fpcr", FPCR}, - {"fpsr", FPSR}, - {"dspsr_el0", DSPSR_EL0}, - {"dlr_el0", DLR_EL0}, - {"ifsr32_el2", IFSR32_EL2}, - {"afsr0_el1", AFSR0_EL1}, - {"afsr0_el2", AFSR0_EL2}, - {"afsr0_el3", AFSR0_EL3}, - {"afsr1_el1", AFSR1_EL1}, - {"afsr1_el2", AFSR1_EL2}, - {"afsr1_el3", AFSR1_EL3}, - {"esr_el1", ESR_EL1}, - {"esr_el2", ESR_EL2}, - {"esr_el3", ESR_EL3}, - {"fpexc32_el2", FPEXC32_EL2}, - {"far_el1", FAR_EL1}, - {"far_el2", FAR_EL2}, - {"far_el3", FAR_EL3}, - {"hpfar_el2", HPFAR_EL2}, - {"par_el1", PAR_EL1}, - {"pmcr_el0", PMCR_EL0}, - {"pmcntenset_el0", PMCNTENSET_EL0}, - {"pmcntenclr_el0", PMCNTENCLR_EL0}, - {"pmovsclr_el0", PMOVSCLR_EL0}, - {"pmselr_el0", PMSELR_EL0}, - {"pmccntr_el0", PMCCNTR_EL0}, - {"pmxevtyper_el0", PMXEVTYPER_EL0}, - {"pmxevcntr_el0", PMXEVCNTR_EL0}, - {"pmuserenr_el0", PMUSERENR_EL0}, - {"pmintenset_el1", PMINTENSET_EL1}, - {"pmintenclr_el1", PMINTENCLR_EL1}, - {"pmovsset_el0", PMOVSSET_EL0}, - {"mair_el1", MAIR_EL1}, - {"mair_el2", MAIR_EL2}, - {"mair_el3", MAIR_EL3}, - {"amair_el1", AMAIR_EL1}, - {"amair_el2", AMAIR_EL2}, - {"amair_el3", AMAIR_EL3}, - {"vbar_el1", VBAR_EL1}, - {"vbar_el2", VBAR_EL2}, - {"vbar_el3", VBAR_EL3}, - {"rmr_el1", RMR_EL1}, - {"rmr_el2", RMR_EL2}, - {"rmr_el3", RMR_EL3}, - {"contextidr_el1", CONTEXTIDR_EL1}, - {"tpidr_el0", TPIDR_EL0}, - {"tpidr_el2", TPIDR_EL2}, - {"tpidr_el3", TPIDR_EL3}, - {"tpidrro_el0", TPIDRRO_EL0}, - {"tpidr_el1", TPIDR_EL1}, - {"cntfrq_el0", CNTFRQ_EL0}, - {"cntvoff_el2", CNTVOFF_EL2}, - {"cntkctl_el1", CNTKCTL_EL1}, - {"cnthctl_el2", CNTHCTL_EL2}, - {"cntp_tval_el0", CNTP_TVAL_EL0}, - {"cnthp_tval_el2", CNTHP_TVAL_EL2}, - {"cntps_tval_el1", CNTPS_TVAL_EL1}, - {"cntp_ctl_el0", CNTP_CTL_EL0}, - {"cnthp_ctl_el2", CNTHP_CTL_EL2}, - {"cntps_ctl_el1", CNTPS_CTL_EL1}, - {"cntp_cval_el0", CNTP_CVAL_EL0}, - {"cnthp_cval_el2", CNTHP_CVAL_EL2}, - {"cntps_cval_el1", CNTPS_CVAL_EL1}, - {"cntv_tval_el0", CNTV_TVAL_EL0}, - {"cntv_ctl_el0", CNTV_CTL_EL0}, - {"cntv_cval_el0", CNTV_CVAL_EL0}, - {"pmevcntr0_el0", PMEVCNTR0_EL0}, - {"pmevcntr1_el0", PMEVCNTR1_EL0}, - {"pmevcntr2_el0", PMEVCNTR2_EL0}, - {"pmevcntr3_el0", PMEVCNTR3_EL0}, - {"pmevcntr4_el0", PMEVCNTR4_EL0}, - {"pmevcntr5_el0", PMEVCNTR5_EL0}, - {"pmevcntr6_el0", PMEVCNTR6_EL0}, - {"pmevcntr7_el0", PMEVCNTR7_EL0}, - {"pmevcntr8_el0", PMEVCNTR8_EL0}, - {"pmevcntr9_el0", PMEVCNTR9_EL0}, - {"pmevcntr10_el0", PMEVCNTR10_EL0}, - {"pmevcntr11_el0", PMEVCNTR11_EL0}, - {"pmevcntr12_el0", PMEVCNTR12_EL0}, - {"pmevcntr13_el0", PMEVCNTR13_EL0}, - {"pmevcntr14_el0", PMEVCNTR14_EL0}, - {"pmevcntr15_el0", PMEVCNTR15_EL0}, - {"pmevcntr16_el0", PMEVCNTR16_EL0}, - {"pmevcntr17_el0", PMEVCNTR17_EL0}, - {"pmevcntr18_el0", PMEVCNTR18_EL0}, - {"pmevcntr19_el0", PMEVCNTR19_EL0}, - {"pmevcntr20_el0", PMEVCNTR20_EL0}, - {"pmevcntr21_el0", PMEVCNTR21_EL0}, - {"pmevcntr22_el0", PMEVCNTR22_EL0}, - {"pmevcntr23_el0", PMEVCNTR23_EL0}, - {"pmevcntr24_el0", PMEVCNTR24_EL0}, - {"pmevcntr25_el0", PMEVCNTR25_EL0}, - {"pmevcntr26_el0", PMEVCNTR26_EL0}, - {"pmevcntr27_el0", PMEVCNTR27_EL0}, - {"pmevcntr28_el0", PMEVCNTR28_EL0}, - {"pmevcntr29_el0", PMEVCNTR29_EL0}, - {"pmevcntr30_el0", PMEVCNTR30_EL0}, - {"pmccfiltr_el0", PMCCFILTR_EL0}, - {"pmevtyper0_el0", PMEVTYPER0_EL0}, - {"pmevtyper1_el0", PMEVTYPER1_EL0}, - {"pmevtyper2_el0", PMEVTYPER2_EL0}, - {"pmevtyper3_el0", PMEVTYPER3_EL0}, - {"pmevtyper4_el0", PMEVTYPER4_EL0}, - {"pmevtyper5_el0", PMEVTYPER5_EL0}, - {"pmevtyper6_el0", PMEVTYPER6_EL0}, - {"pmevtyper7_el0", PMEVTYPER7_EL0}, - {"pmevtyper8_el0", PMEVTYPER8_EL0}, - {"pmevtyper9_el0", PMEVTYPER9_EL0}, - {"pmevtyper10_el0", PMEVTYPER10_EL0}, - {"pmevtyper11_el0", PMEVTYPER11_EL0}, - {"pmevtyper12_el0", PMEVTYPER12_EL0}, - {"pmevtyper13_el0", PMEVTYPER13_EL0}, - {"pmevtyper14_el0", PMEVTYPER14_EL0}, - {"pmevtyper15_el0", PMEVTYPER15_EL0}, - {"pmevtyper16_el0", PMEVTYPER16_EL0}, - {"pmevtyper17_el0", PMEVTYPER17_EL0}, - {"pmevtyper18_el0", PMEVTYPER18_EL0}, - {"pmevtyper19_el0", PMEVTYPER19_EL0}, - {"pmevtyper20_el0", PMEVTYPER20_EL0}, - {"pmevtyper21_el0", PMEVTYPER21_EL0}, - {"pmevtyper22_el0", PMEVTYPER22_EL0}, - {"pmevtyper23_el0", PMEVTYPER23_EL0}, - {"pmevtyper24_el0", PMEVTYPER24_EL0}, - {"pmevtyper25_el0", PMEVTYPER25_EL0}, - {"pmevtyper26_el0", PMEVTYPER26_EL0}, - {"pmevtyper27_el0", PMEVTYPER27_EL0}, - {"pmevtyper28_el0", PMEVTYPER28_EL0}, - {"pmevtyper29_el0", PMEVTYPER29_EL0}, - {"pmevtyper30_el0", PMEVTYPER30_EL0}, + {"osdtrrx_el1", OSDTRRX_EL1, 0}, + {"osdtrtx_el1", OSDTRTX_EL1, 0}, + {"teecr32_el1", TEECR32_EL1, 0}, + {"mdccint_el1", MDCCINT_EL1, 0}, + {"mdscr_el1", MDSCR_EL1, 0}, + {"dbgdtr_el0", DBGDTR_EL0, 0}, + {"oseccr_el1", OSECCR_EL1, 0}, + {"dbgvcr32_el2", DBGVCR32_EL2, 0}, + {"dbgbvr0_el1", DBGBVR0_EL1, 0}, + {"dbgbvr1_el1", DBGBVR1_EL1, 0}, + {"dbgbvr2_el1", DBGBVR2_EL1, 0}, + {"dbgbvr3_el1", DBGBVR3_EL1, 0}, + {"dbgbvr4_el1", DBGBVR4_EL1, 0}, + {"dbgbvr5_el1", DBGBVR5_EL1, 0}, + {"dbgbvr6_el1", DBGBVR6_EL1, 0}, + {"dbgbvr7_el1", DBGBVR7_EL1, 0}, + {"dbgbvr8_el1", DBGBVR8_EL1, 0}, + {"dbgbvr9_el1", DBGBVR9_EL1, 0}, + {"dbgbvr10_el1", DBGBVR10_EL1, 0}, + {"dbgbvr11_el1", DBGBVR11_EL1, 0}, + {"dbgbvr12_el1", DBGBVR12_EL1, 0}, + {"dbgbvr13_el1", DBGBVR13_EL1, 0}, + {"dbgbvr14_el1", DBGBVR14_EL1, 0}, + {"dbgbvr15_el1", DBGBVR15_EL1, 0}, + {"dbgbcr0_el1", DBGBCR0_EL1, 0}, + {"dbgbcr1_el1", DBGBCR1_EL1, 0}, + {"dbgbcr2_el1", DBGBCR2_EL1, 0}, + {"dbgbcr3_el1", DBGBCR3_EL1, 0}, + {"dbgbcr4_el1", DBGBCR4_EL1, 0}, + {"dbgbcr5_el1", DBGBCR5_EL1, 0}, + {"dbgbcr6_el1", DBGBCR6_EL1, 0}, + {"dbgbcr7_el1", DBGBCR7_EL1, 0}, + {"dbgbcr8_el1", DBGBCR8_EL1, 0}, + {"dbgbcr9_el1", DBGBCR9_EL1, 0}, + {"dbgbcr10_el1", DBGBCR10_EL1, 0}, + {"dbgbcr11_el1", DBGBCR11_EL1, 0}, + {"dbgbcr12_el1", DBGBCR12_EL1, 0}, + {"dbgbcr13_el1", DBGBCR13_EL1, 0}, + {"dbgbcr14_el1", DBGBCR14_EL1, 0}, + {"dbgbcr15_el1", DBGBCR15_EL1, 0}, + {"dbgwvr0_el1", DBGWVR0_EL1, 0}, + {"dbgwvr1_el1", DBGWVR1_EL1, 0}, + {"dbgwvr2_el1", DBGWVR2_EL1, 0}, + {"dbgwvr3_el1", DBGWVR3_EL1, 0}, + {"dbgwvr4_el1", DBGWVR4_EL1, 0}, + {"dbgwvr5_el1", DBGWVR5_EL1, 0}, + {"dbgwvr6_el1", DBGWVR6_EL1, 0}, + {"dbgwvr7_el1", DBGWVR7_EL1, 0}, + {"dbgwvr8_el1", DBGWVR8_EL1, 0}, + {"dbgwvr9_el1", DBGWVR9_EL1, 0}, + {"dbgwvr10_el1", DBGWVR10_EL1, 0}, + {"dbgwvr11_el1", DBGWVR11_EL1, 0}, + {"dbgwvr12_el1", DBGWVR12_EL1, 0}, + {"dbgwvr13_el1", DBGWVR13_EL1, 0}, + {"dbgwvr14_el1", DBGWVR14_EL1, 0}, + {"dbgwvr15_el1", DBGWVR15_EL1, 0}, + {"dbgwcr0_el1", DBGWCR0_EL1, 0}, + {"dbgwcr1_el1", DBGWCR1_EL1, 0}, + {"dbgwcr2_el1", DBGWCR2_EL1, 0}, + {"dbgwcr3_el1", DBGWCR3_EL1, 0}, + {"dbgwcr4_el1", DBGWCR4_EL1, 0}, + {"dbgwcr5_el1", DBGWCR5_EL1, 0}, + {"dbgwcr6_el1", DBGWCR6_EL1, 0}, + {"dbgwcr7_el1", DBGWCR7_EL1, 0}, + {"dbgwcr8_el1", DBGWCR8_EL1, 0}, + {"dbgwcr9_el1", DBGWCR9_EL1, 0}, + {"dbgwcr10_el1", DBGWCR10_EL1, 0}, + {"dbgwcr11_el1", DBGWCR11_EL1, 0}, + {"dbgwcr12_el1", DBGWCR12_EL1, 0}, + {"dbgwcr13_el1", DBGWCR13_EL1, 0}, + {"dbgwcr14_el1", DBGWCR14_EL1, 0}, + {"dbgwcr15_el1", DBGWCR15_EL1, 0}, + {"teehbr32_el1", TEEHBR32_EL1, 0}, + {"osdlr_el1", OSDLR_EL1, 0}, + {"dbgprcr_el1", DBGPRCR_EL1, 0}, + {"dbgclaimset_el1", DBGCLAIMSET_EL1, 0}, + {"dbgclaimclr_el1", DBGCLAIMCLR_EL1, 0}, + {"csselr_el1", CSSELR_EL1, 0}, + {"vpidr_el2", VPIDR_EL2, 0}, + {"vmpidr_el2", VMPIDR_EL2, 0}, + {"sctlr_el1", SCTLR_EL1, 0}, + {"sctlr_el2", SCTLR_EL2, 0}, + {"sctlr_el3", SCTLR_EL3, 0}, + {"actlr_el1", ACTLR_EL1, 0}, + {"actlr_el2", ACTLR_EL2, 0}, + {"actlr_el3", ACTLR_EL3, 0}, + {"cpacr_el1", CPACR_EL1, 0}, + {"hcr_el2", HCR_EL2, 0}, + {"scr_el3", SCR_EL3, 0}, + {"mdcr_el2", MDCR_EL2, 0}, + {"sder32_el3", SDER32_EL3, 0}, + {"cptr_el2", CPTR_EL2, 0}, + {"cptr_el3", CPTR_EL3, 0}, + {"hstr_el2", HSTR_EL2, 0}, + {"hacr_el2", HACR_EL2, 0}, + {"mdcr_el3", MDCR_EL3, 0}, + {"ttbr0_el1", TTBR0_EL1, 0}, + {"ttbr0_el2", TTBR0_EL2, 0}, + {"ttbr0_el3", TTBR0_EL3, 0}, + {"ttbr1_el1", TTBR1_EL1, 0}, + {"tcr_el1", TCR_EL1, 0}, + {"tcr_el2", TCR_EL2, 0}, + {"tcr_el3", TCR_EL3, 0}, + {"vttbr_el2", VTTBR_EL2, 0}, + {"vtcr_el2", VTCR_EL2, 0}, + {"dacr32_el2", DACR32_EL2, 0}, + {"spsr_el1", SPSR_EL1, 0}, + {"spsr_el2", SPSR_EL2, 0}, + {"spsr_el3", SPSR_EL3, 0}, + {"elr_el1", ELR_EL1, 0}, + {"elr_el2", ELR_EL2, 0}, + {"elr_el3", ELR_EL3, 0}, + {"sp_el0", SP_EL0, 0}, + {"sp_el1", SP_EL1, 0}, + {"sp_el2", SP_EL2, 0}, + {"spsel", SPSel, 0}, + {"nzcv", NZCV, 0}, + {"daif", DAIF, 0}, + {"currentel", CurrentEL, 0}, + {"spsr_irq", SPSR_irq, 0}, + {"spsr_abt", SPSR_abt, 0}, + {"spsr_und", SPSR_und, 0}, + {"spsr_fiq", SPSR_fiq, 0}, + {"fpcr", FPCR, 0}, + {"fpsr", FPSR, 0}, + {"dspsr_el0", DSPSR_EL0, 0}, + {"dlr_el0", DLR_EL0, 0}, + {"ifsr32_el2", IFSR32_EL2, 0}, + {"afsr0_el1", AFSR0_EL1, 0}, + {"afsr0_el2", AFSR0_EL2, 0}, + {"afsr0_el3", AFSR0_EL3, 0}, + {"afsr1_el1", AFSR1_EL1, 0}, + {"afsr1_el2", AFSR1_EL2, 0}, + {"afsr1_el3", AFSR1_EL3, 0}, + {"esr_el1", ESR_EL1, 0}, + {"esr_el2", ESR_EL2, 0}, + {"esr_el3", ESR_EL3, 0}, + {"fpexc32_el2", FPEXC32_EL2, 0}, + {"far_el1", FAR_EL1, 0}, + {"far_el2", FAR_EL2, 0}, + {"far_el3", FAR_EL3, 0}, + {"hpfar_el2", HPFAR_EL2, 0}, + {"par_el1", PAR_EL1, 0}, + {"pmcr_el0", PMCR_EL0, 0}, + {"pmcntenset_el0", PMCNTENSET_EL0, 0}, + {"pmcntenclr_el0", PMCNTENCLR_EL0, 0}, + {"pmovsclr_el0", PMOVSCLR_EL0, 0}, + {"pmselr_el0", PMSELR_EL0, 0}, + {"pmccntr_el0", PMCCNTR_EL0, 0}, + {"pmxevtyper_el0", PMXEVTYPER_EL0, 0}, + {"pmxevcntr_el0", PMXEVCNTR_EL0, 0}, + {"pmuserenr_el0", PMUSERENR_EL0, 0}, + {"pmintenset_el1", PMINTENSET_EL1, 0}, + {"pmintenclr_el1", PMINTENCLR_EL1, 0}, + {"pmovsset_el0", PMOVSSET_EL0, 0}, + {"mair_el1", MAIR_EL1, 0}, + {"mair_el2", MAIR_EL2, 0}, + {"mair_el3", MAIR_EL3, 0}, + {"amair_el1", AMAIR_EL1, 0}, + {"amair_el2", AMAIR_EL2, 0}, + {"amair_el3", AMAIR_EL3, 0}, + {"vbar_el1", VBAR_EL1, 0}, + {"vbar_el2", VBAR_EL2, 0}, + {"vbar_el3", VBAR_EL3, 0}, + {"rmr_el1", RMR_EL1, 0}, + {"rmr_el2", RMR_EL2, 0}, + {"rmr_el3", RMR_EL3, 0}, + {"contextidr_el1", CONTEXTIDR_EL1, 0}, + {"tpidr_el0", TPIDR_EL0, 0}, + {"tpidr_el2", TPIDR_EL2, 0}, + {"tpidr_el3", TPIDR_EL3, 0}, + {"tpidrro_el0", TPIDRRO_EL0, 0}, + {"tpidr_el1", TPIDR_EL1, 0}, + {"cntfrq_el0", CNTFRQ_EL0, 0}, + {"cntvoff_el2", CNTVOFF_EL2, 0}, + {"cntkctl_el1", CNTKCTL_EL1, 0}, + {"cnthctl_el2", CNTHCTL_EL2, 0}, + {"cntp_tval_el0", CNTP_TVAL_EL0, 0}, + {"cnthp_tval_el2", CNTHP_TVAL_EL2, 0}, + {"cntps_tval_el1", CNTPS_TVAL_EL1, 0}, + {"cntp_ctl_el0", CNTP_CTL_EL0, 0}, + {"cnthp_ctl_el2", CNTHP_CTL_EL2, 0}, + {"cntps_ctl_el1", CNTPS_CTL_EL1, 0}, + {"cntp_cval_el0", CNTP_CVAL_EL0, 0}, + {"cnthp_cval_el2", CNTHP_CVAL_EL2, 0}, + {"cntps_cval_el1", CNTPS_CVAL_EL1, 0}, + {"cntv_tval_el0", CNTV_TVAL_EL0, 0}, + {"cntv_ctl_el0", CNTV_CTL_EL0, 0}, + {"cntv_cval_el0", CNTV_CVAL_EL0, 0}, + {"pmevcntr0_el0", PMEVCNTR0_EL0, 0}, + {"pmevcntr1_el0", PMEVCNTR1_EL0, 0}, + {"pmevcntr2_el0", PMEVCNTR2_EL0, 0}, + {"pmevcntr3_el0", PMEVCNTR3_EL0, 0}, + {"pmevcntr4_el0", PMEVCNTR4_EL0, 0}, + {"pmevcntr5_el0", PMEVCNTR5_EL0, 0}, + {"pmevcntr6_el0", PMEVCNTR6_EL0, 0}, + {"pmevcntr7_el0", PMEVCNTR7_EL0, 0}, + {"pmevcntr8_el0", PMEVCNTR8_EL0, 0}, + {"pmevcntr9_el0", PMEVCNTR9_EL0, 0}, + {"pmevcntr10_el0", PMEVCNTR10_EL0, 0}, + {"pmevcntr11_el0", PMEVCNTR11_EL0, 0}, + {"pmevcntr12_el0", PMEVCNTR12_EL0, 0}, + {"pmevcntr13_el0", PMEVCNTR13_EL0, 0}, + {"pmevcntr14_el0", PMEVCNTR14_EL0, 0}, + {"pmevcntr15_el0", PMEVCNTR15_EL0, 0}, + {"pmevcntr16_el0", PMEVCNTR16_EL0, 0}, + {"pmevcntr17_el0", PMEVCNTR17_EL0, 0}, + {"pmevcntr18_el0", PMEVCNTR18_EL0, 0}, + {"pmevcntr19_el0", PMEVCNTR19_EL0, 0}, + {"pmevcntr20_el0", PMEVCNTR20_EL0, 0}, + {"pmevcntr21_el0", PMEVCNTR21_EL0, 0}, + {"pmevcntr22_el0", PMEVCNTR22_EL0, 0}, + {"pmevcntr23_el0", PMEVCNTR23_EL0, 0}, + {"pmevcntr24_el0", PMEVCNTR24_EL0, 0}, + {"pmevcntr25_el0", PMEVCNTR25_EL0, 0}, + {"pmevcntr26_el0", PMEVCNTR26_EL0, 0}, + {"pmevcntr27_el0", PMEVCNTR27_EL0, 0}, + {"pmevcntr28_el0", PMEVCNTR28_EL0, 0}, + {"pmevcntr29_el0", PMEVCNTR29_EL0, 0}, + {"pmevcntr30_el0", PMEVCNTR30_EL0, 0}, + {"pmccfiltr_el0", PMCCFILTR_EL0, 0}, + {"pmevtyper0_el0", PMEVTYPER0_EL0, 0}, + {"pmevtyper1_el0", PMEVTYPER1_EL0, 0}, + {"pmevtyper2_el0", PMEVTYPER2_EL0, 0}, + {"pmevtyper3_el0", PMEVTYPER3_EL0, 0}, + {"pmevtyper4_el0", PMEVTYPER4_EL0, 0}, + {"pmevtyper5_el0", PMEVTYPER5_EL0, 0}, + {"pmevtyper6_el0", PMEVTYPER6_EL0, 0}, + {"pmevtyper7_el0", PMEVTYPER7_EL0, 0}, + {"pmevtyper8_el0", PMEVTYPER8_EL0, 0}, + {"pmevtyper9_el0", PMEVTYPER9_EL0, 0}, + {"pmevtyper10_el0", PMEVTYPER10_EL0, 0}, + {"pmevtyper11_el0", PMEVTYPER11_EL0, 0}, + {"pmevtyper12_el0", PMEVTYPER12_EL0, 0}, + {"pmevtyper13_el0", PMEVTYPER13_EL0, 0}, + {"pmevtyper14_el0", PMEVTYPER14_EL0, 0}, + {"pmevtyper15_el0", PMEVTYPER15_EL0, 0}, + {"pmevtyper16_el0", PMEVTYPER16_EL0, 0}, + {"pmevtyper17_el0", PMEVTYPER17_EL0, 0}, + {"pmevtyper18_el0", PMEVTYPER18_EL0, 0}, + {"pmevtyper19_el0", PMEVTYPER19_EL0, 0}, + {"pmevtyper20_el0", PMEVTYPER20_EL0, 0}, + {"pmevtyper21_el0", PMEVTYPER21_EL0, 0}, + {"pmevtyper22_el0", PMEVTYPER22_EL0, 0}, + {"pmevtyper23_el0", PMEVTYPER23_EL0, 0}, + {"pmevtyper24_el0", PMEVTYPER24_EL0, 0}, + {"pmevtyper25_el0", PMEVTYPER25_EL0, 0}, + {"pmevtyper26_el0", PMEVTYPER26_EL0, 0}, + {"pmevtyper27_el0", PMEVTYPER27_EL0, 0}, + {"pmevtyper28_el0", PMEVTYPER28_EL0, 0}, + {"pmevtyper29_el0", PMEVTYPER29_EL0, 0}, + {"pmevtyper30_el0", PMEVTYPER30_EL0, 0}, // Trace registers - {"trcprgctlr", TRCPRGCTLR}, - {"trcprocselr", TRCPROCSELR}, - {"trcconfigr", TRCCONFIGR}, - {"trcauxctlr", TRCAUXCTLR}, - {"trceventctl0r", TRCEVENTCTL0R}, - {"trceventctl1r", TRCEVENTCTL1R}, - {"trcstallctlr", TRCSTALLCTLR}, - {"trctsctlr", TRCTSCTLR}, - {"trcsyncpr", TRCSYNCPR}, - {"trcccctlr", TRCCCCTLR}, - {"trcbbctlr", TRCBBCTLR}, - {"trctraceidr", TRCTRACEIDR}, - {"trcqctlr", TRCQCTLR}, - {"trcvictlr", TRCVICTLR}, - {"trcviiectlr", TRCVIIECTLR}, - {"trcvissctlr", TRCVISSCTLR}, - {"trcvipcssctlr", TRCVIPCSSCTLR}, - {"trcvdctlr", TRCVDCTLR}, - {"trcvdsacctlr", TRCVDSACCTLR}, - {"trcvdarcctlr", TRCVDARCCTLR}, - {"trcseqevr0", TRCSEQEVR0}, - {"trcseqevr1", TRCSEQEVR1}, - {"trcseqevr2", TRCSEQEVR2}, - {"trcseqrstevr", TRCSEQRSTEVR}, - {"trcseqstr", TRCSEQSTR}, - {"trcextinselr", TRCEXTINSELR}, - {"trccntrldvr0", TRCCNTRLDVR0}, - {"trccntrldvr1", TRCCNTRLDVR1}, - {"trccntrldvr2", TRCCNTRLDVR2}, - {"trccntrldvr3", TRCCNTRLDVR3}, - {"trccntctlr0", TRCCNTCTLR0}, - {"trccntctlr1", TRCCNTCTLR1}, - {"trccntctlr2", TRCCNTCTLR2}, - {"trccntctlr3", TRCCNTCTLR3}, - {"trccntvr0", TRCCNTVR0}, - {"trccntvr1", TRCCNTVR1}, - {"trccntvr2", TRCCNTVR2}, - {"trccntvr3", TRCCNTVR3}, - {"trcimspec0", TRCIMSPEC0}, - {"trcimspec1", TRCIMSPEC1}, - {"trcimspec2", TRCIMSPEC2}, - {"trcimspec3", TRCIMSPEC3}, - {"trcimspec4", TRCIMSPEC4}, - {"trcimspec5", TRCIMSPEC5}, - {"trcimspec6", TRCIMSPEC6}, - {"trcimspec7", TRCIMSPEC7}, - {"trcrsctlr2", TRCRSCTLR2}, - {"trcrsctlr3", TRCRSCTLR3}, - {"trcrsctlr4", TRCRSCTLR4}, - {"trcrsctlr5", TRCRSCTLR5}, - {"trcrsctlr6", TRCRSCTLR6}, - {"trcrsctlr7", TRCRSCTLR7}, - {"trcrsctlr8", TRCRSCTLR8}, - {"trcrsctlr9", TRCRSCTLR9}, - {"trcrsctlr10", TRCRSCTLR10}, - {"trcrsctlr11", TRCRSCTLR11}, - {"trcrsctlr12", TRCRSCTLR12}, - {"trcrsctlr13", TRCRSCTLR13}, - {"trcrsctlr14", TRCRSCTLR14}, - {"trcrsctlr15", TRCRSCTLR15}, - {"trcrsctlr16", TRCRSCTLR16}, - {"trcrsctlr17", TRCRSCTLR17}, - {"trcrsctlr18", TRCRSCTLR18}, - {"trcrsctlr19", TRCRSCTLR19}, - {"trcrsctlr20", TRCRSCTLR20}, - {"trcrsctlr21", TRCRSCTLR21}, - {"trcrsctlr22", TRCRSCTLR22}, - {"trcrsctlr23", TRCRSCTLR23}, - {"trcrsctlr24", TRCRSCTLR24}, - {"trcrsctlr25", TRCRSCTLR25}, - {"trcrsctlr26", TRCRSCTLR26}, - {"trcrsctlr27", TRCRSCTLR27}, - {"trcrsctlr28", TRCRSCTLR28}, - {"trcrsctlr29", TRCRSCTLR29}, - {"trcrsctlr30", TRCRSCTLR30}, - {"trcrsctlr31", TRCRSCTLR31}, - {"trcssccr0", TRCSSCCR0}, - {"trcssccr1", TRCSSCCR1}, - {"trcssccr2", TRCSSCCR2}, - {"trcssccr3", TRCSSCCR3}, - {"trcssccr4", TRCSSCCR4}, - {"trcssccr5", TRCSSCCR5}, - {"trcssccr6", TRCSSCCR6}, - {"trcssccr7", TRCSSCCR7}, - {"trcsscsr0", TRCSSCSR0}, - {"trcsscsr1", TRCSSCSR1}, - {"trcsscsr2", TRCSSCSR2}, - {"trcsscsr3", TRCSSCSR3}, - {"trcsscsr4", TRCSSCSR4}, - {"trcsscsr5", TRCSSCSR5}, - {"trcsscsr6", TRCSSCSR6}, - {"trcsscsr7", TRCSSCSR7}, - {"trcsspcicr0", TRCSSPCICR0}, - {"trcsspcicr1", TRCSSPCICR1}, - {"trcsspcicr2", TRCSSPCICR2}, - {"trcsspcicr3", TRCSSPCICR3}, - {"trcsspcicr4", TRCSSPCICR4}, - {"trcsspcicr5", TRCSSPCICR5}, - {"trcsspcicr6", TRCSSPCICR6}, - {"trcsspcicr7", TRCSSPCICR7}, - {"trcpdcr", TRCPDCR}, - {"trcacvr0", TRCACVR0}, - {"trcacvr1", TRCACVR1}, - {"trcacvr2", TRCACVR2}, - {"trcacvr3", TRCACVR3}, - {"trcacvr4", TRCACVR4}, - {"trcacvr5", TRCACVR5}, - {"trcacvr6", TRCACVR6}, - {"trcacvr7", TRCACVR7}, - {"trcacvr8", TRCACVR8}, - {"trcacvr9", TRCACVR9}, - {"trcacvr10", TRCACVR10}, - {"trcacvr11", TRCACVR11}, - {"trcacvr12", TRCACVR12}, - {"trcacvr13", TRCACVR13}, - {"trcacvr14", TRCACVR14}, - {"trcacvr15", TRCACVR15}, - {"trcacatr0", TRCACATR0}, - {"trcacatr1", TRCACATR1}, - {"trcacatr2", TRCACATR2}, - {"trcacatr3", TRCACATR3}, - {"trcacatr4", TRCACATR4}, - {"trcacatr5", TRCACATR5}, - {"trcacatr6", TRCACATR6}, - {"trcacatr7", TRCACATR7}, - {"trcacatr8", TRCACATR8}, - {"trcacatr9", TRCACATR9}, - {"trcacatr10", TRCACATR10}, - {"trcacatr11", TRCACATR11}, - {"trcacatr12", TRCACATR12}, - {"trcacatr13", TRCACATR13}, - {"trcacatr14", TRCACATR14}, - {"trcacatr15", TRCACATR15}, - {"trcdvcvr0", TRCDVCVR0}, - {"trcdvcvr1", TRCDVCVR1}, - {"trcdvcvr2", TRCDVCVR2}, - {"trcdvcvr3", TRCDVCVR3}, - {"trcdvcvr4", TRCDVCVR4}, - {"trcdvcvr5", TRCDVCVR5}, - {"trcdvcvr6", TRCDVCVR6}, - {"trcdvcvr7", TRCDVCVR7}, - {"trcdvcmr0", TRCDVCMR0}, - {"trcdvcmr1", TRCDVCMR1}, - {"trcdvcmr2", TRCDVCMR2}, - {"trcdvcmr3", TRCDVCMR3}, - {"trcdvcmr4", TRCDVCMR4}, - {"trcdvcmr5", TRCDVCMR5}, - {"trcdvcmr6", TRCDVCMR6}, - {"trcdvcmr7", TRCDVCMR7}, - {"trccidcvr0", TRCCIDCVR0}, - {"trccidcvr1", TRCCIDCVR1}, - {"trccidcvr2", TRCCIDCVR2}, - {"trccidcvr3", TRCCIDCVR3}, - {"trccidcvr4", TRCCIDCVR4}, - {"trccidcvr5", TRCCIDCVR5}, - {"trccidcvr6", TRCCIDCVR6}, - {"trccidcvr7", TRCCIDCVR7}, - {"trcvmidcvr0", TRCVMIDCVR0}, - {"trcvmidcvr1", TRCVMIDCVR1}, - {"trcvmidcvr2", TRCVMIDCVR2}, - {"trcvmidcvr3", TRCVMIDCVR3}, - {"trcvmidcvr4", TRCVMIDCVR4}, - {"trcvmidcvr5", TRCVMIDCVR5}, - {"trcvmidcvr6", TRCVMIDCVR6}, - {"trcvmidcvr7", TRCVMIDCVR7}, - {"trccidcctlr0", TRCCIDCCTLR0}, - {"trccidcctlr1", TRCCIDCCTLR1}, - {"trcvmidcctlr0", TRCVMIDCCTLR0}, - {"trcvmidcctlr1", TRCVMIDCCTLR1}, - {"trcitctrl", TRCITCTRL}, - {"trcclaimset", TRCCLAIMSET}, - {"trcclaimclr", TRCCLAIMCLR}, + {"trcprgctlr", TRCPRGCTLR, 0}, + {"trcprocselr", TRCPROCSELR, 0}, + {"trcconfigr", TRCCONFIGR, 0}, + {"trcauxctlr", TRCAUXCTLR, 0}, + {"trceventctl0r", TRCEVENTCTL0R, 0}, + {"trceventctl1r", TRCEVENTCTL1R, 0}, + {"trcstallctlr", TRCSTALLCTLR, 0}, + {"trctsctlr", TRCTSCTLR, 0}, + {"trcsyncpr", TRCSYNCPR, 0}, + {"trcccctlr", TRCCCCTLR, 0}, + {"trcbbctlr", TRCBBCTLR, 0}, + {"trctraceidr", TRCTRACEIDR, 0}, + {"trcqctlr", TRCQCTLR, 0}, + {"trcvictlr", TRCVICTLR, 0}, + {"trcviiectlr", TRCVIIECTLR, 0}, + {"trcvissctlr", TRCVISSCTLR, 0}, + {"trcvipcssctlr", TRCVIPCSSCTLR, 0}, + {"trcvdctlr", TRCVDCTLR, 0}, + {"trcvdsacctlr", TRCVDSACCTLR, 0}, + {"trcvdarcctlr", TRCVDARCCTLR, 0}, + {"trcseqevr0", TRCSEQEVR0, 0}, + {"trcseqevr1", TRCSEQEVR1, 0}, + {"trcseqevr2", TRCSEQEVR2, 0}, + {"trcseqrstevr", TRCSEQRSTEVR, 0}, + {"trcseqstr", TRCSEQSTR, 0}, + {"trcextinselr", TRCEXTINSELR, 0}, + {"trccntrldvr0", TRCCNTRLDVR0, 0}, + {"trccntrldvr1", TRCCNTRLDVR1, 0}, + {"trccntrldvr2", TRCCNTRLDVR2, 0}, + {"trccntrldvr3", TRCCNTRLDVR3, 0}, + {"trccntctlr0", TRCCNTCTLR0, 0}, + {"trccntctlr1", TRCCNTCTLR1, 0}, + {"trccntctlr2", TRCCNTCTLR2, 0}, + {"trccntctlr3", TRCCNTCTLR3, 0}, + {"trccntvr0", TRCCNTVR0, 0}, + {"trccntvr1", TRCCNTVR1, 0}, + {"trccntvr2", TRCCNTVR2, 0}, + {"trccntvr3", TRCCNTVR3, 0}, + {"trcimspec0", TRCIMSPEC0, 0}, + {"trcimspec1", TRCIMSPEC1, 0}, + {"trcimspec2", TRCIMSPEC2, 0}, + {"trcimspec3", TRCIMSPEC3, 0}, + {"trcimspec4", TRCIMSPEC4, 0}, + {"trcimspec5", TRCIMSPEC5, 0}, + {"trcimspec6", TRCIMSPEC6, 0}, + {"trcimspec7", TRCIMSPEC7, 0}, + {"trcrsctlr2", TRCRSCTLR2, 0}, + {"trcrsctlr3", TRCRSCTLR3, 0}, + {"trcrsctlr4", TRCRSCTLR4, 0}, + {"trcrsctlr5", TRCRSCTLR5, 0}, + {"trcrsctlr6", TRCRSCTLR6, 0}, + {"trcrsctlr7", TRCRSCTLR7, 0}, + {"trcrsctlr8", TRCRSCTLR8, 0}, + {"trcrsctlr9", TRCRSCTLR9, 0}, + {"trcrsctlr10", TRCRSCTLR10, 0}, + {"trcrsctlr11", TRCRSCTLR11, 0}, + {"trcrsctlr12", TRCRSCTLR12, 0}, + {"trcrsctlr13", TRCRSCTLR13, 0}, + {"trcrsctlr14", TRCRSCTLR14, 0}, + {"trcrsctlr15", TRCRSCTLR15, 0}, + {"trcrsctlr16", TRCRSCTLR16, 0}, + {"trcrsctlr17", TRCRSCTLR17, 0}, + {"trcrsctlr18", TRCRSCTLR18, 0}, + {"trcrsctlr19", TRCRSCTLR19, 0}, + {"trcrsctlr20", TRCRSCTLR20, 0}, + {"trcrsctlr21", TRCRSCTLR21, 0}, + {"trcrsctlr22", TRCRSCTLR22, 0}, + {"trcrsctlr23", TRCRSCTLR23, 0}, + {"trcrsctlr24", TRCRSCTLR24, 0}, + {"trcrsctlr25", TRCRSCTLR25, 0}, + {"trcrsctlr26", TRCRSCTLR26, 0}, + {"trcrsctlr27", TRCRSCTLR27, 0}, + {"trcrsctlr28", TRCRSCTLR28, 0}, + {"trcrsctlr29", TRCRSCTLR29, 0}, + {"trcrsctlr30", TRCRSCTLR30, 0}, + {"trcrsctlr31", TRCRSCTLR31, 0}, + {"trcssccr0", TRCSSCCR0, 0}, + {"trcssccr1", TRCSSCCR1, 0}, + {"trcssccr2", TRCSSCCR2, 0}, + {"trcssccr3", TRCSSCCR3, 0}, + {"trcssccr4", TRCSSCCR4, 0}, + {"trcssccr5", TRCSSCCR5, 0}, + {"trcssccr6", TRCSSCCR6, 0}, + {"trcssccr7", TRCSSCCR7, 0}, + {"trcsscsr0", TRCSSCSR0, 0}, + {"trcsscsr1", TRCSSCSR1, 0}, + {"trcsscsr2", TRCSSCSR2, 0}, + {"trcsscsr3", TRCSSCSR3, 0}, + {"trcsscsr4", TRCSSCSR4, 0}, + {"trcsscsr5", TRCSSCSR5, 0}, + {"trcsscsr6", TRCSSCSR6, 0}, + {"trcsscsr7", TRCSSCSR7, 0}, + {"trcsspcicr0", TRCSSPCICR0, 0}, + {"trcsspcicr1", TRCSSPCICR1, 0}, + {"trcsspcicr2", TRCSSPCICR2, 0}, + {"trcsspcicr3", TRCSSPCICR3, 0}, + {"trcsspcicr4", TRCSSPCICR4, 0}, + {"trcsspcicr5", TRCSSPCICR5, 0}, + {"trcsspcicr6", TRCSSPCICR6, 0}, + {"trcsspcicr7", TRCSSPCICR7, 0}, + {"trcpdcr", TRCPDCR, 0}, + {"trcacvr0", TRCACVR0, 0}, + {"trcacvr1", TRCACVR1, 0}, + {"trcacvr2", TRCACVR2, 0}, + {"trcacvr3", TRCACVR3, 0}, + {"trcacvr4", TRCACVR4, 0}, + {"trcacvr5", TRCACVR5, 0}, + {"trcacvr6", TRCACVR6, 0}, + {"trcacvr7", TRCACVR7, 0}, + {"trcacvr8", TRCACVR8, 0}, + {"trcacvr9", TRCACVR9, 0}, + {"trcacvr10", TRCACVR10, 0}, + {"trcacvr11", TRCACVR11, 0}, + {"trcacvr12", TRCACVR12, 0}, + {"trcacvr13", TRCACVR13, 0}, + {"trcacvr14", TRCACVR14, 0}, + {"trcacvr15", TRCACVR15, 0}, + {"trcacatr0", TRCACATR0, 0}, + {"trcacatr1", TRCACATR1, 0}, + {"trcacatr2", TRCACATR2, 0}, + {"trcacatr3", TRCACATR3, 0}, + {"trcacatr4", TRCACATR4, 0}, + {"trcacatr5", TRCACATR5, 0}, + {"trcacatr6", TRCACATR6, 0}, + {"trcacatr7", TRCACATR7, 0}, + {"trcacatr8", TRCACATR8, 0}, + {"trcacatr9", TRCACATR9, 0}, + {"trcacatr10", TRCACATR10, 0}, + {"trcacatr11", TRCACATR11, 0}, + {"trcacatr12", TRCACATR12, 0}, + {"trcacatr13", TRCACATR13, 0}, + {"trcacatr14", TRCACATR14, 0}, + {"trcacatr15", TRCACATR15, 0}, + {"trcdvcvr0", TRCDVCVR0, 0}, + {"trcdvcvr1", TRCDVCVR1, 0}, + {"trcdvcvr2", TRCDVCVR2, 0}, + {"trcdvcvr3", TRCDVCVR3, 0}, + {"trcdvcvr4", TRCDVCVR4, 0}, + {"trcdvcvr5", TRCDVCVR5, 0}, + {"trcdvcvr6", TRCDVCVR6, 0}, + {"trcdvcvr7", TRCDVCVR7, 0}, + {"trcdvcmr0", TRCDVCMR0, 0}, + {"trcdvcmr1", TRCDVCMR1, 0}, + {"trcdvcmr2", TRCDVCMR2, 0}, + {"trcdvcmr3", TRCDVCMR3, 0}, + {"trcdvcmr4", TRCDVCMR4, 0}, + {"trcdvcmr5", TRCDVCMR5, 0}, + {"trcdvcmr6", TRCDVCMR6, 0}, + {"trcdvcmr7", TRCDVCMR7, 0}, + {"trccidcvr0", TRCCIDCVR0, 0}, + {"trccidcvr1", TRCCIDCVR1, 0}, + {"trccidcvr2", TRCCIDCVR2, 0}, + {"trccidcvr3", TRCCIDCVR3, 0}, + {"trccidcvr4", TRCCIDCVR4, 0}, + {"trccidcvr5", TRCCIDCVR5, 0}, + {"trccidcvr6", TRCCIDCVR6, 0}, + {"trccidcvr7", TRCCIDCVR7, 0}, + {"trcvmidcvr0", TRCVMIDCVR0, 0}, + {"trcvmidcvr1", TRCVMIDCVR1, 0}, + {"trcvmidcvr2", TRCVMIDCVR2, 0}, + {"trcvmidcvr3", TRCVMIDCVR3, 0}, + {"trcvmidcvr4", TRCVMIDCVR4, 0}, + {"trcvmidcvr5", TRCVMIDCVR5, 0}, + {"trcvmidcvr6", TRCVMIDCVR6, 0}, + {"trcvmidcvr7", TRCVMIDCVR7, 0}, + {"trccidcctlr0", TRCCIDCCTLR0, 0}, + {"trccidcctlr1", TRCCIDCCTLR1, 0}, + {"trcvmidcctlr0", TRCVMIDCCTLR0, 0}, + {"trcvmidcctlr1", TRCVMIDCCTLR1, 0}, + {"trcitctrl", TRCITCTRL, 0}, + {"trcclaimset", TRCCLAIMSET, 0}, + {"trcclaimclr", TRCCLAIMCLR, 0}, // GICv3 registers - {"icc_bpr1_el1", ICC_BPR1_EL1}, - {"icc_bpr0_el1", ICC_BPR0_EL1}, - {"icc_pmr_el1", ICC_PMR_EL1}, - {"icc_ctlr_el1", ICC_CTLR_EL1}, - {"icc_ctlr_el3", ICC_CTLR_EL3}, - {"icc_sre_el1", ICC_SRE_EL1}, - {"icc_sre_el2", ICC_SRE_EL2}, - {"icc_sre_el3", ICC_SRE_EL3}, - {"icc_igrpen0_el1", ICC_IGRPEN0_EL1}, - {"icc_igrpen1_el1", ICC_IGRPEN1_EL1}, - {"icc_igrpen1_el3", ICC_IGRPEN1_EL3}, - {"icc_seien_el1", ICC_SEIEN_EL1}, - {"icc_ap0r0_el1", ICC_AP0R0_EL1}, - {"icc_ap0r1_el1", ICC_AP0R1_EL1}, - {"icc_ap0r2_el1", ICC_AP0R2_EL1}, - {"icc_ap0r3_el1", ICC_AP0R3_EL1}, - {"icc_ap1r0_el1", ICC_AP1R0_EL1}, - {"icc_ap1r1_el1", ICC_AP1R1_EL1}, - {"icc_ap1r2_el1", ICC_AP1R2_EL1}, - {"icc_ap1r3_el1", ICC_AP1R3_EL1}, - {"ich_ap0r0_el2", ICH_AP0R0_EL2}, - {"ich_ap0r1_el2", ICH_AP0R1_EL2}, - {"ich_ap0r2_el2", ICH_AP0R2_EL2}, - {"ich_ap0r3_el2", ICH_AP0R3_EL2}, - {"ich_ap1r0_el2", ICH_AP1R0_EL2}, - {"ich_ap1r1_el2", ICH_AP1R1_EL2}, - {"ich_ap1r2_el2", ICH_AP1R2_EL2}, - {"ich_ap1r3_el2", ICH_AP1R3_EL2}, - {"ich_hcr_el2", ICH_HCR_EL2}, - {"ich_misr_el2", ICH_MISR_EL2}, - {"ich_vmcr_el2", ICH_VMCR_EL2}, - {"ich_vseir_el2", ICH_VSEIR_EL2}, - {"ich_lr0_el2", ICH_LR0_EL2}, - {"ich_lr1_el2", ICH_LR1_EL2}, - {"ich_lr2_el2", ICH_LR2_EL2}, - {"ich_lr3_el2", ICH_LR3_EL2}, - {"ich_lr4_el2", ICH_LR4_EL2}, - {"ich_lr5_el2", ICH_LR5_EL2}, - {"ich_lr6_el2", ICH_LR6_EL2}, - {"ich_lr7_el2", ICH_LR7_EL2}, - {"ich_lr8_el2", ICH_LR8_EL2}, - {"ich_lr9_el2", ICH_LR9_EL2}, - {"ich_lr10_el2", ICH_LR10_EL2}, - {"ich_lr11_el2", ICH_LR11_EL2}, - {"ich_lr12_el2", ICH_LR12_EL2}, - {"ich_lr13_el2", ICH_LR13_EL2}, - {"ich_lr14_el2", ICH_LR14_EL2}, - {"ich_lr15_el2", ICH_LR15_EL2} -}; - -const AArch64NamedImmMapper::Mapping -AArch64SysReg::SysRegMapper::CycloneSysRegMappings[] = { - {"cpm_ioacc_ctl_el3", CPM_IOACC_CTL_EL3} + {"icc_bpr1_el1", ICC_BPR1_EL1, 0}, + {"icc_bpr0_el1", ICC_BPR0_EL1, 0}, + {"icc_pmr_el1", ICC_PMR_EL1, 0}, + {"icc_ctlr_el1", ICC_CTLR_EL1, 0}, + {"icc_ctlr_el3", ICC_CTLR_EL3, 0}, + {"icc_sre_el1", ICC_SRE_EL1, 0}, + {"icc_sre_el2", ICC_SRE_EL2, 0}, + {"icc_sre_el3", ICC_SRE_EL3, 0}, + {"icc_igrpen0_el1", ICC_IGRPEN0_EL1, 0}, + {"icc_igrpen1_el1", ICC_IGRPEN1_EL1, 0}, + {"icc_igrpen1_el3", ICC_IGRPEN1_EL3, 0}, + {"icc_seien_el1", ICC_SEIEN_EL1, 0}, + {"icc_ap0r0_el1", ICC_AP0R0_EL1, 0}, + {"icc_ap0r1_el1", ICC_AP0R1_EL1, 0}, + {"icc_ap0r2_el1", ICC_AP0R2_EL1, 0}, + {"icc_ap0r3_el1", ICC_AP0R3_EL1, 0}, + {"icc_ap1r0_el1", ICC_AP1R0_EL1, 0}, + {"icc_ap1r1_el1", ICC_AP1R1_EL1, 0}, + {"icc_ap1r2_el1", ICC_AP1R2_EL1, 0}, + {"icc_ap1r3_el1", ICC_AP1R3_EL1, 0}, + {"ich_ap0r0_el2", ICH_AP0R0_EL2, 0}, + {"ich_ap0r1_el2", ICH_AP0R1_EL2, 0}, + {"ich_ap0r2_el2", ICH_AP0R2_EL2, 0}, + {"ich_ap0r3_el2", ICH_AP0R3_EL2, 0}, + {"ich_ap1r0_el2", ICH_AP1R0_EL2, 0}, + {"ich_ap1r1_el2", ICH_AP1R1_EL2, 0}, + {"ich_ap1r2_el2", ICH_AP1R2_EL2, 0}, + {"ich_ap1r3_el2", ICH_AP1R3_EL2, 0}, + {"ich_hcr_el2", ICH_HCR_EL2, 0}, + {"ich_misr_el2", ICH_MISR_EL2, 0}, + {"ich_vmcr_el2", ICH_VMCR_EL2, 0}, + {"ich_vseir_el2", ICH_VSEIR_EL2, 0}, + {"ich_lr0_el2", ICH_LR0_EL2, 0}, + {"ich_lr1_el2", ICH_LR1_EL2, 0}, + {"ich_lr2_el2", ICH_LR2_EL2, 0}, + {"ich_lr3_el2", ICH_LR3_EL2, 0}, + {"ich_lr4_el2", ICH_LR4_EL2, 0}, + {"ich_lr5_el2", ICH_LR5_EL2, 0}, + {"ich_lr6_el2", ICH_LR6_EL2, 0}, + {"ich_lr7_el2", ICH_LR7_EL2, 0}, + {"ich_lr8_el2", ICH_LR8_EL2, 0}, + {"ich_lr9_el2", ICH_LR9_EL2, 0}, + {"ich_lr10_el2", ICH_LR10_EL2, 0}, + {"ich_lr11_el2", ICH_LR11_EL2, 0}, + {"ich_lr12_el2", ICH_LR12_EL2, 0}, + {"ich_lr13_el2", ICH_LR13_EL2, 0}, + {"ich_lr14_el2", ICH_LR14_EL2, 0}, + {"ich_lr15_el2", ICH_LR15_EL2, 0}, + + // Cyclone registers + {"cpm_ioacc_ctl_el3", CPM_IOACC_CTL_EL3, AArch64::ProcCyclone}, + + // v8.1a "Privileged Access Never" extension-specific system registers + {"pan", PAN, AArch64::HasV8_1aOps}, + + // v8.1a "Limited Ordering Regions" extension-specific system registers + {"lorsa_el1", LORSA_EL1, AArch64::HasV8_1aOps}, + {"lorea_el1", LOREA_EL1, AArch64::HasV8_1aOps}, + {"lorn_el1", LORN_EL1, AArch64::HasV8_1aOps}, + {"lorc_el1", LORC_EL1, AArch64::HasV8_1aOps}, + {"lorid_el1", LORID_EL1, AArch64::HasV8_1aOps}, + + // v8.1a "Virtualization host extensions" system registers + {"ttbr1_el2", TTBR1_EL2, AArch64::HasV8_1aOps}, + {"contextidr_el2", CONTEXTIDR_EL2, AArch64::HasV8_1aOps}, + {"cnthv_tval_el2", CNTHV_TVAL_EL2, AArch64::HasV8_1aOps}, + {"cnthv_cval_el2", CNTHV_CVAL_EL2, AArch64::HasV8_1aOps}, + {"cnthv_ctl_el2", CNTHV_CTL_EL2, AArch64::HasV8_1aOps}, + {"sctlr_el12", SCTLR_EL12, AArch64::HasV8_1aOps}, + {"cpacr_el12", CPACR_EL12, AArch64::HasV8_1aOps}, + {"ttbr0_el12", TTBR0_EL12, AArch64::HasV8_1aOps}, + {"ttbr1_el12", TTBR1_EL12, AArch64::HasV8_1aOps}, + {"tcr_el12", TCR_EL12, AArch64::HasV8_1aOps}, + {"afsr0_el12", AFSR0_EL12, AArch64::HasV8_1aOps}, + {"afsr1_el12", AFSR1_EL12, AArch64::HasV8_1aOps}, + {"esr_el12", ESR_EL12, AArch64::HasV8_1aOps}, + {"far_el12", FAR_EL12, AArch64::HasV8_1aOps}, + {"mair_el12", MAIR_EL12, AArch64::HasV8_1aOps}, + {"amair_el12", AMAIR_EL12, AArch64::HasV8_1aOps}, + {"vbar_el12", VBAR_EL12, AArch64::HasV8_1aOps}, + {"contextidr_el12", CONTEXTIDR_EL12, AArch64::HasV8_1aOps}, + {"cntkctl_el12", CNTKCTL_EL12, AArch64::HasV8_1aOps}, + {"cntp_tval_el02", CNTP_TVAL_EL02, AArch64::HasV8_1aOps}, + {"cntp_ctl_el02", CNTP_CTL_EL02, AArch64::HasV8_1aOps}, + {"cntp_cval_el02", CNTP_CVAL_EL02, AArch64::HasV8_1aOps}, + {"cntv_tval_el02", CNTV_TVAL_EL02, AArch64::HasV8_1aOps}, + {"cntv_ctl_el02", CNTV_CTL_EL02, AArch64::HasV8_1aOps}, + {"cntv_cval_el02", CNTV_CVAL_EL02, AArch64::HasV8_1aOps}, + {"spsr_el12", SPSR_EL12, AArch64::HasV8_1aOps}, + {"elr_el12", ELR_EL12, AArch64::HasV8_1aOps}, }; uint32_t -AArch64SysReg::SysRegMapper::fromString(StringRef Name, bool &Valid) const { +AArch64SysReg::SysRegMapper::fromString(StringRef Name, uint64_t FeatureBits, + bool &Valid) const { std::string NameLower = Name.lower(); // First search the registers shared by all for (unsigned i = 0; i < array_lengthof(SysRegMappings); ++i) { - if (SysRegMappings[i].Name == NameLower) { + if (SysRegMappings[i].isNameEqual(NameLower, FeatureBits)) { Valid = true; return SysRegMappings[i].Value; } } - // Next search for target specific registers - if (FeatureBits & AArch64::ProcCyclone) { - for (unsigned i = 0; i < array_lengthof(CycloneSysRegMappings); ++i) { - if (CycloneSysRegMappings[i].Name == NameLower) { - Valid = true; - return CycloneSysRegMappings[i].Value; - } - } - } - // Now try the instruction-specific registers (either read-only or // write-only). for (unsigned i = 0; i < NumInstMappings; ++i) { - if (InstMappings[i].Name == NameLower) { + if (InstMappings[i].isNameEqual(NameLower, FeatureBits)) { Valid = true; return InstMappings[i].Value; } @@ -814,27 +848,18 @@ AArch64SysReg::SysRegMapper::fromString(StringRef Name, bool &Valid) const { } std::string -AArch64SysReg::SysRegMapper::toString(uint32_t Bits) const { +AArch64SysReg::SysRegMapper::toString(uint32_t Bits, uint64_t FeatureBits) const { // First search the registers shared by all for (unsigned i = 0; i < array_lengthof(SysRegMappings); ++i) { - if (SysRegMappings[i].Value == Bits) { + if (SysRegMappings[i].isValueEqual(Bits, FeatureBits)) { return SysRegMappings[i].Name; } } - // Next search for target specific registers - if (FeatureBits & AArch64::ProcCyclone) { - for (unsigned i = 0; i < array_lengthof(CycloneSysRegMappings); ++i) { - if (CycloneSysRegMappings[i].Value == Bits) { - return CycloneSysRegMappings[i].Name; - } - } - } - // Now try the instruction-specific registers (either read-only or // write-only). for (unsigned i = 0; i < NumInstMappings; ++i) { - if (InstMappings[i].Value == Bits) { + if (InstMappings[i].isValueEqual(Bits, FeatureBits)) { return InstMappings[i].Name; } } @@ -851,38 +876,38 @@ AArch64SysReg::SysRegMapper::toString(uint32_t Bits) const { } const AArch64NamedImmMapper::Mapping AArch64TLBI::TLBIMapper::TLBIMappings[] = { - {"ipas2e1is", IPAS2E1IS}, - {"ipas2le1is", IPAS2LE1IS}, - {"vmalle1is", VMALLE1IS}, - {"alle2is", ALLE2IS}, - {"alle3is", ALLE3IS}, - {"vae1is", VAE1IS}, - {"vae2is", VAE2IS}, - {"vae3is", VAE3IS}, - {"aside1is", ASIDE1IS}, - {"vaae1is", VAAE1IS}, - {"alle1is", ALLE1IS}, - {"vale1is", VALE1IS}, - {"vale2is", VALE2IS}, - {"vale3is", VALE3IS}, - {"vmalls12e1is", VMALLS12E1IS}, - {"vaale1is", VAALE1IS}, - {"ipas2e1", IPAS2E1}, - {"ipas2le1", IPAS2LE1}, - {"vmalle1", VMALLE1}, - {"alle2", ALLE2}, - {"alle3", ALLE3}, - {"vae1", VAE1}, - {"vae2", VAE2}, - {"vae3", VAE3}, - {"aside1", ASIDE1}, - {"vaae1", VAAE1}, - {"alle1", ALLE1}, - {"vale1", VALE1}, - {"vale2", VALE2}, - {"vale3", VALE3}, - {"vmalls12e1", VMALLS12E1}, - {"vaale1", VAALE1} + {"ipas2e1is", IPAS2E1IS, 0}, + {"ipas2le1is", IPAS2LE1IS, 0}, + {"vmalle1is", VMALLE1IS, 0}, + {"alle2is", ALLE2IS, 0}, + {"alle3is", ALLE3IS, 0}, + {"vae1is", VAE1IS, 0}, + {"vae2is", VAE2IS, 0}, + {"vae3is", VAE3IS, 0}, + {"aside1is", ASIDE1IS, 0}, + {"vaae1is", VAAE1IS, 0}, + {"alle1is", ALLE1IS, 0}, + {"vale1is", VALE1IS, 0}, + {"vale2is", VALE2IS, 0}, + {"vale3is", VALE3IS, 0}, + {"vmalls12e1is", VMALLS12E1IS, 0}, + {"vaale1is", VAALE1IS, 0}, + {"ipas2e1", IPAS2E1, 0}, + {"ipas2le1", IPAS2LE1, 0}, + {"vmalle1", VMALLE1, 0}, + {"alle2", ALLE2, 0}, + {"alle3", ALLE3, 0}, + {"vae1", VAE1, 0}, + {"vae2", VAE2, 0}, + {"vae3", VAE3, 0}, + {"aside1", ASIDE1, 0}, + {"vaae1", VAAE1, 0}, + {"alle1", ALLE1, 0}, + {"vale1", VALE1, 0}, + {"vale2", VALE2, 0}, + {"vale3", VALE3, 0}, + {"vmalls12e1", VMALLS12E1, 0}, + {"vaale1", VAALE1, 0} }; AArch64TLBI::TLBIMapper::TLBIMapper() diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/lib/Target/AArch64/Utils/AArch64BaseInfo.h index 2ae6f52..659ea90 100644 --- a/lib/Target/AArch64/Utils/AArch64BaseInfo.h +++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.h @@ -280,14 +280,26 @@ struct AArch64NamedImmMapper { struct Mapping { const char *Name; uint32_t Value; + uint64_t AvailableForFeatures; + // empty AvailableForFeatures means "always-on" + bool isNameEqual(std::string Other, uint64_t FeatureBits=~0ULL) const { + if (AvailableForFeatures && !(AvailableForFeatures & FeatureBits)) + return false; + return Name == Other; + } + bool isValueEqual(uint32_t Other, uint64_t FeatureBits=~0ULL) const { + if (AvailableForFeatures && !(AvailableForFeatures & FeatureBits)) + return false; + return Value == Other; + } }; template AArch64NamedImmMapper(const Mapping (&Mappings)[N], uint32_t TooBigImm) : Mappings(&Mappings[0]), NumMappings(N), TooBigImm(TooBigImm) {} - StringRef toString(uint32_t Value, bool &Valid) const; - uint32_t fromString(StringRef Name, bool &Valid) const; + StringRef toString(uint32_t Value, uint64_t FeatureBits, bool &Valid) const; + uint32_t fromString(StringRef Name, uint64_t FeatureBits, bool &Valid) const; /// Many of the instructions allow an alternative assembly form consisting of /// a simple immediate. Currently the only valid forms are ranges [0, N) where @@ -435,7 +447,10 @@ namespace AArch64PState { Invalid = -1, SPSel = 0x05, DAIFSet = 0x1e, - DAIFClr = 0x1f + DAIFClr = 0x1f, + + // v8.1a "Privileged Access Never" extension-specific PStates + PAN = 0x04, }; struct PStateMapper : AArch64NamedImmMapper { @@ -1122,11 +1137,48 @@ namespace AArch64SysReg { ICH_LR13_EL2 = 0xe66d, // 11 100 1100 1101 101 ICH_LR14_EL2 = 0xe66e, // 11 100 1100 1101 110 ICH_LR15_EL2 = 0xe66f, // 11 100 1100 1101 111 - }; - // Cyclone specific system registers - enum CycloneSysRegValues { - CPM_IOACC_CTL_EL3 = 0xff90 + // v8.1a "Privileged Access Never" extension-specific system registers + PAN = 0xc213, // 11 000 0100 0010 011 + + // v8.1a "Limited Ordering Regions" extension-specific system registers + LORSA_EL1 = 0xc520, // 11 000 1010 0100 000 + LOREA_EL1 = 0xc521, // 11 000 1010 0100 001 + LORN_EL1 = 0xc522, // 11 000 1010 0100 010 + LORC_EL1 = 0xc523, // 11 000 1010 0100 011 + LORID_EL1 = 0xc527, // 11 000 1010 0100 111 + + // v8.1a "Virtualization host extensions" system registers + TTBR1_EL2 = 0xe101, // 11 100 0010 0000 001 + CONTEXTIDR_EL2 = 0xe681, // 11 100 1101 0000 001 + CNTHV_TVAL_EL2 = 0xe718, // 11 100 1110 0011 000 + CNTHV_CVAL_EL2 = 0xe71a, // 11 100 1110 0011 010 + CNTHV_CTL_EL2 = 0xe719, // 11 100 1110 0011 001 + SCTLR_EL12 = 0xe880, // 11 101 0001 0000 000 + CPACR_EL12 = 0xe882, // 11 101 0001 0000 010 + TTBR0_EL12 = 0xe900, // 11 101 0010 0000 000 + TTBR1_EL12 = 0xe901, // 11 101 0010 0000 001 + TCR_EL12 = 0xe902, // 11 101 0010 0000 010 + AFSR0_EL12 = 0xea88, // 11 101 0101 0001 000 + AFSR1_EL12 = 0xea89, // 11 101 0101 0001 001 + ESR_EL12 = 0xea90, // 11 101 0101 0010 000 + FAR_EL12 = 0xeb00, // 11 101 0110 0000 000 + MAIR_EL12 = 0xed10, // 11 101 1010 0010 000 + AMAIR_EL12 = 0xed18, // 11 101 1010 0011 000 + VBAR_EL12 = 0xee00, // 11 101 1100 0000 000 + CONTEXTIDR_EL12 = 0xee81, // 11 101 1101 0000 001 + CNTKCTL_EL12 = 0xef08, // 11 101 1110 0001 000 + CNTP_TVAL_EL02 = 0xef10, // 11 101 1110 0010 000 + CNTP_CTL_EL02 = 0xef11, // 11 101 1110 0010 001 + CNTP_CVAL_EL02 = 0xef12, // 11 101 1110 0010 010 + CNTV_TVAL_EL02 = 0xef18, // 11 101 1110 0011 000 + CNTV_CTL_EL02 = 0xef19, // 11 101 1110 0011 001 + CNTV_CVAL_EL02 = 0xef1a, // 11 101 1110 0011 010 + SPSR_EL12 = 0xea00, // 11 101 0100 0000 000 + ELR_EL12 = 0xea01, // 11 101 0100 0000 001 + + // Cyclone specific system registers + CPM_IOACC_CTL_EL3 = 0xff90, }; // Note that these do not inherit from AArch64NamedImmMapper. This class is @@ -1135,25 +1187,23 @@ namespace AArch64SysReg { // this one case. struct SysRegMapper { static const AArch64NamedImmMapper::Mapping SysRegMappings[]; - static const AArch64NamedImmMapper::Mapping CycloneSysRegMappings[]; const AArch64NamedImmMapper::Mapping *InstMappings; size_t NumInstMappings; - uint64_t FeatureBits; - SysRegMapper(uint64_t FeatureBits) : FeatureBits(FeatureBits) { } - uint32_t fromString(StringRef Name, bool &Valid) const; - std::string toString(uint32_t Bits) const; + SysRegMapper() { } + uint32_t fromString(StringRef Name, uint64_t FeatureBits, bool &Valid) const; + std::string toString(uint32_t Bits, uint64_t FeatureBits) const; }; struct MSRMapper : SysRegMapper { static const AArch64NamedImmMapper::Mapping MSRMappings[]; - MSRMapper(uint64_t FeatureBits); + MSRMapper(); }; struct MRSMapper : SysRegMapper { static const AArch64NamedImmMapper::Mapping MRSMappings[]; - MRSMapper(uint64_t FeatureBits); + MRSMapper(); }; uint32_t ParseGenericRegister(StringRef Name, bool &Valid); diff --git a/lib/Target/ARM/ARM.td b/lib/Target/ARM/ARM.td index ce0aed9..bd1c7af 100644 --- a/lib/Target/ARM/ARM.td +++ b/lib/Target/ARM/ARM.td @@ -180,7 +180,7 @@ def HasV8Ops : SubtargetFeature<"v8", "HasV8Ops", "true", "Support ARM v8 instructions", [HasV7Ops, FeatureVirtualization, FeatureMP]>; -def FeatureV8_1a : SubtargetFeature<"v8.1a", "HasV8_1a", "true", +def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true", "Support ARM v8.1a instructions", [HasV8Ops, FeatureAClass, FeatureCRC]>; @@ -260,6 +260,14 @@ def ProcA57 : SubtargetFeature<"a57", "ARMProcFamily", "CortexA57", FeatureTrustZone, FeatureT2XtPk, FeatureCrypto, FeatureCRC]>; +def ProcR4 : SubtargetFeature<"r4", "ARMProcFamily", "CortexR4", + "Cortex-R4 ARM processors", + [FeatureHWDiv, + FeatureAvoidPartialCPSR, + FeatureDSPThumb2, FeatureT2XtPk, + HasV7Ops, FeatureDB, FeatureHasRAS, + FeatureRClass]>; + def ProcR5 : SubtargetFeature<"r5", "ARMProcFamily", "CortexR5", "Cortex-R5 ARM processors", [FeatureSlowFPBrcc, @@ -396,6 +404,16 @@ def : ProcessorModel<"krait", CortexA9Model, FeatureDSPThumb2, FeatureHasRAS, FeatureAClass]>; +// FIXME: R4 has currently the same ProcessorModel as A8. +def : ProcessorModel<"cortex-r4", CortexA8Model, + [ProcR4]>; + +// FIXME: R4F has currently the same ProcessorModel as A8. +def : ProcessorModel<"cortex-r4f", CortexA8Model, + [ProcR4, + FeatureSlowFPBrcc, FeatureHasSlowFPVMLx, + FeatureVFP3, FeatureVFPOnlySP, FeatureD16]>; + // FIXME: R5 has currently the same ProcessorModel as A8. def : ProcessorModel<"cortex-r5", CortexA8Model, [ProcR5, HasV7Ops, FeatureDB, @@ -457,14 +475,6 @@ def : ProcessorModel<"cyclone", SwiftModel, FeatureDB,FeatureDSPThumb2, FeatureHasRAS, FeatureZCZeroing]>; -// V8.1 Processors -def : ProcNoItin<"generic-armv8.1-a", [HasV8Ops, FeatureV8_1a, - FeatureDB, FeatureFPARMv8, - FeatureNEON, FeatureDSPThumb2, - FeatureHWDiv, FeatureHWDivARM, - FeatureTrustZone, FeatureT2XtPk, - FeatureCrypto]>; - //===----------------------------------------------------------------------===// // Register File Description //===----------------------------------------------------------------------===// @@ -485,7 +495,15 @@ def ARMInstrInfo : InstrInfo; // Declare the target which we are implementing //===----------------------------------------------------------------------===// +def ARMAsmWriter : AsmWriter { + string AsmWriterClassName = "InstPrinter"; + int PassSubtarget = 1; + int Variant = 0; + bit isMCAsmWriter = 1; +} + def ARM : Target { // Pull in Instruction Info: let InstructionSet = ARMInstrInfo; + let AssemblyWriters = [ARMAsmWriter]; } diff --git a/lib/Target/ARM/ARMAsmPrinter.cpp b/lib/Target/ARM/ARMAsmPrinter.cpp index 102def1..1a2acf5 100644 --- a/lib/Target/ARM/ARMAsmPrinter.cpp +++ b/lib/Target/ARM/ARMAsmPrinter.cpp @@ -207,7 +207,7 @@ GetARMJTIPICJumpTableLabel2(unsigned uid, unsigned uid2) const { SmallString<60> Name; raw_svector_ostream(Name) << DL->getPrivateGlobalPrefix() << "JTI" << getFunctionNumber() << '_' << uid << '_' << uid2; - return OutContext.GetOrCreateSymbol(Name.str()); + return OutContext.GetOrCreateSymbol(Name); } @@ -216,7 +216,7 @@ MCSymbol *ARMAsmPrinter::GetARMSJLJEHLabel() const { SmallString<60> Name; raw_svector_ostream(Name) << DL->getPrivateGlobalPrefix() << "SJLJEH" << getFunctionNumber(); - return OutContext.GetOrCreateSymbol(Name.str()); + return OutContext.GetOrCreateSymbol(Name); } bool ARMAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, @@ -520,28 +520,6 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) { // generates code that does this, it is always safe to set. OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols); } - - // Emit a .data.rel section containing any stubs that were created. - if (TT.isOSBinFormatELF()) { - const TargetLoweringObjectFileELF &TLOFELF = - static_cast(getObjFileLowering()); - - MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo(); - - // Output stubs for external and common global variables. - MachineModuleInfoELF::SymbolListTy Stubs = MMIELF.GetGVStubList(); - if (!Stubs.empty()) { - OutStreamer.SwitchSection(TLOFELF.getDataRelSection()); - const DataLayout *TD = TM.getDataLayout(); - - for (auto &stub: Stubs) { - OutStreamer.EmitLabel(stub.first); - OutStreamer.EmitSymbolValue(stub.second.getPointer(), - TD->getPointerSize(0)); - } - Stubs.clear(); - } - } } //===----------------------------------------------------------------------===// @@ -597,7 +575,7 @@ void ARMAsmPrinter::emitAttributes() { std::string ArchFS = ARM_MC::ParseARMTriple(TT, CPU); if (!FS.empty()) { if (!ArchFS.empty()) - ArchFS = ArchFS + "," + FS.str(); + ArchFS = (Twine(ArchFS) + "," + FS).str(); else ArchFS = FS; } @@ -661,8 +639,8 @@ void ARMAsmPrinter::emitAttributes() { // Emit Tag_Advanced_SIMD_arch for ARMv8 architecture if (STI.hasV8Ops()) ATS.emitAttribute(ARMBuildAttrs::Advanced_SIMD_arch, - STI.hasV8_1a() ? ARMBuildAttrs::AllowNeonARMv8_1a: - ARMBuildAttrs::AllowNeonARMv8); + STI.hasV8_1aOps() ? ARMBuildAttrs::AllowNeonARMv8_1a: + ARMBuildAttrs::AllowNeonARMv8); } else { if (STI.hasFPARMv8()) // FPv5 and FP-ARMv8 have the same instructions, so are modeled as one diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/lib/Target/ARM/ARMBaseRegisterInfo.cpp index a8c7657..3f79a9b 100644 --- a/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -245,11 +245,15 @@ ARMBaseRegisterInfo::getRegAllocationHints(unsigned VirtReg, // This register should preferably be even (Odd == 0) or odd (Odd == 1). // Check if the other part of the pair has already been assigned, and provide // the paired register as the first hint. + unsigned Paired = Hint.second; + if (Paired == 0) + return; + unsigned PairedPhys = 0; - if (VRM && VRM->hasPhys(Hint.second)) { - PairedPhys = getPairedGPR(VRM->getPhys(Hint.second), Odd, this); - if (PairedPhys && MRI.isReserved(PairedPhys)) - PairedPhys = 0; + if (TargetRegisterInfo::isPhysicalRegister(Paired)) { + PairedPhys = Paired; + } else if (VRM && VRM->hasPhys(Paired)) { + PairedPhys = getPairedGPR(VRM->getPhys(Paired), Odd, this); } // First prefer the paired physreg. @@ -284,9 +288,14 @@ ARMBaseRegisterInfo::updateRegAllocHint(unsigned Reg, unsigned NewReg, // change. unsigned OtherReg = Hint.second; Hint = MRI->getRegAllocationHint(OtherReg); - if (Hint.second == Reg) - // Make sure the pair has not already divorced. + // Make sure the pair has not already divorced. + if (Hint.second == Reg) { MRI->setRegAllocationHint(OtherReg, Hint.first, NewReg); + if (TargetRegisterInfo::isVirtualRegister(NewReg)) + MRI->setRegAllocationHint(NewReg, + Hint.first == (unsigned)ARMRI::RegPairOdd ? ARMRI::RegPairEven + : ARMRI::RegPairOdd, OtherReg); + } } } diff --git a/lib/Target/ARM/ARMConstantPoolValue.h b/lib/Target/ARM/ARMConstantPoolValue.h index 13bef54..36f63e2 100644 --- a/lib/Target/ARM/ARMConstantPoolValue.h +++ b/lib/Target/ARM/ARMConstantPoolValue.h @@ -86,7 +86,7 @@ protected: } public: - virtual ~ARMConstantPoolValue(); + ~ARMConstantPoolValue() override; ARMCP::ARMCPModifier getModifier() const { return Modifier; } const char *getModifierText() const; diff --git a/lib/Target/ARM/ARMFrameLowering.cpp b/lib/Target/ARM/ARMFrameLowering.cpp index 830953b..9d2b09b 100644 --- a/lib/Target/ARM/ARMFrameLowering.cpp +++ b/lib/Target/ARM/ARMFrameLowering.cpp @@ -311,6 +311,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF) const { return; StackAdjustingInsts DefCFAOffsetCandidates; + bool HasFP = hasFP(MF); // Allocate the vararg register save area. if (ArgRegsSaveSize) { @@ -327,6 +328,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF) const { DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes - ArgRegsSaveSize, true); } + DefCFAOffsetCandidates.emitDefCFAOffsets(MMI, MBB, dl, TII, HasFP); return; } @@ -375,7 +377,6 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF) const { } // Determine starting offsets of spill areas. - bool HasFP = hasFP(MF); unsigned GPRCS1Offset = NumBytes - ArgRegsSaveSize - GPRCS1Size; unsigned GPRCS2Offset = GPRCS1Offset - GPRCS2Size; unsigned DPRAlign = DPRCSSize ? std::min(8U, Align) : 4U; diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index 3b1b8dd..72afd2c 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -3504,25 +3504,34 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { ARMCC::CondCodes CondCode, CondCode2; FPCCToARMCC(CC, CondCode, CondCode2); - // Try to generate VSEL on ARMv8. + // Try to generate VMAXNM/VMINNM on ARMv8. if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 || TrueVal.getValueType() == MVT::f64)) { - // We can select VMAXNM/VMINNM from a compare followed by a select with the + // We can use VMAXNM/VMINNM for a compare followed by a select with the // same operands, as follows: - // c = fcmp [ogt, olt, ugt, ult] a, b + // c = fcmp [?gt, ?ge, ?lt, ?le] a, b // select c, a, b - // We only do this in unsafe-fp-math, because signed zeros and NaNs are - // handled differently than the original code sequence. + // In NoNaNsFPMath the CC will have been changed from, e.g., 'ogt' to 'gt'. + // We only do this transformation in UnsafeFPMath and for no-NaNs + // comparisons, because signed zeros and NaNs are handled differently than + // the original code sequence. + // FIXME: There are more cases that can be transformed even with NaNs, + // signed zeroes and safe math. E.g. in the following, the result will be + // FalseVal if a is a NaN or -0./0. and that's what vmaxnm will give, too. + // c = fcmp ogt, a, 0. ; select c, a, 0. => vmaxnm a, 0. + // FIXME: There is similar code that allows some extensions in + // AArch64TargetLowering::LowerSELECT_CC that should be shared with this + // code. if (getTargetMachine().Options.UnsafeFPMath) { if (LHS == TrueVal && RHS == FalseVal) { - if (CC == ISD::SETOGT || CC == ISD::SETUGT) + if (CC == ISD::SETGT || CC == ISD::SETGE) return DAG.getNode(ARMISD::VMAXNM, dl, VT, TrueVal, FalseVal); - if (CC == ISD::SETOLT || CC == ISD::SETULT) + if (CC == ISD::SETLT || CC == ISD::SETLE) return DAG.getNode(ARMISD::VMINNM, dl, VT, TrueVal, FalseVal); } else if (LHS == FalseVal && RHS == TrueVal) { - if (CC == ISD::SETOLT || CC == ISD::SETULT) + if (CC == ISD::SETLT || CC == ISD::SETLE) return DAG.getNode(ARMISD::VMAXNM, dl, VT, TrueVal, FalseVal); - if (CC == ISD::SETOGT || CC == ISD::SETUGT) + if (CC == ISD::SETGT || CC == ISD::SETGE) return DAG.getNode(ARMISD::VMINNM, dl, VT, TrueVal, FalseVal); } } diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td index c3984ca..52f3555 100644 --- a/lib/Target/ARM/ARMInstrInfo.td +++ b/lib/Target/ARM/ARMInstrInfo.td @@ -208,6 +208,8 @@ def HasV8 : Predicate<"Subtarget->hasV8Ops()">, AssemblerPredicate<"HasV8Ops", "armv8">; def PreV8 : Predicate<"!Subtarget->hasV8Ops()">, AssemblerPredicate<"!HasV8Ops", "armv7 or earlier">; +def HasV8_1a : Predicate<"Subtarget->hasV8_1aOps()">, + AssemblerPredicate<"HasV8_1aOps", "armv8.1a">; def NoVFP : Predicate<"!Subtarget->hasVFP2()">; def HasVFP2 : Predicate<"Subtarget->hasVFP2()">, AssemblerPredicate<"FeatureVFP2", "VFP2">; @@ -226,8 +228,6 @@ def HasCrypto : Predicate<"Subtarget->hasCrypto()">, AssemblerPredicate<"FeatureCrypto", "crypto">; def HasCRC : Predicate<"Subtarget->hasCRC()">, AssemblerPredicate<"FeatureCRC", "crc">; -def HasV8_1a : Predicate<"Subtarget->hasV8_1a()">, - AssemblerPredicate<"FeatureV8_1a", "v8.1a">; def HasFP16 : Predicate<"Subtarget->hasFP16()">, AssemblerPredicate<"FeatureFP16","half-float">; def HasDivide : Predicate<"Subtarget->hasDivide()">, @@ -388,6 +388,9 @@ def fsub_mlx : PatFrag<(ops node:$lhs, node:$rhs),(fsub node:$lhs, node:$rhs),[{ // Immediate operands with a shared generic asm render method. class ImmAsmOperand : AsmOperandClass { let RenderMethod = "addImmOperands"; } +// Operands that are part of a memory addressing mode. +class MemOperand : Operand { let OperandType = "OPERAND_MEMORY"; } + // Branch target. // FIXME: rename brtarget to t2_brtarget def brtarget : Operand { @@ -790,7 +793,7 @@ def imm1_16 : Operand, PatLeaf<(imm), [{ return Imm > 0 && Imm <= 16; }], // addrmode_imm12 := reg +/- imm12 // def MemImm12OffsetAsmOperand : AsmOperandClass { let Name = "MemImm12Offset"; } -class AddrMode_Imm12 : Operand, +class AddrMode_Imm12 : MemOperand, ComplexPattern { // 12-bit immediate operand. Note that instructions using this encode // #0 and #-0 differently. We flag #-0 as the magic value INT32_MIN. All other @@ -813,7 +816,7 @@ def addrmode_imm12_pre : AddrMode_Imm12 { // ldst_so_reg := reg +/- reg shop imm // def MemRegOffsetAsmOperand : AsmOperandClass { let Name = "MemRegOffset"; } -def ldst_so_reg : Operand, +def ldst_so_reg : MemOperand, ComplexPattern { let EncoderMethod = "getLdStSORegOpValue"; // FIXME: Simplify the printer @@ -829,7 +832,7 @@ def ldst_so_reg : Operand, // {8} 1 is imm8 is non-negative. 0 otherwise. // {7-0} [0,255] imm8 value. def PostIdxImm8AsmOperand : AsmOperandClass { let Name = "PostIdxImm8"; } -def postidx_imm8 : Operand { +def postidx_imm8 : MemOperand { let PrintMethod = "printPostIdxImm8Operand"; let ParserMatchClass = PostIdxImm8AsmOperand; let MIOperandInfo = (ops i32imm); @@ -841,7 +844,7 @@ def postidx_imm8 : Operand { // {8} 1 is imm8 is non-negative. 0 otherwise. // {7-0} [0,255] imm8 value, scaled by 4. def PostIdxImm8s4AsmOperand : AsmOperandClass { let Name = "PostIdxImm8s4"; } -def postidx_imm8s4 : Operand { +def postidx_imm8s4 : MemOperand { let PrintMethod = "printPostIdxImm8s4Operand"; let ParserMatchClass = PostIdxImm8s4AsmOperand; let MIOperandInfo = (ops i32imm); @@ -854,7 +857,7 @@ def PostIdxRegAsmOperand : AsmOperandClass { let Name = "PostIdxReg"; let ParserMethod = "parsePostIdxReg"; } -def postidx_reg : Operand { +def postidx_reg : MemOperand { let EncoderMethod = "getPostIdxRegOpValue"; let DecoderMethod = "DecodePostIdxReg"; let PrintMethod = "printPostIdxRegOperand"; @@ -869,7 +872,7 @@ def postidx_reg : Operand { // FIXME: addrmode2 should be refactored the rest of the way to always // use explicit imm vs. reg versions above (addrmode_imm12 and ldst_so_reg). def AddrMode2AsmOperand : AsmOperandClass { let Name = "AddrMode2"; } -def addrmode2 : Operand, +def addrmode2 : MemOperand, ComplexPattern { let EncoderMethod = "getAddrMode2OpValue"; let PrintMethod = "printAddrMode2Operand"; @@ -881,7 +884,7 @@ def PostIdxRegShiftedAsmOperand : AsmOperandClass { let Name = "PostIdxRegShifted"; let ParserMethod = "parsePostIdxReg"; } -def am2offset_reg : Operand, +def am2offset_reg : MemOperand, ComplexPattern { let EncoderMethod = "getAddrMode2OffsetOpValue"; @@ -894,7 +897,7 @@ def am2offset_reg : Operand, // FIXME: am2offset_imm should only need the immediate, not the GPR. Having // the GPR is purely vestigal at this point. def AM2OffsetImmAsmOperand : AsmOperandClass { let Name = "AM2OffsetImm"; } -def am2offset_imm : Operand, +def am2offset_imm : MemOperand, ComplexPattern { let EncoderMethod = "getAddrMode2OffsetOpValue"; @@ -909,7 +912,7 @@ def am2offset_imm : Operand, // // FIXME: split into imm vs. reg versions. def AddrMode3AsmOperand : AsmOperandClass { let Name = "AddrMode3"; } -class AddrMode3 : Operand, +class AddrMode3 : MemOperand, ComplexPattern { let EncoderMethod = "getAddrMode3OpValue"; let ParserMatchClass = AddrMode3AsmOperand; @@ -932,7 +935,7 @@ def AM3OffsetAsmOperand : AsmOperandClass { let Name = "AM3Offset"; let ParserMethod = "parseAM3Offset"; } -def am3offset : Operand, +def am3offset : MemOperand, ComplexPattern { let EncoderMethod = "getAddrMode3OffsetOpValue"; @@ -951,7 +954,7 @@ def ldstm_mode : OptionalDefOperand { // addrmode5 := reg +/- imm8*4 // def AddrMode5AsmOperand : AsmOperandClass { let Name = "AddrMode5"; } -class AddrMode5 : Operand, +class AddrMode5 : MemOperand, ComplexPattern { let EncoderMethod = "getAddrMode5OpValue"; let DecoderMethod = "DecodeAddrMode5Operand"; @@ -970,7 +973,7 @@ def addrmode5_pre : AddrMode5 { // addrmode6 := reg with optional alignment // def AddrMode6AsmOperand : AsmOperandClass { let Name = "AlignedMemory"; } -def addrmode6 : Operand, +def addrmode6 : MemOperand, ComplexPattern{ let PrintMethod = "printAddrMode6Operand"; let MIOperandInfo = (ops GPR:$addr, i32imm:$align); @@ -979,7 +982,7 @@ def addrmode6 : Operand, let ParserMatchClass = AddrMode6AsmOperand; } -def am6offset : Operand, +def am6offset : MemOperand, ComplexPattern { let PrintMethod = "printAddrMode6OffsetOperand"; @@ -990,7 +993,7 @@ def am6offset : Operand, // Special version of addrmode6 to handle alignment encoding for VST1/VLD1 // (single element from one lane) for size 32. -def addrmode6oneL32 : Operand, +def addrmode6oneL32 : MemOperand, ComplexPattern{ let PrintMethod = "printAddrMode6Operand"; let MIOperandInfo = (ops GPR:$addr, i32imm); @@ -998,7 +1001,7 @@ def addrmode6oneL32 : Operand, } // Base class for addrmode6 with specific alignment restrictions. -class AddrMode6Align : Operand, +class AddrMode6Align : MemOperand, ComplexPattern{ let PrintMethod = "printAddrMode6Operand"; let MIOperandInfo = (ops GPR:$addr, i32imm:$align); @@ -1074,7 +1077,7 @@ def addrmode6align64or128or256 : AddrMode6Align { // Special version of addrmode6 to handle alignment encoding for VLD-dup // instructions, specifically VLD4-dup. -def addrmode6dup : Operand, +def addrmode6dup : MemOperand, ComplexPattern{ let PrintMethod = "printAddrMode6Operand"; let MIOperandInfo = (ops GPR:$addr, i32imm); @@ -1085,7 +1088,7 @@ def addrmode6dup : Operand, } // Base class for addrmode6dup with specific alignment restrictions. -class AddrMode6DupAlign : Operand, +class AddrMode6DupAlign : MemOperand, ComplexPattern{ let PrintMethod = "printAddrMode6Operand"; let MIOperandInfo = (ops GPR:$addr, i32imm); @@ -1149,7 +1152,7 @@ def addrmode6dupalign64or128 : AddrMode6DupAlign { // addrmodepc := pc + reg // -def addrmodepc : Operand, +def addrmodepc : MemOperand, ComplexPattern { let PrintMethod = "printAddrModePCOperand"; let MIOperandInfo = (ops GPR, i32imm); @@ -1158,7 +1161,7 @@ def addrmodepc : Operand, // addr_offset_none := reg // def MemNoOffsetAsmOperand : AsmOperandClass { let Name = "MemNoOffset"; } -def addr_offset_none : Operand, +def addr_offset_none : MemOperand, ComplexPattern { let PrintMethod = "printAddrMode7Operand"; let DecoderMethod = "DecodeAddrMode7Operand"; @@ -1417,7 +1420,8 @@ multiclass AsI1_rbin_s_is opcod, string opc, InstrItinClass iii, InstrItinClass iir, InstrItinClass iis, - PatFrag opnode, bit Commutable = 0> { + PatFrag opnode, bit Commutable = 0, + string rrDecoderMethod = ""> { def ri : AI1, @@ -1445,6 +1449,7 @@ multiclass AI1_cmp_irs opcod, string opc, let Inst{15-12} = 0b0000; let Inst{11-4} = 0b00000000; let Inst{3-0} = Rm; + let DecoderMethod = rrDecoderMethod; let Unpredictable{15-12} = 0b1111; } @@ -4263,6 +4268,30 @@ def CRC32W : AI_crc32<0, 0b10, "w", int_arm_crc32w>; def CRC32CW : AI_crc32<1, 0b10, "cw", int_arm_crc32cw>; //===----------------------------------------------------------------------===// +// ARMv8.1a Privilege Access Never extension +// +// SETPAN #imm1 + +def SETPAN : AInoP<(outs), (ins imm0_1:$imm), MiscFrm, NoItinerary, "setpan", + "\t$imm", []>, Requires<[IsARM, HasV8, HasV8_1a]> { + bits<1> imm; + + let Inst{31-28} = 0b1111; + let Inst{27-20} = 0b00010001; + let Inst{19-16} = 0b0000; + let Inst{15-10} = 0b000000; + let Inst{9} = imm; + let Inst{8} = 0b0; + let Inst{7-4} = 0b0000; + let Inst{3-0} = 0b0000; + + let Unpredictable{19-16} = 0b1111; + let Unpredictable{15-10} = 0b111111; + let Unpredictable{8} = 0b1; + let Unpredictable{3-0} = 0b1111; +} + +//===----------------------------------------------------------------------===// // Comparison Instructions... // @@ -4366,7 +4395,8 @@ def : ARMPat<(ARMcmpZ GPR:$src, mod_imm_neg:$imm), // Note that TST/TEQ don't set all the same flags that CMP does! defm TST : AI1_cmp_irs<0b1000, "tst", IIC_iTSTi, IIC_iTSTr, IIC_iTSTsr, - BinOpFrag<(ARMcmpZ (and_su node:$LHS, node:$RHS), 0)>, 1>; + BinOpFrag<(ARMcmpZ (and_su node:$LHS, node:$RHS), 0)>, 1, + "DecodeTSTInstruction">; defm TEQ : AI1_cmp_irs<0b1001, "teq", IIC_iTSTi, IIC_iTSTr, IIC_iTSTsr, BinOpFrag<(ARMcmpZ (xor_su node:$LHS, node:$RHS), 0)>, 1>; diff --git a/lib/Target/ARM/ARMInstrThumb.td b/lib/Target/ARM/ARMInstrThumb.td index 3c62e0e..d0ade77 100644 --- a/lib/Target/ARM/ARMInstrThumb.td +++ b/lib/Target/ARM/ARMInstrThumb.td @@ -142,7 +142,7 @@ def t_blxtarget : Operand { // t_addrmode_pc :=