diff options
Diffstat (limited to 'lib/Target/X86')
102 files changed, 10511 insertions, 6535 deletions
diff --git a/lib/Target/X86/Android.mk b/lib/Target/X86/Android.mk index c9840c3..73031de 100644 --- a/lib/Target/X86/Android.mk +++ b/lib/Target/X86/Android.mk @@ -53,6 +53,7 @@ include $(BUILD_HOST_STATIC_LIBRARY) # For the device only # ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) @@ -68,3 +69,4 @@ include $(LLVM_DEVICE_BUILD_MK) include $(LLVM_TBLGEN_RULES_MK) include $(LLVM_GEN_INTRINSICS_MK) include $(BUILD_STATIC_LIBRARY) +endif diff --git a/lib/Target/X86/AsmParser/Android.mk b/lib/Target/X86/AsmParser/Android.mk index d6de437..4235cb1 100644 --- a/lib/Target/X86/AsmParser/Android.mk +++ b/lib/Target/X86/AsmParser/Android.mk @@ -6,6 +6,7 @@ include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) x86_asm_parser_SRC_FILES := \ + X86AsmInstrumentation.cpp \ X86AsmParser.cpp x86_asm_parser_TBLGEN_TABLES := \ @@ -41,6 +42,7 @@ include $(BUILD_HOST_STATIC_LIBRARY) #===---------------------------------------------------------------=== # libX86AsmParser (target) #===---------------------------------------------------------------=== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) @@ -54,3 +56,4 @@ TBLGEN_TD_DIR := $(x86_asm_parser_TBLGEN_TD_DIR) include $(LLVM_DEVICE_BUILD_MK) include $(LLVM_TBLGEN_RULES_MK) include $(BUILD_STATIC_LIBRARY) +endif diff --git a/lib/Target/X86/AsmParser/CMakeLists.txt b/lib/Target/X86/AsmParser/CMakeLists.txt index 54204d4..b022a41 100644 --- a/lib/Target/X86/AsmParser/CMakeLists.txt +++ b/lib/Target/X86/AsmParser/CMakeLists.txt @@ -1,7 +1,4 @@ -include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. ) - add_llvm_library(LLVMX86AsmParser + X86AsmInstrumentation.cpp X86AsmParser.cpp ) - -add_dependencies(LLVMX86AsmParser X86CommonTableGen) diff --git a/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp b/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp new file mode 100644 index 0000000..db29228 --- /dev/null +++ b/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp @@ -0,0 +1,236 @@ +//===-- X86AsmInstrumentation.cpp - Instrument X86 inline assembly C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "MCTargetDesc/X86BaseInfo.h" +#include "X86AsmInstrumentation.h" +#include "X86Operand.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstBuilder.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Compiler.h" +#include "llvm/MC/MCParser/MCParsedAsmOperand.h" + +namespace llvm { +namespace { + +static cl::opt<bool> ClAsanInstrumentInlineAssembly( + "asan-instrument-inline-assembly", cl::desc("instrument inline assembly"), + cl::Hidden, cl::init(false)); + +bool IsStackReg(unsigned Reg) { + return Reg == X86::RSP || Reg == X86::ESP || Reg == X86::SP; +} + +std::string FuncName(unsigned AccessSize, bool IsWrite) { + return std::string("__sanitizer_sanitize_") + (IsWrite ? "store" : "load") + + (utostr(AccessSize)); +} + +class X86AddressSanitizer : public X86AsmInstrumentation { +public: + X86AddressSanitizer(MCSubtargetInfo &sti) : STI(sti) {} + virtual ~X86AddressSanitizer() {} + + // X86AsmInstrumentation implementation: + virtual void InstrumentInstruction( + const MCInst &Inst, SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCContext &Ctx, MCStreamer &Out) override { + InstrumentMOV(Inst, Operands, Ctx, Out); + } + + // Should be implemented differently in x86_32 and x86_64 subclasses. + virtual void InstrumentMemOperandImpl(X86Operand *Op, unsigned AccessSize, + bool IsWrite, MCContext &Ctx, + MCStreamer &Out) = 0; + + void InstrumentMemOperand(MCParsedAsmOperand *Op, unsigned AccessSize, + bool IsWrite, MCContext &Ctx, MCStreamer &Out); + void InstrumentMOV(const MCInst &Inst, + SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCContext &Ctx, MCStreamer &Out); + void EmitInstruction(MCStreamer &Out, const MCInst &Inst) { + Out.EmitInstruction(Inst, STI); + } + +protected: + MCSubtargetInfo &STI; +}; + +void X86AddressSanitizer::InstrumentMemOperand( + MCParsedAsmOperand *Op, unsigned AccessSize, bool IsWrite, MCContext &Ctx, + MCStreamer &Out) { + assert(Op && Op->isMem() && "Op should be a memory operand."); + assert((AccessSize & (AccessSize - 1)) == 0 && AccessSize <= 16 && + "AccessSize should be a power of two, less or equal than 16."); + + X86Operand *MemOp = static_cast<X86Operand *>(Op); + // FIXME: get rid of this limitation. + if (IsStackReg(MemOp->getMemBaseReg()) || IsStackReg(MemOp->getMemIndexReg())) + return; + + InstrumentMemOperandImpl(MemOp, AccessSize, IsWrite, Ctx, Out); +} + +void X86AddressSanitizer::InstrumentMOV( + const MCInst &Inst, SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCContext &Ctx, MCStreamer &Out) { + // Access size in bytes. + unsigned AccessSize = 0; + unsigned long OpIx = Operands.size(); + switch (Inst.getOpcode()) { + case X86::MOV8mi: + case X86::MOV8mr: + AccessSize = 1; + OpIx = 2; + break; + case X86::MOV8rm: + AccessSize = 1; + OpIx = 1; + break; + case X86::MOV16mi: + case X86::MOV16mr: + AccessSize = 2; + OpIx = 2; + break; + case X86::MOV16rm: + AccessSize = 2; + OpIx = 1; + break; + case X86::MOV32mi: + case X86::MOV32mr: + AccessSize = 4; + OpIx = 2; + break; + case X86::MOV32rm: + AccessSize = 4; + OpIx = 1; + break; + case X86::MOV64mi32: + case X86::MOV64mr: + AccessSize = 8; + OpIx = 2; + break; + case X86::MOV64rm: + AccessSize = 8; + OpIx = 1; + break; + case X86::MOVAPDmr: + case X86::MOVAPSmr: + AccessSize = 16; + OpIx = 2; + break; + case X86::MOVAPDrm: + case X86::MOVAPSrm: + AccessSize = 16; + OpIx = 1; + break; + } + if (OpIx >= Operands.size()) + return; + + const bool IsWrite = (OpIx != 1); + InstrumentMemOperand(Operands[OpIx], AccessSize, IsWrite, Ctx, Out); +} + +class X86AddressSanitizer32 : public X86AddressSanitizer { +public: + X86AddressSanitizer32(MCSubtargetInfo &sti) : X86AddressSanitizer(sti) {} + virtual ~X86AddressSanitizer32() {} + + virtual void InstrumentMemOperandImpl(X86Operand *Op, unsigned AccessSize, + bool IsWrite, MCContext &Ctx, + MCStreamer &Out) override; +}; + +void X86AddressSanitizer32::InstrumentMemOperandImpl( + X86Operand *Op, unsigned AccessSize, bool IsWrite, MCContext &Ctx, + MCStreamer &Out) { + // FIXME: emit .cfi directives for correct stack unwinding. + EmitInstruction(Out, MCInstBuilder(X86::PUSH32r).addReg(X86::EAX)); + { + MCInst Inst; + Inst.setOpcode(X86::LEA32r); + Inst.addOperand(MCOperand::CreateReg(X86::EAX)); + Op->addMemOperands(Inst, 5); + EmitInstruction(Out, Inst); + } + EmitInstruction(Out, MCInstBuilder(X86::PUSH32r).addReg(X86::EAX)); + { + const std::string Func = FuncName(AccessSize, IsWrite); + const MCSymbol *FuncSym = Ctx.GetOrCreateSymbol(StringRef(Func)); + const MCSymbolRefExpr *FuncExpr = + MCSymbolRefExpr::Create(FuncSym, MCSymbolRefExpr::VK_PLT, Ctx); + EmitInstruction(Out, MCInstBuilder(X86::CALLpcrel32).addExpr(FuncExpr)); + } + EmitInstruction(Out, MCInstBuilder(X86::ADD32ri).addReg(X86::ESP) + .addReg(X86::ESP).addImm(4)); + EmitInstruction(Out, MCInstBuilder(X86::POP32r).addReg(X86::EAX)); +} + +class X86AddressSanitizer64 : public X86AddressSanitizer { +public: + X86AddressSanitizer64(MCSubtargetInfo &sti) : X86AddressSanitizer(sti) {} + virtual ~X86AddressSanitizer64() {} + + virtual void InstrumentMemOperandImpl(X86Operand *Op, unsigned AccessSize, + bool IsWrite, MCContext &Ctx, + MCStreamer &Out) override; +}; + +void X86AddressSanitizer64::InstrumentMemOperandImpl( + X86Operand *Op, unsigned AccessSize, bool IsWrite, MCContext &Ctx, + MCStreamer &Out) { + // FIXME: emit .cfi directives for correct stack unwinding. + // Set %rsp below current red zone (128 bytes wide) + EmitInstruction(Out, MCInstBuilder(X86::SUB64ri32).addReg(X86::RSP) + .addReg(X86::RSP).addImm(128)); + EmitInstruction(Out, MCInstBuilder(X86::PUSH64r).addReg(X86::RDI)); + { + MCInst Inst; + Inst.setOpcode(X86::LEA64r); + Inst.addOperand(MCOperand::CreateReg(X86::RDI)); + Op->addMemOperands(Inst, 5); + EmitInstruction(Out, Inst); + } + { + const std::string Func = FuncName(AccessSize, IsWrite); + const MCSymbol *FuncSym = Ctx.GetOrCreateSymbol(StringRef(Func)); + const MCSymbolRefExpr *FuncExpr = + MCSymbolRefExpr::Create(FuncSym, MCSymbolRefExpr::VK_PLT, Ctx); + EmitInstruction(Out, MCInstBuilder(X86::CALL64pcrel32).addExpr(FuncExpr)); + } + EmitInstruction(Out, MCInstBuilder(X86::POP64r).addReg(X86::RDI)); + EmitInstruction(Out, MCInstBuilder(X86::ADD64ri32).addReg(X86::RSP) + .addReg(X86::RSP).addImm(128)); +} + +} // End anonymous namespace + +X86AsmInstrumentation::X86AsmInstrumentation() {} +X86AsmInstrumentation::~X86AsmInstrumentation() {} + +void X86AsmInstrumentation::InstrumentInstruction( + const MCInst &Inst, SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCContext &Ctx, MCStreamer &Out) {} + +X86AsmInstrumentation *CreateX86AsmInstrumentation(MCSubtargetInfo &STI) { + if (ClAsanInstrumentInlineAssembly) { + if ((STI.getFeatureBits() & X86::Mode32Bit) != 0) + return new X86AddressSanitizer32(STI); + if ((STI.getFeatureBits() & X86::Mode64Bit) != 0) + return new X86AddressSanitizer64(STI); + } + return new X86AsmInstrumentation(); +} + +} // End llvm namespace diff --git a/lib/Target/X86/AsmParser/X86AsmInstrumentation.h b/lib/Target/X86/AsmParser/X86AsmInstrumentation.h new file mode 100644 index 0000000..c783a78 --- /dev/null +++ b/lib/Target/X86/AsmParser/X86AsmInstrumentation.h @@ -0,0 +1,46 @@ +//===- X86AsmInstrumentation.h - Instrument X86 inline assembly *- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef X86_ASM_INSTRUMENTATION_H +#define X86_ASM_INSTRUMENTATION_H + +#include "llvm/ADT/SmallVector.h" + +namespace llvm { + +class MCContext; +class MCInst; +class MCParsedAsmOperand; +class MCStreamer; +class MCSubtargetInfo; + +class X86AsmInstrumentation; + +X86AsmInstrumentation *CreateX86AsmInstrumentation(MCSubtargetInfo &STI); + +class X86AsmInstrumentation { +public: + virtual ~X86AsmInstrumentation(); + + // Instruments Inst. Should be called just before the original + // instruction is sent to Out. + virtual void InstrumentInstruction( + const MCInst &Inst, SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCContext &Ctx, MCStreamer &Out); + +protected: + friend X86AsmInstrumentation * + CreateX86AsmInstrumentation(MCSubtargetInfo &STI); + + X86AsmInstrumentation(); +}; + +} // End llvm namespace + +#endif // X86_ASM_INSTRUMENTATION_H diff --git a/lib/Target/X86/AsmParser/X86AsmParser.cpp b/lib/Target/X86/AsmParser/X86AsmParser.cpp index bc8f367..9eddc74 100644 --- a/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -8,6 +8,9 @@ //===----------------------------------------------------------------------===// #include "MCTargetDesc/X86BaseInfo.h" +#include "X86AsmInstrumentation.h" +#include "X86AsmParserCommon.h" +#include "X86Operand.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" @@ -28,19 +31,23 @@ #include "llvm/Support/SourceMgr.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" +#include <memory> using namespace llvm; namespace { -struct X86Operand; static const char OpPrecedence[] = { - 0, // IC_PLUS - 0, // IC_MINUS - 1, // IC_MULTIPLY - 1, // IC_DIVIDE - 2, // IC_RPAREN - 3, // IC_LPAREN + 0, // IC_OR + 1, // IC_AND + 2, // IC_LSHIFT + 2, // IC_RSHIFT + 3, // IC_PLUS + 3, // IC_MINUS + 4, // IC_MULTIPLY + 4, // IC_DIVIDE + 5, // IC_RPAREN + 6, // IC_LPAREN 0, // IC_IMM 0 // IC_REGISTER }; @@ -49,9 +56,20 @@ class X86AsmParser : public MCTargetAsmParser { MCSubtargetInfo &STI; MCAsmParser &Parser; ParseInstructionInfo *InstInfo; + std::unique_ptr<X86AsmInstrumentation> Instrumentation; private: + SMLoc consumeToken() { + SMLoc Result = Parser.getTok().getLoc(); + Parser.Lex(); + return Result; + } + enum InfixCalculatorTok { - IC_PLUS = 0, + IC_OR = 0, + IC_AND, + IC_LSHIFT, + IC_RSHIFT, + IC_PLUS, IC_MINUS, IC_MULTIPLY, IC_DIVIDE, @@ -176,6 +194,30 @@ private: Val = Op1.second / Op2.second; OperandStack.push_back(std::make_pair(IC_IMM, Val)); break; + case IC_OR: + assert (Op1.first == IC_IMM && Op2.first == IC_IMM && + "Or operation with an immediate and a register!"); + Val = Op1.second | Op2.second; + OperandStack.push_back(std::make_pair(IC_IMM, Val)); + break; + case IC_AND: + assert (Op1.first == IC_IMM && Op2.first == IC_IMM && + "And operation with an immediate and a register!"); + Val = Op1.second & Op2.second; + OperandStack.push_back(std::make_pair(IC_IMM, Val)); + break; + case IC_LSHIFT: + assert (Op1.first == IC_IMM && Op2.first == IC_IMM && + "Left shift operation with an immediate and a register!"); + Val = Op1.second << Op2.second; + OperandStack.push_back(std::make_pair(IC_IMM, Val)); + break; + case IC_RSHIFT: + assert (Op1.first == IC_IMM && Op2.first == IC_IMM && + "Right shift operation with an immediate and a register!"); + Val = Op1.second >> Op2.second; + OperandStack.push_back(std::make_pair(IC_IMM, Val)); + break; } } } @@ -185,6 +227,10 @@ private: }; enum IntelExprState { + IES_OR, + IES_AND, + IES_LSHIFT, + IES_RSHIFT, IES_PLUS, IES_MINUS, IES_MULTIPLY, @@ -231,6 +277,66 @@ private: return Info; } + void onOr() { + IntelExprState CurrState = State; + switch (State) { + default: + State = IES_ERROR; + break; + case IES_INTEGER: + case IES_RPAREN: + case IES_REGISTER: + State = IES_OR; + IC.pushOperator(IC_OR); + break; + } + PrevState = CurrState; + } + void onAnd() { + IntelExprState CurrState = State; + switch (State) { + default: + State = IES_ERROR; + break; + case IES_INTEGER: + case IES_RPAREN: + case IES_REGISTER: + State = IES_AND; + IC.pushOperator(IC_AND); + break; + } + PrevState = CurrState; + } + void onLShift() { + IntelExprState CurrState = State; + switch (State) { + default: + State = IES_ERROR; + break; + case IES_INTEGER: + case IES_RPAREN: + case IES_REGISTER: + State = IES_LSHIFT; + IC.pushOperator(IC_LSHIFT); + break; + } + PrevState = CurrState; + } + void onRShift() { + IntelExprState CurrState = State; + switch (State) { + default: + State = IES_ERROR; + break; + case IES_INTEGER: + case IES_RPAREN: + case IES_REGISTER: + State = IES_RSHIFT; + IC.pushOperator(IC_RSHIFT); + break; + } + PrevState = CurrState; + } void onPlus() { IntelExprState CurrState = State; switch (State) { @@ -337,7 +443,7 @@ private: break; } } - void onInteger(int64_t TmpInt) { + bool onInteger(int64_t TmpInt, StringRef &ErrMsg) { IntelExprState CurrState = State; switch (State) { default: @@ -345,6 +451,10 @@ private: break; case IES_PLUS: case IES_MINUS: + case IES_OR: + case IES_AND: + case IES_LSHIFT: + case IES_RSHIFT: case IES_DIVIDE: case IES_MULTIPLY: case IES_LPAREN: @@ -354,9 +464,15 @@ private: assert (!IndexReg && "IndexReg already set!"); IndexReg = TmpReg; Scale = TmpInt; + if(Scale != 1 && Scale != 2 && Scale != 4 && Scale != 8) { + ErrMsg = "scale factor in address must be 1, 2, 4 or 8"; + return true; + } // Get the scale and replace the 'Register * Scale' with '0'. IC.popOperator(); } else if ((PrevState == IES_PLUS || PrevState == IES_MINUS || + PrevState == IES_OR || PrevState == IES_AND || + PrevState == IES_LSHIFT || PrevState == IES_RSHIFT || PrevState == IES_MULTIPLY || PrevState == IES_DIVIDE || PrevState == IES_LPAREN || PrevState == IES_LBRAC) && CurrState == IES_MINUS) { @@ -369,6 +485,7 @@ private: break; } PrevState = CurrState; + return false; } void onStar() { PrevState = State; @@ -442,11 +559,17 @@ private: break; case IES_PLUS: case IES_MINUS: + case IES_OR: + case IES_AND: + case IES_LSHIFT: + case IES_RSHIFT: case IES_MULTIPLY: case IES_DIVIDE: case IES_LPAREN: // FIXME: We don't handle this type of unary minus, yet. if ((PrevState == IES_PLUS || PrevState == IES_MINUS || + PrevState == IES_OR || PrevState == IES_AND || + PrevState == IES_LSHIFT || PrevState == IES_RSHIFT || PrevState == IES_MULTIPLY || PrevState == IES_DIVIDE || PrevState == IES_LPAREN || PrevState == IES_LBRAC) && CurrState == IES_MINUS) { @@ -486,11 +609,20 @@ private: return Parser.Error(L, Msg, Ranges); } + bool ErrorAndEatStatement(SMLoc L, const Twine &Msg, + ArrayRef<SMRange> Ranges = None, + bool MatchingInlineAsm = false) { + Parser.eatToEndOfStatement(); + return Error(L, Msg, Ranges, MatchingInlineAsm); + } + X86Operand *ErrorOperand(SMLoc Loc, StringRef Msg) { Error(Loc, Msg); return 0; } + X86Operand *DefaultMemSIOperand(SMLoc Loc); + X86Operand *DefaultMemDIOperand(SMLoc Loc); X86Operand *ParseOperand(); X86Operand *ParseATTOperand(); X86Operand *ParseIntelOperand(); @@ -521,26 +653,47 @@ private: bool processInstruction(MCInst &Inst, const SmallVectorImpl<MCParsedAsmOperand*> &Ops); + /// Wrapper around MCStreamer::EmitInstruction(). Possibly adds + /// instrumentation around Inst. + void EmitInstruction(MCInst &Inst, + SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCStreamer &Out); + bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, SmallVectorImpl<MCParsedAsmOperand*> &Operands, MCStreamer &Out, unsigned &ErrorInfo, - bool MatchingInlineAsm); + bool MatchingInlineAsm) override; - /// isSrcOp - Returns true if operand is either (%rsi) or %ds:%(rsi) - /// in 64bit mode or (%esi) or %es:(%esi) in 32bit mode. - bool isSrcOp(X86Operand &Op); + /// doSrcDstMatch - Returns true if operands are matching in their + /// word size (%si and %di, %esi and %edi, etc.). Order depends on + /// the parsing mode (Intel vs. AT&T). + bool doSrcDstMatch(X86Operand &Op1, X86Operand &Op2); - /// isDstOp - Returns true if operand is either (%rdi) or %es:(%rdi) - /// in 64bit mode or (%edi) or %es:(%edi) in 32bit mode. - bool isDstOp(X86Operand &Op); + /// Parses AVX512 specific operand primitives: masked registers ({%k<NUM>}, {z}) + /// and memory broadcasting ({1to<NUM>}) primitives, updating Operands vector if required. + /// \return \c true if no parsing errors occurred, \c false otherwise. + bool HandleAVX512Operand(SmallVectorImpl<MCParsedAsmOperand*> &Operands, + const MCParsedAsmOperand &Op); bool is64BitMode() const { // FIXME: Can tablegen auto-generate this? return (STI.getFeatureBits() & X86::Mode64Bit) != 0; } - void SwitchMode() { - unsigned FB = ComputeAvailableFeatures(STI.ToggleFeature(X86::Mode64Bit)); + bool is32BitMode() const { + // FIXME: Can tablegen auto-generate this? + return (STI.getFeatureBits() & X86::Mode32Bit) != 0; + } + bool is16BitMode() const { + // FIXME: Can tablegen auto-generate this? + return (STI.getFeatureBits() & X86::Mode16Bit) != 0; + } + void SwitchMode(uint64_t mode) { + uint64_t oldMode = STI.getFeatureBits() & + (X86::Mode64Bit | X86::Mode32Bit | X86::Mode16Bit); + unsigned FB = ComputeAvailableFeatures(STI.ToggleFeature(oldMode | mode)); setAvailableFeatures(FB); + assert(mode == (STI.getFeatureBits() & + (X86::Mode64Bit | X86::Mode32Bit | X86::Mode16Bit))); } bool isParsingIntelSyntax() { @@ -562,14 +715,15 @@ public: // Initialize the set of available features. setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits())); + Instrumentation.reset(CreateX86AsmInstrumentation(STI)); } - virtual bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc); + bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override; - virtual bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name, - SMLoc NameLoc, - SmallVectorImpl<MCParsedAsmOperand*> &Operands); + bool + ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, + SmallVectorImpl<MCParsedAsmOperand*> &Operands) override; - virtual bool ParseDirective(AsmToken DirectiveID); + bool ParseDirective(AsmToken DirectiveID) override; }; } // end anonymous namespace @@ -580,470 +734,63 @@ static unsigned MatchRegisterName(StringRef Name); /// } -static bool isImmSExti16i8Value(uint64_t Value) { - return (( Value <= 0x000000000000007FULL)|| - (0x000000000000FF80ULL <= Value && Value <= 0x000000000000FFFFULL)|| - (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); -} - -static bool isImmSExti32i8Value(uint64_t Value) { - return (( Value <= 0x000000000000007FULL)|| - (0x00000000FFFFFF80ULL <= Value && Value <= 0x00000000FFFFFFFFULL)|| - (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); -} - -static bool isImmZExtu32u8Value(uint64_t Value) { - return (Value <= 0x00000000000000FFULL); -} - -static bool isImmSExti64i8Value(uint64_t Value) { - return (( Value <= 0x000000000000007FULL)|| - (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); -} - -static bool isImmSExti64i32Value(uint64_t Value) { - return (( Value <= 0x000000007FFFFFFFULL)|| - (0xFFFFFFFF80000000ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); -} -namespace { - -/// X86Operand - Instances of this class represent a parsed X86 machine -/// instruction. -struct X86Operand : public MCParsedAsmOperand { - enum KindTy { - Token, - Register, - Immediate, - Memory - } Kind; - - SMLoc StartLoc, EndLoc; - SMLoc OffsetOfLoc; - StringRef SymName; - void *OpDecl; - bool AddressOf; - - struct TokOp { - const char *Data; - unsigned Length; - }; - - struct RegOp { - unsigned RegNo; - }; - - struct ImmOp { - const MCExpr *Val; - }; - - struct MemOp { - unsigned SegReg; - const MCExpr *Disp; - unsigned BaseReg; - unsigned IndexReg; - unsigned Scale; - unsigned Size; - }; - - union { - struct TokOp Tok; - struct RegOp Reg; - struct ImmOp Imm; - struct MemOp Mem; - }; - - X86Operand(KindTy K, SMLoc Start, SMLoc End) - : Kind(K), StartLoc(Start), EndLoc(End) {} - - StringRef getSymName() { return SymName; } - void *getOpDecl() { return OpDecl; } - - /// getStartLoc - Get the location of the first token of this operand. - SMLoc getStartLoc() const { return StartLoc; } - /// getEndLoc - Get the location of the last token of this operand. - SMLoc getEndLoc() const { return EndLoc; } - /// getLocRange - Get the range between the first and last token of this - /// operand. - SMRange getLocRange() const { return SMRange(StartLoc, EndLoc); } - /// getOffsetOfLoc - Get the location of the offset operator. - SMLoc getOffsetOfLoc() const { return OffsetOfLoc; } - - virtual void print(raw_ostream &OS) const {} - - StringRef getToken() const { - assert(Kind == Token && "Invalid access!"); - return StringRef(Tok.Data, Tok.Length); - } - void setTokenValue(StringRef Value) { - assert(Kind == Token && "Invalid access!"); - Tok.Data = Value.data(); - Tok.Length = Value.size(); - } - - unsigned getReg() const { - assert(Kind == Register && "Invalid access!"); - return Reg.RegNo; - } - - const MCExpr *getImm() const { - assert(Kind == Immediate && "Invalid access!"); - return Imm.Val; - } - - const MCExpr *getMemDisp() const { - assert(Kind == Memory && "Invalid access!"); - return Mem.Disp; - } - unsigned getMemSegReg() const { - assert(Kind == Memory && "Invalid access!"); - return Mem.SegReg; - } - unsigned getMemBaseReg() const { - assert(Kind == Memory && "Invalid access!"); - return Mem.BaseReg; - } - unsigned getMemIndexReg() const { - assert(Kind == Memory && "Invalid access!"); - return Mem.IndexReg; - } - unsigned getMemScale() const { - assert(Kind == Memory && "Invalid access!"); - return Mem.Scale; - } - - bool isToken() const {return Kind == Token; } - - bool isImm() const { return Kind == Immediate; } - - bool isImmSExti16i8() const { - if (!isImm()) - return false; - - // If this isn't a constant expr, just assume it fits and let relaxation - // handle it. - const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); - if (!CE) - return true; - - // Otherwise, check the value is in a range that makes sense for this - // extension. - return isImmSExti16i8Value(CE->getValue()); - } - bool isImmSExti32i8() const { - if (!isImm()) - return false; - - // If this isn't a constant expr, just assume it fits and let relaxation - // handle it. - const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); - if (!CE) - return true; - - // Otherwise, check the value is in a range that makes sense for this - // extension. - return isImmSExti32i8Value(CE->getValue()); - } - bool isImmZExtu32u8() const { - if (!isImm()) - return false; - - // If this isn't a constant expr, just assume it fits and let relaxation - // handle it. - const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); - if (!CE) - return true; - - // Otherwise, check the value is in a range that makes sense for this - // extension. - return isImmZExtu32u8Value(CE->getValue()); - } - bool isImmSExti64i8() const { - if (!isImm()) - return false; - - // If this isn't a constant expr, just assume it fits and let relaxation - // handle it. - const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); - if (!CE) +static bool CheckBaseRegAndIndexReg(unsigned BaseReg, unsigned IndexReg, + StringRef &ErrMsg) { + // If we have both a base register and an index register make sure they are + // both 64-bit or 32-bit registers. + // To support VSIB, IndexReg can be 128-bit or 256-bit registers. + if (BaseReg != 0 && IndexReg != 0) { + if (X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg) && + (X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg) || + X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg)) && + IndexReg != X86::RIZ) { + ErrMsg = "base register is 64-bit, but index register is not"; return true; - - // Otherwise, check the value is in a range that makes sense for this - // extension. - return isImmSExti64i8Value(CE->getValue()); - } - bool isImmSExti64i32() const { - if (!isImm()) - return false; - - // If this isn't a constant expr, just assume it fits and let relaxation - // handle it. - const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); - if (!CE) + } + if (X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg) && + (X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg) || + X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg)) && + IndexReg != X86::EIZ){ + ErrMsg = "base register is 32-bit, but index register is not"; return true; - - // Otherwise, check the value is in a range that makes sense for this - // extension. - return isImmSExti64i32Value(CE->getValue()); - } - - bool isOffsetOf() const { - return OffsetOfLoc.getPointer(); - } - - bool needAddressOf() const { - return AddressOf; - } - - bool isMem() const { return Kind == Memory; } - bool isMem8() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 8); - } - bool isMem16() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 16); - } - bool isMem32() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 32); - } - bool isMem64() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 64); - } - bool isMem80() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 80); - } - bool isMem128() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 128); - } - bool isMem256() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 256); - } - bool isMem512() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 512); - } - - bool isMemVX32() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 32) && - getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM15; - } - bool isMemVY32() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 32) && - getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM15; - } - bool isMemVX64() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 64) && - getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM15; - } - bool isMemVY64() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 64) && - getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM15; - } - bool isMemVZ32() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 32) && - getMemIndexReg() >= X86::ZMM0 && getMemIndexReg() <= X86::ZMM31; - } - bool isMemVZ64() const { - return Kind == Memory && (!Mem.Size || Mem.Size == 64) && - getMemIndexReg() >= X86::ZMM0 && getMemIndexReg() <= X86::ZMM31; - } - - bool isAbsMem() const { - return Kind == Memory && !getMemSegReg() && !getMemBaseReg() && - !getMemIndexReg() && getMemScale() == 1; - } - - bool isMemOffs8() const { - return Kind == Memory && !getMemSegReg() && !getMemBaseReg() && - !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 8); - } - bool isMemOffs16() const { - return Kind == Memory && !getMemSegReg() && !getMemBaseReg() && - !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 16); - } - bool isMemOffs32() const { - return Kind == Memory && !getMemSegReg() && !getMemBaseReg() && - !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 32); - } - bool isMemOffs64() const { - return Kind == Memory && !getMemSegReg() && !getMemBaseReg() && - !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 64); - } - - bool isReg() const { return Kind == Register; } - - bool isGR32orGR64() const { - return Kind == Register && - (X86MCRegisterClasses[X86::GR32RegClassID].contains(getReg()) || - X86MCRegisterClasses[X86::GR64RegClassID].contains(getReg())); - } - - void addExpr(MCInst &Inst, const MCExpr *Expr) const { - // Add as immediates when possible. - if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr)) - Inst.addOperand(MCOperand::CreateImm(CE->getValue())); - else - Inst.addOperand(MCOperand::CreateExpr(Expr)); - } - - void addRegOperands(MCInst &Inst, unsigned N) const { - assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::CreateReg(getReg())); - } - - static unsigned getGR32FromGR64(unsigned RegNo) { - switch (RegNo) { - default: llvm_unreachable("Unexpected register"); - case X86::RAX: return X86::EAX; - case X86::RCX: return X86::ECX; - case X86::RDX: return X86::EDX; - case X86::RBX: return X86::EBX; - case X86::RBP: return X86::EBP; - case X86::RSP: return X86::ESP; - case X86::RSI: return X86::ESI; - case X86::RDI: return X86::EDI; - case X86::R8: return X86::R8D; - case X86::R9: return X86::R9D; - case X86::R10: return X86::R10D; - case X86::R11: return X86::R11D; - case X86::R12: return X86::R12D; - case X86::R13: return X86::R13D; - case X86::R14: return X86::R14D; - case X86::R15: return X86::R15D; - case X86::RIP: return X86::EIP; - } - } - - void addGR32orGR64Operands(MCInst &Inst, unsigned N) const { - assert(N == 1 && "Invalid number of operands!"); - unsigned RegNo = getReg(); - if (X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo)) - RegNo = getGR32FromGR64(RegNo); - Inst.addOperand(MCOperand::CreateReg(RegNo)); - } - - void addImmOperands(MCInst &Inst, unsigned N) const { - assert(N == 1 && "Invalid number of operands!"); - addExpr(Inst, getImm()); - } - - void addMemOperands(MCInst &Inst, unsigned N) const { - assert((N == 5) && "Invalid number of operands!"); - Inst.addOperand(MCOperand::CreateReg(getMemBaseReg())); - Inst.addOperand(MCOperand::CreateImm(getMemScale())); - Inst.addOperand(MCOperand::CreateReg(getMemIndexReg())); - addExpr(Inst, getMemDisp()); - Inst.addOperand(MCOperand::CreateReg(getMemSegReg())); - } - - void addAbsMemOperands(MCInst &Inst, unsigned N) const { - assert((N == 1) && "Invalid number of operands!"); - // Add as immediates when possible. - if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemDisp())) - Inst.addOperand(MCOperand::CreateImm(CE->getValue())); - else - Inst.addOperand(MCOperand::CreateExpr(getMemDisp())); - } - - void addMemOffsOperands(MCInst &Inst, unsigned N) const { - assert((N == 1) && "Invalid number of operands!"); - // Add as immediates when possible. - if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemDisp())) - Inst.addOperand(MCOperand::CreateImm(CE->getValue())); - else - Inst.addOperand(MCOperand::CreateExpr(getMemDisp())); - } - - static X86Operand *CreateToken(StringRef Str, SMLoc Loc) { - SMLoc EndLoc = SMLoc::getFromPointer(Loc.getPointer() + Str.size()); - X86Operand *Res = new X86Operand(Token, Loc, EndLoc); - Res->Tok.Data = Str.data(); - Res->Tok.Length = Str.size(); - return Res; - } - - static X86Operand *CreateReg(unsigned RegNo, SMLoc StartLoc, SMLoc EndLoc, - bool AddressOf = false, - SMLoc OffsetOfLoc = SMLoc(), - StringRef SymName = StringRef(), - void *OpDecl = 0) { - X86Operand *Res = new X86Operand(Register, StartLoc, EndLoc); - Res->Reg.RegNo = RegNo; - Res->AddressOf = AddressOf; - Res->OffsetOfLoc = OffsetOfLoc; - Res->SymName = SymName; - Res->OpDecl = OpDecl; - return Res; - } - - static X86Operand *CreateImm(const MCExpr *Val, SMLoc StartLoc, SMLoc EndLoc){ - X86Operand *Res = new X86Operand(Immediate, StartLoc, EndLoc); - Res->Imm.Val = Val; - return Res; - } - - /// Create an absolute memory operand. - static X86Operand *CreateMem(const MCExpr *Disp, SMLoc StartLoc, SMLoc EndLoc, - unsigned Size = 0, StringRef SymName = StringRef(), - void *OpDecl = 0) { - X86Operand *Res = new X86Operand(Memory, StartLoc, EndLoc); - Res->Mem.SegReg = 0; - Res->Mem.Disp = Disp; - Res->Mem.BaseReg = 0; - Res->Mem.IndexReg = 0; - Res->Mem.Scale = 1; - Res->Mem.Size = Size; - Res->SymName = SymName; - Res->OpDecl = OpDecl; - Res->AddressOf = false; - return Res; - } - - /// Create a generalized memory operand. - static X86Operand *CreateMem(unsigned SegReg, const MCExpr *Disp, - unsigned BaseReg, unsigned IndexReg, - unsigned Scale, SMLoc StartLoc, SMLoc EndLoc, - unsigned Size = 0, - StringRef SymName = StringRef(), - void *OpDecl = 0) { - // We should never just have a displacement, that should be parsed as an - // absolute memory operand. - assert((SegReg || BaseReg || IndexReg) && "Invalid memory operand!"); - - // The scale should always be one of {1,2,4,8}. - assert(((Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8)) && - "Invalid scale!"); - X86Operand *Res = new X86Operand(Memory, StartLoc, EndLoc); - Res->Mem.SegReg = SegReg; - Res->Mem.Disp = Disp; - Res->Mem.BaseReg = BaseReg; - Res->Mem.IndexReg = IndexReg; - Res->Mem.Scale = Scale; - Res->Mem.Size = Size; - Res->SymName = SymName; - Res->OpDecl = OpDecl; - Res->AddressOf = false; - return Res; + } + if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) { + if (X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg) || + X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg)) { + ErrMsg = "base register is 16-bit, but index register is not"; + return true; + } + if (((BaseReg == X86::BX || BaseReg == X86::BP) && + IndexReg != X86::SI && IndexReg != X86::DI) || + ((BaseReg == X86::SI || BaseReg == X86::DI) && + IndexReg != X86::BX && IndexReg != X86::BP)) { + ErrMsg = "invalid 16-bit base/index register combination"; + return true; + } + } } -}; - -} // end anonymous namespace. - -bool X86AsmParser::isSrcOp(X86Operand &Op) { - unsigned basereg = is64BitMode() ? X86::RSI : X86::ESI; - - return (Op.isMem() && - (Op.Mem.SegReg == 0 || Op.Mem.SegReg == X86::DS) && - isa<MCConstantExpr>(Op.Mem.Disp) && - cast<MCConstantExpr>(Op.Mem.Disp)->getValue() == 0 && - Op.Mem.BaseReg == basereg && Op.Mem.IndexReg == 0); + return false; } -bool X86AsmParser::isDstOp(X86Operand &Op) { - unsigned basereg = is64BitMode() ? X86::RDI : X86::EDI; +bool X86AsmParser::doSrcDstMatch(X86Operand &Op1, X86Operand &Op2) +{ + // Return true and let a normal complaint about bogus operands happen. + if (!Op1.isMem() || !Op2.isMem()) + return true; - return Op.isMem() && - (Op.Mem.SegReg == 0 || Op.Mem.SegReg == X86::ES) && - isa<MCConstantExpr>(Op.Mem.Disp) && - cast<MCConstantExpr>(Op.Mem.Disp)->getValue() == 0 && - Op.Mem.BaseReg == basereg && Op.Mem.IndexReg == 0; + // Actually these might be the other way round if Intel syntax is + // being used. It doesn't matter. + unsigned diReg = Op1.Mem.BaseReg; + unsigned siReg = Op2.Mem.BaseReg; + + if (X86MCRegisterClasses[X86::GR16RegClassID].contains(siReg)) + return X86MCRegisterClasses[X86::GR16RegClassID].contains(diReg); + if (X86MCRegisterClasses[X86::GR32RegClassID].contains(siReg)) + return X86MCRegisterClasses[X86::GR32RegClassID].contains(diReg); + if (X86MCRegisterClasses[X86::GR64RegClassID].contains(siReg)) + return X86MCRegisterClasses[X86::GR64RegClassID].contains(diReg); + // Again, return true and let another error happen. + return true; } bool X86AsmParser::ParseRegister(unsigned &RegNo, @@ -1073,7 +820,7 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo, RegNo = MatchRegisterName(Tok.getString().lower()); if (!is64BitMode()) { - // FIXME: This should be done using Requires<In32BitMode> and + // FIXME: This should be done using Requires<Not64BitMode> and // Requires<In64BitMode> so "eiz" usage in 64-bit instructions can be also // checked. // FIXME: Check AH, CH, DH, BH cannot be used in an instruction requiring a @@ -1155,6 +902,22 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo, return false; } +X86Operand *X86AsmParser::DefaultMemSIOperand(SMLoc Loc) { + unsigned basereg = + is64BitMode() ? X86::RSI : (is32BitMode() ? X86::ESI : X86::SI); + const MCExpr *Disp = MCConstantExpr::Create(0, getContext()); + return X86Operand::CreateMem(/*SegReg=*/0, Disp, /*BaseReg=*/basereg, + /*IndexReg=*/0, /*Scale=*/1, Loc, Loc, 0); +} + +X86Operand *X86AsmParser::DefaultMemDIOperand(SMLoc Loc) { + unsigned basereg = + is64BitMode() ? X86::RDI : (is32BitMode() ? X86::EDI : X86::DI); + const MCExpr *Disp = MCConstantExpr::Create(0, getContext()); + return X86Operand::CreateMem(/*SegReg=*/0, Disp, /*BaseReg=*/basereg, + /*IndexReg=*/0, /*Scale=*/1, Loc, Loc, 0); +} + X86Operand *X86AsmParser::ParseOperand() { if (isParsingIntelSyntax()) return ParseIntelOperand(); @@ -1171,6 +934,8 @@ static unsigned getIntelMemOperandSize(StringRef OpStr) { .Cases("XWORD", "xword", 80) .Cases("XMMWORD", "xmmword", 128) .Cases("YMMWORD", "ymmword", 256) + .Cases("ZMMWORD", "zmmword", 512) + .Cases("OPAQUE", "opaque", -1U) // needs to be non-zero, but doesn't matter .Default(0); return Size; } @@ -1181,16 +946,24 @@ X86AsmParser::CreateMemForInlineAsm(unsigned SegReg, const MCExpr *Disp, unsigned Scale, SMLoc Start, SMLoc End, unsigned Size, StringRef Identifier, InlineAsmIdentifierInfo &Info){ - if (isa<MCSymbolRefExpr>(Disp)) { - // If this is not a VarDecl then assume it is a FuncDecl or some other label - // reference. We need an 'r' constraint here, so we need to create register - // operand to ensure proper matching. Just pick a GPR based on the size of - // a pointer. - if (!Info.IsVarDecl) { - unsigned RegNo = is64BitMode() ? X86::RBX : X86::EBX; - return X86Operand::CreateReg(RegNo, Start, End, /*AddressOf=*/true, - SMLoc(), Identifier, Info.OpDecl); - } + // If this is not a VarDecl then assume it is a FuncDecl or some other label + // reference. We need an 'r' constraint here, so we need to create register + // operand to ensure proper matching. Just pick a GPR based on the size of + // a pointer. + if (isa<MCSymbolRefExpr>(Disp) && !Info.IsVarDecl) { + unsigned RegNo = + is64BitMode() ? X86::RBX : (is32BitMode() ? X86::EBX : X86::BX); + return X86Operand::CreateReg(RegNo, Start, End, /*AddressOf=*/true, + SMLoc(), Identifier, Info.OpDecl); + } + + // We either have a direct symbol reference, or an offset from a symbol. The + // parser always puts the symbol on the LHS, so look there for size + // calculation purposes. + const MCBinaryExpr *BinOp = dyn_cast<MCBinaryExpr>(Disp); + bool IsSymRef = + isa<MCSymbolRefExpr>(BinOp ? BinOp->getLHS() : Disp); + if (IsSymRef) { if (!Size) { Size = Info.Type * 8; // Size is in terms of bits in this context. if (Size) @@ -1312,10 +1085,15 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) { if (getParser().parsePrimaryExpr(Val, End)) return Error(Tok.getLoc(), "Unexpected identifier!"); } else { - InlineAsmIdentifierInfo &Info = SM.getIdentifierInfo(); - if (ParseIntelIdentifier(Val, Identifier, Info, - /*Unevaluated=*/false, End)) - return true; + // This is a dot operator, not an adjacent identifier. + if (Identifier.find('.') != StringRef::npos) { + return false; + } else { + InlineAsmIdentifierInfo &Info = SM.getIdentifierInfo(); + if (ParseIntelIdentifier(Val, Identifier, Info, + /*Unevaluated=*/false, End)) + return true; + } } SM.onIdentifierExpr(Val, Identifier); UpdateLocLex = false; @@ -1323,16 +1101,49 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) { } return Error(Tok.getLoc(), "Unexpected identifier!"); } - case AsmToken::Integer: + case AsmToken::Integer: { + StringRef ErrMsg; if (isParsingInlineAsm() && SM.getAddImmPrefix()) InstInfo->AsmRewrites->push_back(AsmRewrite(AOK_ImmPrefix, Tok.getLoc())); - SM.onInteger(Tok.getIntVal()); + // Look for 'b' or 'f' following an Integer as a directional label + SMLoc Loc = getTok().getLoc(); + int64_t IntVal = getTok().getIntVal(); + End = consumeToken(); + UpdateLocLex = false; + if (getLexer().getKind() == AsmToken::Identifier) { + StringRef IDVal = getTok().getString(); + if (IDVal == "f" || IDVal == "b") { + MCSymbol *Sym = + getContext().GetDirectionalLocalSymbol(IntVal, IDVal == "b"); + MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None; + const MCExpr *Val = + MCSymbolRefExpr::Create(Sym, Variant, getContext()); + if (IDVal == "b" && Sym->isUndefined()) + return Error(Loc, "invalid reference to undefined symbol"); + StringRef Identifier = Sym->getName(); + SM.onIdentifierExpr(Val, Identifier); + End = consumeToken(); + } else { + if (SM.onInteger(IntVal, ErrMsg)) + return Error(Loc, ErrMsg); + } + } else { + if (SM.onInteger(IntVal, ErrMsg)) + return Error(Loc, ErrMsg); + } break; + } case AsmToken::Plus: SM.onPlus(); break; case AsmToken::Minus: SM.onMinus(); break; case AsmToken::Star: SM.onStar(); break; case AsmToken::Slash: SM.onDivide(); break; + case AsmToken::Pipe: SM.onOr(); break; + case AsmToken::Amp: SM.onAnd(); break; + case AsmToken::LessLess: + SM.onLShift(); break; + case AsmToken::GreaterGreater: + SM.onRShift(); break; case AsmToken::LBrac: SM.onLBrac(); break; case AsmToken::RBrac: SM.onRBrac(); break; case AsmToken::LParen: SM.onLParen(); break; @@ -1341,10 +1152,8 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) { if (SM.hadError()) return Error(Tok.getLoc(), "unknown token in expression"); - if (!Done && UpdateLocLex) { - End = Tok.getLoc(); - Parser.Lex(); // Consume the token. - } + if (!Done && UpdateLocLex) + End = consumeToken(); } return false; } @@ -1366,7 +1175,7 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg, SMLoc Start, if (ParseIntelExpression(SM, End)) return 0; - const MCExpr *Disp; + const MCExpr *Disp = 0; if (const MCExpr *Sym = SM.getSym()) { // A symbolic displacement. Disp = Sym; @@ -1374,13 +1183,20 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg, SMLoc Start, RewriteIntelBracExpression(InstInfo->AsmRewrites, SM.getSymName(), ImmDisp, SM.getImm(), BracLoc, StartInBrac, End); - } else { - // An immediate displacement only. - Disp = MCConstantExpr::Create(SM.getImm(), getContext()); } - // Parse the dot operator (e.g., [ebx].foo.bar). - if (Tok.getString().startswith(".")) { + if (SM.getImm() || !Disp) { + const MCExpr *Imm = MCConstantExpr::Create(SM.getImm(), getContext()); + if (Disp) + Disp = MCBinaryExpr::CreateAdd(Disp, Imm, getContext()); + else + Disp = Imm; // An immediate displacement only. + } + + // Parse struct field access. Intel requires a dot, but MSVC doesn't. MSVC + // will in fact do global lookup the field name inside all global typedefs, + // but we don't emulate that. + if (Tok.getString().find('.') != StringRef::npos) { const MCExpr *NewDisp; if (ParseIntelDotOperator(Disp, NewDisp)) return 0; @@ -1401,6 +1217,11 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg, SMLoc Start, else return X86Operand::CreateMem(SegReg, Disp, 0, 0, 1, Start, End, Size); } + StringRef ErrMsg; + if (CheckBaseRegAndIndexReg(BaseReg, IndexReg, ErrMsg)) { + Error(StartInBrac, ErrMsg); + return 0; + } return X86Operand::CreateMem(SegReg, Disp, BaseReg, IndexReg, Scale, Start, End, Size); } @@ -1502,6 +1323,7 @@ X86Operand *X86AsmParser::ParseIntelMemOperand(int64_t ImmDisp, SMLoc Start, // Parse ImmDisp [ BaseReg + Scale*IndexReg + Disp ]. if (getLexer().is(AsmToken::LBrac)) return ParseIntelBracExpression(/*SegReg=*/0, Start, ImmDisp, Size); + assert(ImmDisp == 0); const MCExpr *Val; if (!isParsingInlineAsm()) { @@ -1516,8 +1338,39 @@ X86Operand *X86AsmParser::ParseIntelMemOperand(int64_t ImmDisp, SMLoc Start, if (ParseIntelIdentifier(Val, Identifier, Info, /*Unevaluated=*/false, End)) return 0; - return CreateMemForInlineAsm(/*SegReg=*/0, Val, /*BaseReg=*/0, /*IndexReg=*/0, - /*Scale=*/1, Start, End, Size, Identifier, Info); + + if (!getLexer().is(AsmToken::LBrac)) + return CreateMemForInlineAsm(/*SegReg=*/0, Val, /*BaseReg=*/0, /*IndexReg=*/0, + /*Scale=*/1, Start, End, Size, Identifier, Info); + + Parser.Lex(); // Eat '[' + + // Parse Identifier [ ImmDisp ] + IntelExprStateMachine SM(/*ImmDisp=*/0, /*StopOnLBrac=*/true, + /*AddImmPrefix=*/false); + if (ParseIntelExpression(SM, End)) + return 0; + + if (SM.getSym()) { + Error(Start, "cannot use more than one symbol in memory operand"); + return 0; + } + if (SM.getBaseReg()) { + Error(Start, "cannot use base register with variable reference"); + return 0; + } + if (SM.getIndexReg()) { + Error(Start, "cannot use index register with variable reference"); + return 0; + } + + const MCExpr *Disp = MCConstantExpr::Create(SM.getImm(), getContext()); + // BaseReg is non-zero to avoid assertions. In the context of inline asm, + // we're pointing to a local variable in memory, so the base register is + // really the frame or stack pointer. + return X86Operand::CreateMem(/*SegReg=*/0, Disp, /*BaseReg=*/1, /*IndexReg=*/0, + /*Scale=*/1, Start, End, Size, Identifier, + Info.OpDecl); } /// Parse the '.' operator. @@ -1532,8 +1385,10 @@ bool X86AsmParser::ParseIntelDotOperator(const MCExpr *Disp, else return Error(Tok.getLoc(), "Non-constant offsets are not supported!"); - // Drop the '.'. - StringRef DotDispStr = Tok.getString().drop_front(1); + // Drop the optional '.'. + StringRef DotDispStr = Tok.getString(); + if (DotDispStr.startswith(".")) + DotDispStr = DotDispStr.drop_front(1); // .Imm gets lexed as a real. if (Tok.is(AsmToken::Real)) { @@ -1583,7 +1438,8 @@ X86Operand *X86AsmParser::ParseIntelOffsetOfOperator() { // The offset operator will have an 'r' constraint, thus we need to create // register operand to ensure proper matching. Just pick a GPR based on // the size of a pointer. - unsigned RegNo = is64BitMode() ? X86::RBX : X86::EBX; + unsigned RegNo = + is64BitMode() ? X86::RBX : (is32BitMode() ? X86::EBX : X86::BX); return X86Operand::CreateReg(RegNo, Start, End, /*GetAddress=*/true, OffsetOfLoc, Identifier, Info.OpDecl); } @@ -1680,6 +1536,13 @@ X86Operand *X86AsmParser::ParseIntelOperand() { } if (getLexer().isNot(AsmToken::LBrac)) { + // If a directional label (ie. 1f or 2b) was parsed above from + // ParseIntelExpression() then SM.getSym() was set to a pointer to + // to the MCExpr with the directional local symbol and this is a + // memory operand not an immediate operand. + if (SM.getSym()) + return X86Operand::CreateMem(SM.getSym(), Start, End, Size); + const MCExpr *ImmExpr = MCConstantExpr::Create(Imm, getContext()); return X86Operand::CreateImm(ImmExpr, Start, End); } @@ -1744,6 +1607,73 @@ X86Operand *X86AsmParser::ParseATTOperand() { } } +bool +X86AsmParser::HandleAVX512Operand(SmallVectorImpl<MCParsedAsmOperand*> &Operands, + const MCParsedAsmOperand &Op) { + if(STI.getFeatureBits() & X86::FeatureAVX512) { + if (getLexer().is(AsmToken::LCurly)) { + // Eat "{" and mark the current place. + const SMLoc consumedToken = consumeToken(); + // Distinguish {1to<NUM>} from {%k<NUM>}. + if(getLexer().is(AsmToken::Integer)) { + // Parse memory broadcasting ({1to<NUM>}). + if (getLexer().getTok().getIntVal() != 1) + return !ErrorAndEatStatement(getLexer().getLoc(), + "Expected 1to<NUM> at this point"); + Parser.Lex(); // Eat "1" of 1to8 + if (!getLexer().is(AsmToken::Identifier) || + !getLexer().getTok().getIdentifier().startswith("to")) + return !ErrorAndEatStatement(getLexer().getLoc(), + "Expected 1to<NUM> at this point"); + // Recognize only reasonable suffixes. + const char *BroadcastPrimitive = + StringSwitch<const char*>(getLexer().getTok().getIdentifier()) + .Case("to8", "{1to8}") + .Case("to16", "{1to16}") + .Default(0); + if (!BroadcastPrimitive) + return !ErrorAndEatStatement(getLexer().getLoc(), + "Invalid memory broadcast primitive."); + Parser.Lex(); // Eat "toN" of 1toN + if (!getLexer().is(AsmToken::RCurly)) + return !ErrorAndEatStatement(getLexer().getLoc(), + "Expected } at this point"); + Parser.Lex(); // Eat "}" + Operands.push_back(X86Operand::CreateToken(BroadcastPrimitive, + consumedToken)); + // No AVX512 specific primitives can pass + // after memory broadcasting, so return. + return true; + } else { + // Parse mask register {%k1} + Operands.push_back(X86Operand::CreateToken("{", consumedToken)); + if (X86Operand *Op = ParseOperand()) { + Operands.push_back(Op); + if (!getLexer().is(AsmToken::RCurly)) + return !ErrorAndEatStatement(getLexer().getLoc(), + "Expected } at this point"); + Operands.push_back(X86Operand::CreateToken("}", consumeToken())); + + // Parse "zeroing non-masked" semantic {z} + if (getLexer().is(AsmToken::LCurly)) { + Operands.push_back(X86Operand::CreateToken("{z}", consumeToken())); + if (!getLexer().is(AsmToken::Identifier) || + getLexer().getTok().getIdentifier() != "z") + return !ErrorAndEatStatement(getLexer().getLoc(), + "Expected z at this point"); + Parser.Lex(); // Eat the z + if (!getLexer().is(AsmToken::RCurly)) + return !ErrorAndEatStatement(getLexer().getLoc(), + "Expected } at this point"); + Parser.Lex(); // Eat the } + } + } + } + } + } + return true; +} + /// ParseMemOperand: segment: disp(basereg, indexreg, scale). The '%ds:' prefix /// has already been parsed if present. X86Operand *X86AsmParser::ParseMemOperand(unsigned SegReg, SMLoc MemStart) { @@ -1801,10 +1731,11 @@ X86Operand *X86AsmParser::ParseMemOperand(unsigned SegReg, SMLoc MemStart) { // If we reached here, then we just ate the ( of the memory operand. Process // the rest of the memory operand. unsigned BaseReg = 0, IndexReg = 0, Scale = 1; - SMLoc IndexLoc; + SMLoc IndexLoc, BaseLoc; if (getLexer().is(AsmToken::Percent)) { SMLoc StartLoc, EndLoc; + BaseLoc = Parser.getTok().getLoc(); if (ParseRegister(BaseReg, StartLoc, EndLoc)) return 0; if (BaseReg == X86::EIZ || BaseReg == X86::RIZ) { Error(StartLoc, "eiz and riz can only be used as index registers", @@ -1847,6 +1778,11 @@ X86Operand *X86AsmParser::ParseMemOperand(unsigned SegReg, SMLoc MemStart) { } // Validate the scale amount. + if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg) && + ScaleVal != 1) { + Error(Loc, "scale factor in 16-bit address must be 1"); + return 0; + } if (ScaleVal != 1 && ScaleVal != 2 && ScaleVal != 4 && ScaleVal != 8){ Error(Loc, "scale factor in address must be 1, 2, 4 or 8"); return 0; @@ -1877,24 +1813,26 @@ X86Operand *X86AsmParser::ParseMemOperand(unsigned SegReg, SMLoc MemStart) { SMLoc MemEnd = Parser.getTok().getEndLoc(); Parser.Lex(); // Eat the ')'. - // If we have both a base register and an index register make sure they are - // both 64-bit or 32-bit registers. - // To support VSIB, IndexReg can be 128-bit or 256-bit registers. - if (BaseReg != 0 && IndexReg != 0) { - if (X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg) && - (X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg) || - X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg)) && - IndexReg != X86::RIZ) { - Error(IndexLoc, "index register is 32-bit, but base register is 64-bit"); - return 0; - } - if (X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg) && - (X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg) || - X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg)) && - IndexReg != X86::EIZ){ - Error(IndexLoc, "index register is 64-bit, but base register is 32-bit"); - return 0; - } + // Check for use of invalid 16-bit registers. Only BX/BP/SI/DI are allowed, + // and then only in non-64-bit modes. Except for DX, which is a special case + // because an unofficial form of in/out instructions uses it. + if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg) && + (is64BitMode() || (BaseReg != X86::BX && BaseReg != X86::BP && + BaseReg != X86::SI && BaseReg != X86::DI)) && + BaseReg != X86::DX) { + Error(BaseLoc, "invalid 16-bit base register"); + return 0; + } + if (BaseReg == 0 && + X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg)) { + Error(IndexLoc, "16-bit memory operand may not include only index register"); + return 0; + } + + StringRef ErrMsg; + if (CheckBaseRegAndIndexReg(BaseReg, IndexReg, ErrMsg)) { + Error(BaseLoc, ErrMsg); + return 0; } return X86Operand::CreateMem(SegReg, Disp, BaseReg, IndexReg, Scale, @@ -1991,84 +1929,35 @@ ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, if (getLexer().isNot(AsmToken::EndOfStatement) && !isPrefix) { // Parse '*' modifier. - if (getLexer().is(AsmToken::Star)) { - SMLoc Loc = Parser.getTok().getLoc(); - Operands.push_back(X86Operand::CreateToken("*", Loc)); - Parser.Lex(); // Eat the star. - } - - // Read the first operand. - if (X86Operand *Op = ParseOperand()) - Operands.push_back(Op); - else { - Parser.eatToEndOfStatement(); - return true; - } - - while (getLexer().is(AsmToken::Comma)) { - Parser.Lex(); // Eat the comma. - - // Parse and remember the operand. - if (X86Operand *Op = ParseOperand()) - Operands.push_back(Op); - else { - Parser.eatToEndOfStatement(); - return true; - } - } - - if (STI.getFeatureBits() & X86::FeatureAVX512) { - // Parse mask register {%k1} - if (getLexer().is(AsmToken::LCurly)) { - SMLoc Loc = Parser.getTok().getLoc(); - Operands.push_back(X86Operand::CreateToken("{", Loc)); - Parser.Lex(); // Eat the { - if (X86Operand *Op = ParseOperand()) { - Operands.push_back(Op); - if (!getLexer().is(AsmToken::RCurly)) { - SMLoc Loc = getLexer().getLoc(); - Parser.eatToEndOfStatement(); - return Error(Loc, "Expected } at this point"); - } - Loc = Parser.getTok().getLoc(); - Operands.push_back(X86Operand::CreateToken("}", Loc)); - Parser.Lex(); // Eat the } - } else { - Parser.eatToEndOfStatement(); + if (getLexer().is(AsmToken::Star)) + Operands.push_back(X86Operand::CreateToken("*", consumeToken())); + + // Read the operands. + while(1) { + if (X86Operand *Op = ParseOperand()) { + Operands.push_back(Op); + if (!HandleAVX512Operand(Operands, *Op)) return true; - } - } - // Parse "zeroing non-masked" semantic {z} - if (getLexer().is(AsmToken::LCurly)) { - SMLoc Loc = Parser.getTok().getLoc(); - Operands.push_back(X86Operand::CreateToken("{z}", Loc)); - Parser.Lex(); // Eat the { - if (!getLexer().is(AsmToken::Identifier) || getLexer().getTok().getIdentifier() != "z") { - SMLoc Loc = getLexer().getLoc(); - Parser.eatToEndOfStatement(); - return Error(Loc, "Expected z at this point"); - } - Parser.Lex(); // Eat the z - if (!getLexer().is(AsmToken::RCurly)) { - SMLoc Loc = getLexer().getLoc(); - Parser.eatToEndOfStatement(); - return Error(Loc, "Expected } at this point"); - } - Parser.Lex(); // Eat the } + } else { + Parser.eatToEndOfStatement(); + return true; } - } + // check for comma and eat it + if (getLexer().is(AsmToken::Comma)) + Parser.Lex(); + else + break; + } - if (getLexer().isNot(AsmToken::EndOfStatement)) { - SMLoc Loc = getLexer().getLoc(); - Parser.eatToEndOfStatement(); - return Error(Loc, "unexpected token in argument list"); - } - } + if (getLexer().isNot(AsmToken::EndOfStatement)) + return ErrorAndEatStatement(getLexer().getLoc(), + "unexpected token in argument list"); + } - if (getLexer().is(AsmToken::EndOfStatement)) - Parser.Lex(); // Consume the EndOfStatement - else if (isPrefix && getLexer().is(AsmToken::Slash)) - Parser.Lex(); // Consume the prefix separator Slash + // Consume the EndOfStatement or the prefix separator Slash + if (getLexer().is(AsmToken::EndOfStatement) || + (isPrefix && getLexer().is(AsmToken::Slash))) + Parser.Lex(); if (ExtraImmOp && isParsingIntelSyntax()) Operands.push_back(X86Operand::CreateImm(ExtraImmOp, NameLoc, NameLoc)); @@ -2101,103 +1990,101 @@ ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, delete &Op; } } - // Transform "ins[bwl] %dx, %es:(%edi)" into "ins[bwl]" - if (Name.startswith("ins") && Operands.size() == 3 && - (Name == "insb" || Name == "insw" || Name == "insl")) { - X86Operand &Op = *(X86Operand*)Operands.begin()[1]; - X86Operand &Op2 = *(X86Operand*)Operands.begin()[2]; - if (Op.isReg() && Op.getReg() == X86::DX && isDstOp(Op2)) { - Operands.pop_back(); - Operands.pop_back(); - delete &Op; - delete &Op2; - } - } - // Transform "outs[bwl] %ds:(%esi), %dx" into "out[bwl]" - if (Name.startswith("outs") && Operands.size() == 3 && - (Name == "outsb" || Name == "outsw" || Name == "outsl")) { - X86Operand &Op = *(X86Operand*)Operands.begin()[1]; - X86Operand &Op2 = *(X86Operand*)Operands.begin()[2]; - if (isSrcOp(Op) && Op2.isReg() && Op2.getReg() == X86::DX) { - Operands.pop_back(); - Operands.pop_back(); - delete &Op; - delete &Op2; + // Append default arguments to "ins[bwld]" + if (Name.startswith("ins") && Operands.size() == 1 && + (Name == "insb" || Name == "insw" || Name == "insl" || + Name == "insd" )) { + if (isParsingIntelSyntax()) { + Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc)); + Operands.push_back(DefaultMemDIOperand(NameLoc)); + } else { + Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc)); + Operands.push_back(DefaultMemDIOperand(NameLoc)); } } - // Transform "movs[bwl] %ds:(%esi), %es:(%edi)" into "movs[bwl]" - if (Name.startswith("movs") && Operands.size() == 3 && - (Name == "movsb" || Name == "movsw" || Name == "movsl" || - (is64BitMode() && Name == "movsq"))) { - X86Operand &Op = *(X86Operand*)Operands.begin()[1]; - X86Operand &Op2 = *(X86Operand*)Operands.begin()[2]; - if (isSrcOp(Op) && isDstOp(Op2)) { - Operands.pop_back(); - Operands.pop_back(); - delete &Op; - delete &Op2; + // Append default arguments to "outs[bwld]" + if (Name.startswith("outs") && Operands.size() == 1 && + (Name == "outsb" || Name == "outsw" || Name == "outsl" || + Name == "outsd" )) { + if (isParsingIntelSyntax()) { + Operands.push_back(DefaultMemSIOperand(NameLoc)); + Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc)); + } else { + Operands.push_back(DefaultMemSIOperand(NameLoc)); + Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc)); } } - // Transform "lods[bwl] %ds:(%esi),{%al,%ax,%eax,%rax}" into "lods[bwl]" - if (Name.startswith("lods") && Operands.size() == 3 && + + // Transform "lods[bwlq]" into "lods[bwlq] ($SIREG)" for appropriate + // values of $SIREG according to the mode. It would be nice if this + // could be achieved with InstAlias in the tables. + if (Name.startswith("lods") && Operands.size() == 1 && (Name == "lods" || Name == "lodsb" || Name == "lodsw" || - Name == "lodsl" || (is64BitMode() && Name == "lodsq"))) { - X86Operand *Op1 = static_cast<X86Operand*>(Operands[1]); - X86Operand *Op2 = static_cast<X86Operand*>(Operands[2]); - if (isSrcOp(*Op1) && Op2->isReg()) { - const char *ins; - unsigned reg = Op2->getReg(); - bool isLods = Name == "lods"; - if (reg == X86::AL && (isLods || Name == "lodsb")) - ins = "lodsb"; - else if (reg == X86::AX && (isLods || Name == "lodsw")) - ins = "lodsw"; - else if (reg == X86::EAX && (isLods || Name == "lodsl")) - ins = "lodsl"; - else if (reg == X86::RAX && (isLods || Name == "lodsq")) - ins = "lodsq"; - else - ins = NULL; - if (ins != NULL) { - Operands.pop_back(); - Operands.pop_back(); - delete Op1; - delete Op2; - if (Name != ins) - static_cast<X86Operand*>(Operands[0])->setTokenValue(ins); - } - } - } - // Transform "stos[bwl] {%al,%ax,%eax,%rax},%es:(%edi)" into "stos[bwl]" - if (Name.startswith("stos") && Operands.size() == 3 && + Name == "lodsl" || Name == "lodsd" || Name == "lodsq")) + Operands.push_back(DefaultMemSIOperand(NameLoc)); + + // Transform "stos[bwlq]" into "stos[bwlq] ($DIREG)" for appropriate + // values of $DIREG according to the mode. It would be nice if this + // could be achieved with InstAlias in the tables. + if (Name.startswith("stos") && Operands.size() == 1 && (Name == "stos" || Name == "stosb" || Name == "stosw" || - Name == "stosl" || (is64BitMode() && Name == "stosq"))) { - X86Operand *Op1 = static_cast<X86Operand*>(Operands[1]); - X86Operand *Op2 = static_cast<X86Operand*>(Operands[2]); - if (isDstOp(*Op2) && Op1->isReg()) { - const char *ins; - unsigned reg = Op1->getReg(); - bool isStos = Name == "stos"; - if (reg == X86::AL && (isStos || Name == "stosb")) - ins = "stosb"; - else if (reg == X86::AX && (isStos || Name == "stosw")) - ins = "stosw"; - else if (reg == X86::EAX && (isStos || Name == "stosl")) - ins = "stosl"; - else if (reg == X86::RAX && (isStos || Name == "stosq")) - ins = "stosq"; - else - ins = NULL; - if (ins != NULL) { - Operands.pop_back(); - Operands.pop_back(); - delete Op1; - delete Op2; - if (Name != ins) - static_cast<X86Operand*>(Operands[0])->setTokenValue(ins); + Name == "stosl" || Name == "stosd" || Name == "stosq")) + Operands.push_back(DefaultMemDIOperand(NameLoc)); + + // Transform "scas[bwlq]" into "scas[bwlq] ($DIREG)" for appropriate + // values of $DIREG according to the mode. It would be nice if this + // could be achieved with InstAlias in the tables. + if (Name.startswith("scas") && Operands.size() == 1 && + (Name == "scas" || Name == "scasb" || Name == "scasw" || + Name == "scasl" || Name == "scasd" || Name == "scasq")) + Operands.push_back(DefaultMemDIOperand(NameLoc)); + + // Add default SI and DI operands to "cmps[bwlq]". + if (Name.startswith("cmps") && + (Name == "cmps" || Name == "cmpsb" || Name == "cmpsw" || + Name == "cmpsl" || Name == "cmpsd" || Name == "cmpsq")) { + if (Operands.size() == 1) { + if (isParsingIntelSyntax()) { + Operands.push_back(DefaultMemSIOperand(NameLoc)); + Operands.push_back(DefaultMemDIOperand(NameLoc)); + } else { + Operands.push_back(DefaultMemDIOperand(NameLoc)); + Operands.push_back(DefaultMemSIOperand(NameLoc)); } + } else if (Operands.size() == 3) { + X86Operand &Op = *(X86Operand*)Operands.begin()[1]; + X86Operand &Op2 = *(X86Operand*)Operands.begin()[2]; + if (!doSrcDstMatch(Op, Op2)) + return Error(Op.getStartLoc(), + "mismatching source and destination index registers"); + } + } + + // Add default SI and DI operands to "movs[bwlq]". + if ((Name.startswith("movs") && + (Name == "movs" || Name == "movsb" || Name == "movsw" || + Name == "movsl" || Name == "movsd" || Name == "movsq")) || + (Name.startswith("smov") && + (Name == "smov" || Name == "smovb" || Name == "smovw" || + Name == "smovl" || Name == "smovd" || Name == "smovq"))) { + if (Operands.size() == 1) { + if (Name == "movsd") + Operands.back() = X86Operand::CreateToken("movsl", NameLoc); + if (isParsingIntelSyntax()) { + Operands.push_back(DefaultMemDIOperand(NameLoc)); + Operands.push_back(DefaultMemSIOperand(NameLoc)); + } else { + Operands.push_back(DefaultMemSIOperand(NameLoc)); + Operands.push_back(DefaultMemDIOperand(NameLoc)); + } + } else if (Operands.size() == 3) { + X86Operand &Op = *(X86Operand*)Operands.begin()[1]; + X86Operand &Op2 = *(X86Operand*)Operands.begin()[2]; + if (!doSrcDstMatch(Op, Op2)) + return Error(Op.getStartLoc(), + "mismatching source and destination index registers"); } } @@ -2362,6 +2249,14 @@ processInstruction(MCInst &Inst, } static const char *getSubtargetFeatureName(unsigned Val); + +void X86AsmParser::EmitInstruction( + MCInst &Inst, SmallVectorImpl<MCParsedAsmOperand *> &Operands, + MCStreamer &Out) { + Instrumentation->InstrumentInstruction(Inst, Operands, getContext(), Out); + Out.EmitInstruction(Inst, STI); +} + bool X86AsmParser:: MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, SmallVectorImpl<MCParsedAsmOperand*> &Operands, @@ -2384,7 +2279,7 @@ MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, Inst.setOpcode(X86::WAIT); Inst.setLoc(IDLoc); if (!MatchingInlineAsm) - Out.EmitInstruction(Inst); + EmitInstruction(Inst, Operands, Out); const char *Repl = StringSwitch<const char*>(Op->getToken()) @@ -2420,7 +2315,7 @@ MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, Inst.setLoc(IDLoc); if (!MatchingInlineAsm) - Out.EmitInstruction(Inst); + EmitInstruction(Inst, Operands, Out); Opcode = Inst.getOpcode(); return false; case Match_MissingFeature: { @@ -2507,7 +2402,7 @@ MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, if (NumSuccessfulMatches == 1) { Inst.setLoc(IDLoc); if (!MatchingInlineAsm) - Out.EmitInstruction(Inst); + EmitInstruction(Inst, Operands, Out); Opcode = Inst.getOpcode(); return false; } @@ -2614,11 +2509,9 @@ bool X86AsmParser::ParseDirective(AsmToken DirectiveID) { } else if (IDVal.startswith(".intel_syntax")) { getParser().setAssemblerDialect(1); if (getLexer().isNot(AsmToken::EndOfStatement)) { - if(Parser.getTok().getString() == "noprefix") { - // FIXME : Handle noprefix + // FIXME: Handle noprefix + if (Parser.getTok().getString() == "noprefix") Parser.Lex(); - } else - return true; } return false; } @@ -2632,7 +2525,7 @@ bool X86AsmParser::ParseDirectiveWord(unsigned Size, SMLoc L) { for (;;) { const MCExpr *Value; if (getParser().parseExpression(Value)) - return true; + return false; getParser().getStreamer().EmitValue(Value, Size); @@ -2640,8 +2533,10 @@ bool X86AsmParser::ParseDirectiveWord(unsigned Size, SMLoc L) { break; // FIXME: Improve diagnostic. - if (getLexer().isNot(AsmToken::Comma)) - return Error(L, "unexpected token in directive"); + if (getLexer().isNot(AsmToken::Comma)) { + Error(L, "unexpected token in directive"); + return false; + } Parser.Lex(); } } @@ -2651,22 +2546,29 @@ bool X86AsmParser::ParseDirectiveWord(unsigned Size, SMLoc L) { } /// ParseDirectiveCode -/// ::= .code32 | .code64 +/// ::= .code16 | .code32 | .code64 bool X86AsmParser::ParseDirectiveCode(StringRef IDVal, SMLoc L) { - if (IDVal == ".code32") { + if (IDVal == ".code16") { Parser.Lex(); - if (is64BitMode()) { - SwitchMode(); + if (!is16BitMode()) { + SwitchMode(X86::Mode16Bit); + getParser().getStreamer().EmitAssemblerFlag(MCAF_Code16); + } + } else if (IDVal == ".code32") { + Parser.Lex(); + if (!is32BitMode()) { + SwitchMode(X86::Mode32Bit); getParser().getStreamer().EmitAssemblerFlag(MCAF_Code32); } } else if (IDVal == ".code64") { Parser.Lex(); if (!is64BitMode()) { - SwitchMode(); + SwitchMode(X86::Mode64Bit); getParser().getStreamer().EmitAssemblerFlag(MCAF_Code64); } } else { - return Error(L, "unexpected directive " + IDVal); + Error(L, "unknown directive " + IDVal); + return false; } return false; diff --git a/lib/Target/X86/AsmParser/X86AsmParserCommon.h b/lib/Target/X86/AsmParser/X86AsmParserCommon.h new file mode 100644 index 0000000..ef1565f --- /dev/null +++ b/lib/Target/X86/AsmParser/X86AsmParserCommon.h @@ -0,0 +1,43 @@ +//===-- X86AsmParserCommon.h - Common functions for X86AsmParser ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef X86_ASM_PARSER_COMMON_H +#define X86_ASM_PARSER_COMMON_H + +namespace llvm { + +inline bool isImmSExti16i8Value(uint64_t Value) { + return (( Value <= 0x000000000000007FULL)|| + (0x000000000000FF80ULL <= Value && Value <= 0x000000000000FFFFULL)|| + (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); +} + +inline bool isImmSExti32i8Value(uint64_t Value) { + return (( Value <= 0x000000000000007FULL)|| + (0x00000000FFFFFF80ULL <= Value && Value <= 0x00000000FFFFFFFFULL)|| + (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); +} + +inline bool isImmZExtu32u8Value(uint64_t Value) { + return (Value <= 0x00000000000000FFULL); +} + +inline bool isImmSExti64i8Value(uint64_t Value) { + return (( Value <= 0x000000000000007FULL)|| + (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); +} + +inline bool isImmSExti64i32Value(uint64_t Value) { + return (( Value <= 0x000000007FFFFFFFULL)|| + (0xFFFFFFFF80000000ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL)); +} + +} // End of namespace llvm + +#endif // X86_ASM_PARSER_COMMON_H diff --git a/lib/Target/X86/AsmParser/X86Operand.h b/lib/Target/X86/AsmParser/X86Operand.h new file mode 100644 index 0000000..45fe2a9 --- /dev/null +++ b/lib/Target/X86/AsmParser/X86Operand.h @@ -0,0 +1,488 @@ +//===-- X86Operand.h - Parsed X86 machine instruction --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef X86_OPERAND_H +#define X86_OPERAND_H + +#include "X86AsmParserCommon.h" +#include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCParser/MCParsedAsmOperand.h" + +namespace llvm { + +/// X86Operand - Instances of this class represent a parsed X86 machine +/// instruction. +struct X86Operand : public MCParsedAsmOperand { + enum KindTy { + Token, + Register, + Immediate, + Memory + } Kind; + + SMLoc StartLoc, EndLoc; + SMLoc OffsetOfLoc; + StringRef SymName; + void *OpDecl; + bool AddressOf; + + struct TokOp { + const char *Data; + unsigned Length; + }; + + struct RegOp { + unsigned RegNo; + }; + + struct ImmOp { + const MCExpr *Val; + }; + + struct MemOp { + unsigned SegReg; + const MCExpr *Disp; + unsigned BaseReg; + unsigned IndexReg; + unsigned Scale; + unsigned Size; + }; + + union { + struct TokOp Tok; + struct RegOp Reg; + struct ImmOp Imm; + struct MemOp Mem; + }; + + X86Operand(KindTy K, SMLoc Start, SMLoc End) + : Kind(K), StartLoc(Start), EndLoc(End) {} + + StringRef getSymName() override { return SymName; } + void *getOpDecl() override { return OpDecl; } + + /// getStartLoc - Get the location of the first token of this operand. + SMLoc getStartLoc() const override { return StartLoc; } + /// getEndLoc - Get the location of the last token of this operand. + SMLoc getEndLoc() const override { return EndLoc; } + /// getLocRange - Get the range between the first and last token of this + /// operand. + SMRange getLocRange() const { return SMRange(StartLoc, EndLoc); } + /// getOffsetOfLoc - Get the location of the offset operator. + SMLoc getOffsetOfLoc() const override { return OffsetOfLoc; } + + void print(raw_ostream &OS) const override {} + + StringRef getToken() const { + assert(Kind == Token && "Invalid access!"); + return StringRef(Tok.Data, Tok.Length); + } + void setTokenValue(StringRef Value) { + assert(Kind == Token && "Invalid access!"); + Tok.Data = Value.data(); + Tok.Length = Value.size(); + } + + unsigned getReg() const override { + assert(Kind == Register && "Invalid access!"); + return Reg.RegNo; + } + + const MCExpr *getImm() const { + assert(Kind == Immediate && "Invalid access!"); + return Imm.Val; + } + + const MCExpr *getMemDisp() const { + assert(Kind == Memory && "Invalid access!"); + return Mem.Disp; + } + unsigned getMemSegReg() const { + assert(Kind == Memory && "Invalid access!"); + return Mem.SegReg; + } + unsigned getMemBaseReg() const { + assert(Kind == Memory && "Invalid access!"); + return Mem.BaseReg; + } + unsigned getMemIndexReg() const { + assert(Kind == Memory && "Invalid access!"); + return Mem.IndexReg; + } + unsigned getMemScale() const { + assert(Kind == Memory && "Invalid access!"); + return Mem.Scale; + } + + bool isToken() const override {return Kind == Token; } + + bool isImm() const override { return Kind == Immediate; } + + bool isImmSExti16i8() const { + if (!isImm()) + return false; + + // If this isn't a constant expr, just assume it fits and let relaxation + // handle it. + const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); + if (!CE) + return true; + + // Otherwise, check the value is in a range that makes sense for this + // extension. + return isImmSExti16i8Value(CE->getValue()); + } + bool isImmSExti32i8() const { + if (!isImm()) + return false; + + // If this isn't a constant expr, just assume it fits and let relaxation + // handle it. + const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); + if (!CE) + return true; + + // Otherwise, check the value is in a range that makes sense for this + // extension. + return isImmSExti32i8Value(CE->getValue()); + } + bool isImmZExtu32u8() const { + if (!isImm()) + return false; + + // If this isn't a constant expr, just assume it fits and let relaxation + // handle it. + const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); + if (!CE) + return true; + + // Otherwise, check the value is in a range that makes sense for this + // extension. + return isImmZExtu32u8Value(CE->getValue()); + } + bool isImmSExti64i8() const { + if (!isImm()) + return false; + + // If this isn't a constant expr, just assume it fits and let relaxation + // handle it. + const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); + if (!CE) + return true; + + // Otherwise, check the value is in a range that makes sense for this + // extension. + return isImmSExti64i8Value(CE->getValue()); + } + bool isImmSExti64i32() const { + if (!isImm()) + return false; + + // If this isn't a constant expr, just assume it fits and let relaxation + // handle it. + const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm()); + if (!CE) + return true; + + // Otherwise, check the value is in a range that makes sense for this + // extension. + return isImmSExti64i32Value(CE->getValue()); + } + + bool isOffsetOf() const override { + return OffsetOfLoc.getPointer(); + } + + bool needAddressOf() const override { + return AddressOf; + } + + bool isMem() const override { return Kind == Memory; } + bool isMem8() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 8); + } + bool isMem16() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 16); + } + bool isMem32() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 32); + } + bool isMem64() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 64); + } + bool isMem80() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 80); + } + bool isMem128() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 128); + } + bool isMem256() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 256); + } + bool isMem512() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 512); + } + + bool isMemVX32() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 32) && + getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM15; + } + bool isMemVY32() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 32) && + getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM15; + } + bool isMemVX64() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 64) && + getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM15; + } + bool isMemVY64() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 64) && + getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM15; + } + bool isMemVZ32() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 32) && + getMemIndexReg() >= X86::ZMM0 && getMemIndexReg() <= X86::ZMM31; + } + bool isMemVZ64() const { + return Kind == Memory && (!Mem.Size || Mem.Size == 64) && + getMemIndexReg() >= X86::ZMM0 && getMemIndexReg() <= X86::ZMM31; + } + + bool isAbsMem() const { + return Kind == Memory && !getMemSegReg() && !getMemBaseReg() && + !getMemIndexReg() && getMemScale() == 1; + } + + bool isSrcIdx() const { + return !getMemIndexReg() && getMemScale() == 1 && + (getMemBaseReg() == X86::RSI || getMemBaseReg() == X86::ESI || + getMemBaseReg() == X86::SI) && isa<MCConstantExpr>(getMemDisp()) && + cast<MCConstantExpr>(getMemDisp())->getValue() == 0; + } + bool isSrcIdx8() const { + return isMem8() && isSrcIdx(); + } + bool isSrcIdx16() const { + return isMem16() && isSrcIdx(); + } + bool isSrcIdx32() const { + return isMem32() && isSrcIdx(); + } + bool isSrcIdx64() const { + return isMem64() && isSrcIdx(); + } + + bool isDstIdx() const { + return !getMemIndexReg() && getMemScale() == 1 && + (getMemSegReg() == 0 || getMemSegReg() == X86::ES) && + (getMemBaseReg() == X86::RDI || getMemBaseReg() == X86::EDI || + getMemBaseReg() == X86::DI) && isa<MCConstantExpr>(getMemDisp()) && + cast<MCConstantExpr>(getMemDisp())->getValue() == 0; + } + bool isDstIdx8() const { + return isMem8() && isDstIdx(); + } + bool isDstIdx16() const { + return isMem16() && isDstIdx(); + } + bool isDstIdx32() const { + return isMem32() && isDstIdx(); + } + bool isDstIdx64() const { + return isMem64() && isDstIdx(); + } + + bool isMemOffs8() const { + return Kind == Memory && !getMemBaseReg() && + !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 8); + } + bool isMemOffs16() const { + return Kind == Memory && !getMemBaseReg() && + !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 16); + } + bool isMemOffs32() const { + return Kind == Memory && !getMemBaseReg() && + !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 32); + } + bool isMemOffs64() const { + return Kind == Memory && !getMemBaseReg() && + !getMemIndexReg() && getMemScale() == 1 && (!Mem.Size || Mem.Size == 64); + } + + bool isReg() const override { return Kind == Register; } + + bool isGR32orGR64() const { + return Kind == Register && + (X86MCRegisterClasses[X86::GR32RegClassID].contains(getReg()) || + X86MCRegisterClasses[X86::GR64RegClassID].contains(getReg())); + } + + void addExpr(MCInst &Inst, const MCExpr *Expr) const { + // Add as immediates when possible. + if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr)) + Inst.addOperand(MCOperand::CreateImm(CE->getValue())); + else + Inst.addOperand(MCOperand::CreateExpr(Expr)); + } + + void addRegOperands(MCInst &Inst, unsigned N) const { + assert(N == 1 && "Invalid number of operands!"); + Inst.addOperand(MCOperand::CreateReg(getReg())); + } + + static unsigned getGR32FromGR64(unsigned RegNo) { + switch (RegNo) { + default: llvm_unreachable("Unexpected register"); + case X86::RAX: return X86::EAX; + case X86::RCX: return X86::ECX; + case X86::RDX: return X86::EDX; + case X86::RBX: return X86::EBX; + case X86::RBP: return X86::EBP; + case X86::RSP: return X86::ESP; + case X86::RSI: return X86::ESI; + case X86::RDI: return X86::EDI; + case X86::R8: return X86::R8D; + case X86::R9: return X86::R9D; + case X86::R10: return X86::R10D; + case X86::R11: return X86::R11D; + case X86::R12: return X86::R12D; + case X86::R13: return X86::R13D; + case X86::R14: return X86::R14D; + case X86::R15: return X86::R15D; + case X86::RIP: return X86::EIP; + } + } + + void addGR32orGR64Operands(MCInst &Inst, unsigned N) const { + assert(N == 1 && "Invalid number of operands!"); + unsigned RegNo = getReg(); + if (X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo)) + RegNo = getGR32FromGR64(RegNo); + Inst.addOperand(MCOperand::CreateReg(RegNo)); + } + + void addImmOperands(MCInst &Inst, unsigned N) const { + assert(N == 1 && "Invalid number of operands!"); + addExpr(Inst, getImm()); + } + + void addMemOperands(MCInst &Inst, unsigned N) const { + assert((N == 5) && "Invalid number of operands!"); + Inst.addOperand(MCOperand::CreateReg(getMemBaseReg())); + Inst.addOperand(MCOperand::CreateImm(getMemScale())); + Inst.addOperand(MCOperand::CreateReg(getMemIndexReg())); + addExpr(Inst, getMemDisp()); + Inst.addOperand(MCOperand::CreateReg(getMemSegReg())); + } + + void addAbsMemOperands(MCInst &Inst, unsigned N) const { + assert((N == 1) && "Invalid number of operands!"); + // Add as immediates when possible. + if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemDisp())) + Inst.addOperand(MCOperand::CreateImm(CE->getValue())); + else + Inst.addOperand(MCOperand::CreateExpr(getMemDisp())); + } + + void addSrcIdxOperands(MCInst &Inst, unsigned N) const { + assert((N == 2) && "Invalid number of operands!"); + Inst.addOperand(MCOperand::CreateReg(getMemBaseReg())); + Inst.addOperand(MCOperand::CreateReg(getMemSegReg())); + } + void addDstIdxOperands(MCInst &Inst, unsigned N) const { + assert((N == 1) && "Invalid number of operands!"); + Inst.addOperand(MCOperand::CreateReg(getMemBaseReg())); + } + + void addMemOffsOperands(MCInst &Inst, unsigned N) const { + assert((N == 2) && "Invalid number of operands!"); + // Add as immediates when possible. + if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemDisp())) + Inst.addOperand(MCOperand::CreateImm(CE->getValue())); + else + Inst.addOperand(MCOperand::CreateExpr(getMemDisp())); + Inst.addOperand(MCOperand::CreateReg(getMemSegReg())); + } + + static X86Operand *CreateToken(StringRef Str, SMLoc Loc) { + SMLoc EndLoc = SMLoc::getFromPointer(Loc.getPointer() + Str.size()); + X86Operand *Res = new X86Operand(Token, Loc, EndLoc); + Res->Tok.Data = Str.data(); + Res->Tok.Length = Str.size(); + return Res; + } + + static X86Operand *CreateReg(unsigned RegNo, SMLoc StartLoc, SMLoc EndLoc, + bool AddressOf = false, + SMLoc OffsetOfLoc = SMLoc(), + StringRef SymName = StringRef(), + void *OpDecl = 0) { + X86Operand *Res = new X86Operand(Register, StartLoc, EndLoc); + Res->Reg.RegNo = RegNo; + Res->AddressOf = AddressOf; + Res->OffsetOfLoc = OffsetOfLoc; + Res->SymName = SymName; + Res->OpDecl = OpDecl; + return Res; + } + + static X86Operand *CreateImm(const MCExpr *Val, SMLoc StartLoc, SMLoc EndLoc){ + X86Operand *Res = new X86Operand(Immediate, StartLoc, EndLoc); + Res->Imm.Val = Val; + return Res; + } + + /// Create an absolute memory operand. + static X86Operand *CreateMem(const MCExpr *Disp, SMLoc StartLoc, SMLoc EndLoc, + unsigned Size = 0, StringRef SymName = StringRef(), + void *OpDecl = 0) { + X86Operand *Res = new X86Operand(Memory, StartLoc, EndLoc); + Res->Mem.SegReg = 0; + Res->Mem.Disp = Disp; + Res->Mem.BaseReg = 0; + Res->Mem.IndexReg = 0; + Res->Mem.Scale = 1; + Res->Mem.Size = Size; + Res->SymName = SymName; + Res->OpDecl = OpDecl; + Res->AddressOf = false; + return Res; + } + + /// Create a generalized memory operand. + static X86Operand *CreateMem(unsigned SegReg, const MCExpr *Disp, + unsigned BaseReg, unsigned IndexReg, + unsigned Scale, SMLoc StartLoc, SMLoc EndLoc, + unsigned Size = 0, + StringRef SymName = StringRef(), + void *OpDecl = 0) { + // We should never just have a displacement, that should be parsed as an + // absolute memory operand. + assert((SegReg || BaseReg || IndexReg) && "Invalid memory operand!"); + + // The scale should always be one of {1,2,4,8}. + assert(((Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8)) && + "Invalid scale!"); + X86Operand *Res = new X86Operand(Memory, StartLoc, EndLoc); + Res->Mem.SegReg = SegReg; + Res->Mem.Disp = Disp; + Res->Mem.BaseReg = BaseReg; + Res->Mem.IndexReg = IndexReg; + Res->Mem.Scale = Scale; + Res->Mem.Size = Size; + Res->SymName = SymName; + Res->OpDecl = OpDecl; + Res->AddressOf = false; + return Res; + } +}; + +} // End of namespace llvm + +#endif // X86_OPERAND diff --git a/lib/Target/X86/CMakeLists.txt b/lib/Target/X86/CMakeLists.txt index 7e20151..206b651 100644 --- a/lib/Target/X86/CMakeLists.txt +++ b/lib/Target/X86/CMakeLists.txt @@ -37,11 +37,6 @@ set(sources ) if( CMAKE_CL_64 ) - # A workaround for a bug in cmake 2.8.3. See PR 8885. - if( CMAKE_VERSION STREQUAL "2.8.3" ) - include(CMakeDetermineCompilerId) - endif() - # end of workaround. enable_language(ASM_MASM) ADD_CUSTOM_COMMAND( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/X86CompilationCallback_Win64.obj @@ -53,8 +48,6 @@ endif() add_llvm_target(X86CodeGen ${sources}) -add_dependencies(LLVMX86CodeGen X86CommonTableGen intrinsics_gen) - add_subdirectory(AsmParser) add_subdirectory(Disassembler) add_subdirectory(InstPrinter) diff --git a/lib/Target/X86/Disassembler/Android.mk b/lib/Target/X86/Disassembler/Android.mk index 67400ad..3984266 100644 --- a/lib/Target/X86/Disassembler/Android.mk +++ b/lib/Target/X86/Disassembler/Android.mk @@ -3,7 +3,8 @@ LOCAL_PATH := $(call my-dir) x86_disassembler_TBLGEN_TABLES := \ X86GenDisassemblerTables.inc \ X86GenInstrInfo.inc \ - X86GenRegisterInfo.inc + X86GenRegisterInfo.inc \ + X86GenSubtargetInfo.inc x86_disassembler_SRC_FILES := \ X86Disassembler.cpp \ @@ -11,6 +12,7 @@ x86_disassembler_SRC_FILES := \ # For the device # ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) @@ -29,6 +31,7 @@ LOCAL_MODULE_TAGS := optional include $(LLVM_DEVICE_BUILD_MK) include $(LLVM_TBLGEN_RULES_MK) include $(BUILD_STATIC_LIBRARY) +endif # For the host # ===================================================== diff --git a/lib/Target/X86/Disassembler/CMakeLists.txt b/lib/Target/X86/Disassembler/CMakeLists.txt index 0cd6db9..deed115 100644 --- a/lib/Target/X86/Disassembler/CMakeLists.txt +++ b/lib/Target/X86/Disassembler/CMakeLists.txt @@ -1,16 +1,4 @@ -include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. ) - add_llvm_library(LLVMX86Disassembler X86Disassembler.cpp X86DisassemblerDecoder.c ) - -# workaround for hanging compilation on MSVC9 and 10 -if( MSVC_VERSION EQUAL 1400 OR MSVC_VERSION EQUAL 1500 OR MSVC_VERSION EQUAL 1600 ) -set_property( - SOURCE X86Disassembler.cpp - PROPERTY COMPILE_FLAGS "/Od" - ) -endif() - -add_dependencies(LLVMX86Disassembler X86CommonTableGen) diff --git a/lib/Target/X86/Disassembler/LLVMBuild.txt b/lib/Target/X86/Disassembler/LLVMBuild.txt index 0609f3c..cac7adf 100644 --- a/lib/Target/X86/Disassembler/LLVMBuild.txt +++ b/lib/Target/X86/Disassembler/LLVMBuild.txt @@ -19,5 +19,5 @@ type = Library name = X86Disassembler parent = X86 -required_libraries = MC Support X86Desc X86Info +required_libraries = MC Support X86Info add_to_library_groups = X86 diff --git a/lib/Target/X86/Disassembler/X86Disassembler.cpp b/lib/Target/X86/Disassembler/X86Disassembler.cpp index 903e36c..d5759cd 100644 --- a/lib/Target/X86/Disassembler/X86Disassembler.cpp +++ b/lib/Target/X86/Disassembler/X86Disassembler.cpp @@ -31,6 +31,8 @@ #include "X86GenRegisterInfo.inc" #define GET_INSTRINFO_ENUM #include "X86GenInstrInfo.inc" +#define GET_SUBTARGETINFO_ENUM +#include "X86GenSubtargetInfo.inc" using namespace llvm; using namespace llvm::X86Disassembler; @@ -73,9 +75,23 @@ static bool translateInstruction(MCInst &target, const MCDisassembler *Dis); X86GenericDisassembler::X86GenericDisassembler(const MCSubtargetInfo &STI, - DisassemblerMode mode, const MCInstrInfo *MII) - : MCDisassembler(STI), MII(MII), fMode(mode) {} + : MCDisassembler(STI), MII(MII) { + switch (STI.getFeatureBits() & + (X86::Mode16Bit | X86::Mode32Bit | X86::Mode64Bit)) { + case X86::Mode16Bit: + fMode = MODE_16BIT; + break; + case X86::Mode32Bit: + fMode = MODE_32BIT; + break; + case X86::Mode64Bit: + fMode = MODE_64BIT; + break; + default: + llvm_unreachable("Invalid CPU mode"); + } +} X86GenericDisassembler::~X86GenericDisassembler() { delete MII; @@ -207,6 +223,61 @@ static void tryAddingPcLoadReferenceComment(uint64_t Address, uint64_t Value, Dis->tryAddingPcLoadReferenceComment(Value, Address); } +static const uint8_t segmentRegnums[SEG_OVERRIDE_max] = { + 0, // SEG_OVERRIDE_NONE + X86::CS, + X86::SS, + X86::DS, + X86::ES, + X86::FS, + X86::GS +}; + +/// translateSrcIndex - Appends a source index operand to an MCInst. +/// +/// @param mcInst - The MCInst to append to. +/// @param insn - The internal instruction. +static bool translateSrcIndex(MCInst &mcInst, InternalInstruction &insn) { + unsigned baseRegNo; + + if (insn.mode == MODE_64BIT) + baseRegNo = insn.prefixPresent[0x67] ? X86::ESI : X86::RSI; + else if (insn.mode == MODE_32BIT) + baseRegNo = insn.prefixPresent[0x67] ? X86::SI : X86::ESI; + else { + assert(insn.mode == MODE_16BIT); + baseRegNo = insn.prefixPresent[0x67] ? X86::ESI : X86::SI; + } + MCOperand baseReg = MCOperand::CreateReg(baseRegNo); + mcInst.addOperand(baseReg); + + MCOperand segmentReg; + segmentReg = MCOperand::CreateReg(segmentRegnums[insn.segmentOverride]); + mcInst.addOperand(segmentReg); + return false; +} + +/// translateDstIndex - Appends a destination index operand to an MCInst. +/// +/// @param mcInst - The MCInst to append to. +/// @param insn - The internal instruction. + +static bool translateDstIndex(MCInst &mcInst, InternalInstruction &insn) { + unsigned baseRegNo; + + if (insn.mode == MODE_64BIT) + baseRegNo = insn.prefixPresent[0x67] ? X86::EDI : X86::RDI; + else if (insn.mode == MODE_32BIT) + baseRegNo = insn.prefixPresent[0x67] ? X86::DI : X86::EDI; + else { + assert(insn.mode == MODE_16BIT); + baseRegNo = insn.prefixPresent[0x67] ? X86::EDI : X86::DI; + } + MCOperand baseReg = MCOperand::CreateReg(baseRegNo); + mcInst.addOperand(baseReg); + return false; +} + /// translateImmediate - Appends an immediate operand to an MCInst. /// /// @param mcInst - The MCInst to append to. @@ -315,6 +386,13 @@ static void translateImmediate(MCInst &mcInst, uint64_t immediate, insn.immediateOffset, insn.immediateSize, mcInst, Dis)) mcInst.addOperand(MCOperand::CreateImm(immediate)); + + if (type == TYPE_MOFFS8 || type == TYPE_MOFFS16 || + type == TYPE_MOFFS32 || type == TYPE_MOFFS64) { + MCOperand segmentReg; + segmentReg = MCOperand::CreateReg(segmentRegnums[insn.segmentOverride]); + mcInst.addOperand(segmentReg); + } } /// translateRMRegister - Translates a register stored in the R/M field of the @@ -418,13 +496,22 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn, bool IndexIs256 = (Opcode == X86::VGATHERQPDYrm || Opcode == X86::VGATHERDPSYrm || Opcode == X86::VGATHERQPSYrm || + Opcode == X86::VGATHERDPDZrm || + Opcode == X86::VPGATHERDQZrm || Opcode == X86::VPGATHERQQYrm || Opcode == X86::VPGATHERDDYrm || Opcode == X86::VPGATHERQDYrm); - if (IndexIs128 || IndexIs256) { + bool IndexIs512 = (Opcode == X86::VGATHERQPDZrm || + Opcode == X86::VGATHERDPSZrm || + Opcode == X86::VGATHERQPSZrm || + Opcode == X86::VPGATHERQQZrm || + Opcode == X86::VPGATHERDDZrm || + Opcode == X86::VPGATHERQDZrm); + if (IndexIs128 || IndexIs256 || IndexIs512) { unsigned IndexOffset = insn.sibIndex - (insn.addressSize == 8 ? SIB_INDEX_RAX:SIB_INDEX_EAX); - SIBIndex IndexBase = IndexIs256 ? SIB_INDEX_YMM0 : SIB_INDEX_XMM0; + SIBIndex IndexBase = IndexIs512 ? SIB_INDEX_ZMM0 : + IndexIs256 ? SIB_INDEX_YMM0 : SIB_INDEX_XMM0; insn.sibIndex = (SIBIndex)(IndexBase + (insn.sibIndex == SIB_INDEX_NONE ? 4 : IndexOffset)); } @@ -513,17 +600,7 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn, } displacement = MCOperand::CreateImm(insn.displacement); - - static const uint8_t segmentRegnums[SEG_OVERRIDE_max] = { - 0, // SEG_OVERRIDE_NONE - X86::CS, - X86::SS, - X86::DS, - X86::ES, - X86::FS, - X86::GS - }; - + segmentReg = MCOperand::CreateReg(segmentRegnums[insn.segmentOverride]); mcInst.addOperand(baseReg); @@ -565,6 +642,9 @@ static bool translateRM(MCInst &mcInst, const OperandSpecifier &operand, case TYPE_XMM128: case TYPE_XMM256: case TYPE_XMM512: + case TYPE_VK1: + case TYPE_VK8: + case TYPE_VK16: case TYPE_DEBUGREG: case TYPE_CONTROLREG: return translateRMRegister(mcInst, insn); @@ -596,16 +676,25 @@ static bool translateRM(MCInst &mcInst, const OperandSpecifier &operand, /// /// @param mcInst - The MCInst to append to. /// @param stackPos - The stack position to translate. -/// @return - 0 on success; nonzero otherwise. -static bool translateFPRegister(MCInst &mcInst, - uint8_t stackPos) { - if (stackPos >= 8) { - debug("Invalid FP stack position"); +static void translateFPRegister(MCInst &mcInst, + uint8_t stackPos) { + mcInst.addOperand(MCOperand::CreateReg(X86::ST0 + stackPos)); +} + +/// translateMaskRegister - Translates a 3-bit mask register number to +/// LLVM form, and appends it to an MCInst. +/// +/// @param mcInst - The MCInst to append to. +/// @param maskRegNum - Number of mask register from 0 to 7. +/// @return - false on success; true otherwise. +static bool translateMaskRegister(MCInst &mcInst, + uint8_t maskRegNum) { + if (maskRegNum >= 8) { + debug("Invalid mask register number"); return true; } - - mcInst.addOperand(MCOperand::CreateReg(X86::ST0 + stackPos)); + mcInst.addOperand(MCOperand::CreateReg(X86::K0 + maskRegNum)); return false; } @@ -626,6 +715,8 @@ static bool translateOperand(MCInst &mcInst, const OperandSpecifier &operand, case ENCODING_REG: translateRegister(mcInst, insn.reg); return false; + case ENCODING_WRITEMASK: + return translateMaskRegister(mcInst, insn.writemask); case ENCODING_RM: return translateRM(mcInst, operand, insn, Dis); case ENCODING_CB: @@ -648,17 +739,20 @@ static bool translateOperand(MCInst &mcInst, const OperandSpecifier &operand, insn, Dis); return false; + case ENCODING_SI: + return translateSrcIndex(mcInst, insn); + case ENCODING_DI: + return translateDstIndex(mcInst, insn); case ENCODING_RB: case ENCODING_RW: case ENCODING_RD: case ENCODING_RO: - translateRegister(mcInst, insn.opcodeRegister); - return false; - case ENCODING_I: - return translateFPRegister(mcInst, insn.opcodeModifier); case ENCODING_Rv: translateRegister(mcInst, insn.opcodeRegister); return false; + case ENCODING_FP: + translateFPRegister(mcInst, insn.modRM & 7); + return false; case ENCODING_VVVV: translateRegister(mcInst, insn.vvvv); return false; @@ -708,22 +802,16 @@ static bool translateInstruction(MCInst &mcInst, return false; } -static MCDisassembler *createX86_32Disassembler(const Target &T, - const MCSubtargetInfo &STI) { - return new X86Disassembler::X86GenericDisassembler(STI, MODE_32BIT, - T.createMCInstrInfo()); -} - -static MCDisassembler *createX86_64Disassembler(const Target &T, - const MCSubtargetInfo &STI) { - return new X86Disassembler::X86GenericDisassembler(STI, MODE_64BIT, +static MCDisassembler *createX86Disassembler(const Target &T, + const MCSubtargetInfo &STI) { + return new X86Disassembler::X86GenericDisassembler(STI, T.createMCInstrInfo()); } extern "C" void LLVMInitializeX86Disassembler() { // Register the disassembler. TargetRegistry::RegisterMCDisassembler(TheX86_32Target, - createX86_32Disassembler); + createX86Disassembler); TargetRegistry::RegisterMCDisassembler(TheX86_64Target, - createX86_64Disassembler); + createX86Disassembler); } diff --git a/lib/Target/X86/Disassembler/X86Disassembler.h b/lib/Target/X86/Disassembler/X86Disassembler.h index b92427a..4e6e297 100644 --- a/lib/Target/X86/Disassembler/X86Disassembler.h +++ b/lib/Target/X86/Disassembler/X86Disassembler.h @@ -105,20 +105,16 @@ class X86GenericDisassembler : public MCDisassembler { public: /// Constructor - Initializes the disassembler. /// - /// @param mode - The X86 architecture mode to decode for. - X86GenericDisassembler(const MCSubtargetInfo &STI, DisassemblerMode mode, - const MCInstrInfo *MII); + X86GenericDisassembler(const MCSubtargetInfo &STI, const MCInstrInfo *MII); private: ~X86GenericDisassembler(); public: /// getInstruction - See MCDisassembler. - DecodeStatus getInstruction(MCInst &instr, - uint64_t &size, - const MemoryObject ®ion, - uint64_t address, + DecodeStatus getInstruction(MCInst &instr, uint64_t &size, + const MemoryObject ®ion, uint64_t address, raw_ostream &vStream, - raw_ostream &cStream) const; + raw_ostream &cStream) const override; private: DisassemblerMode fMode; diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c index c81a857..0801c96 100644 --- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c +++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c @@ -40,7 +40,7 @@ * @return - The InstructionContext to use when looking up an * an instruction with these attributes. */ -static InstructionContext contextForAttrs(uint8_t attrMask) { +static InstructionContext contextForAttrs(uint16_t attrMask) { return CONTEXTS_SYM[attrMask]; } @@ -57,7 +57,7 @@ static InstructionContext contextForAttrs(uint8_t attrMask) { */ static int modRMRequired(OpcodeType type, InstructionContext insnContext, - uint8_t opcode) { + uint16_t opcode) { const struct ContextDecision* decision = 0; switch (type) { @@ -73,12 +73,6 @@ static int modRMRequired(OpcodeType type, case THREEBYTE_3A: decision = &THREEBYTE3A_SYM; break; - case THREEBYTE_A6: - decision = &THREEBYTEA6_SYM; - break; - case THREEBYTE_A7: - decision = &THREEBYTEA7_SYM; - break; case XOP8_MAP: decision = &XOP8_MAP_SYM; break; @@ -123,12 +117,6 @@ static InstrUID decode(OpcodeType type, case THREEBYTE_3A: dec = &THREEBYTE3A_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode]; break; - case THREEBYTE_A6: - dec = &THREEBYTEA6_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode]; - break; - case THREEBYTE_A7: - dec = &THREEBYTEA7_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode]; - break; case XOP8_MAP: dec = &XOP8_MAP_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode]; break; @@ -444,9 +432,58 @@ static int readPrefixes(struct InternalInstruction* insn) { dbgprintf(insn, "Found prefix 0x%hhx", byte); } - insn->vexXopType = TYPE_NO_VEX_XOP; + insn->vectorExtensionType = TYPE_NO_VEX_XOP; + + if (byte == 0x62) { + uint8_t byte1, byte2; + + if (consumeByte(insn, &byte1)) { + dbgprintf(insn, "Couldn't read second byte of EVEX prefix"); + return -1; + } + + if (lookAtByte(insn, &byte2)) { + dbgprintf(insn, "Couldn't read third byte of EVEX prefix"); + return -1; + } + + if ((insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) && + ((~byte1 & 0xc) == 0xc) && ((byte2 & 0x4) == 0x4)) { + insn->vectorExtensionType = TYPE_EVEX; + } + else { + unconsumeByte(insn); /* unconsume byte1 */ + unconsumeByte(insn); /* unconsume byte */ + insn->necessaryPrefixLocation = insn->readerCursor - 2; + } - if (byte == 0xc4) { + if (insn->vectorExtensionType == TYPE_EVEX) { + insn->vectorExtensionPrefix[0] = byte; + insn->vectorExtensionPrefix[1] = byte1; + if (consumeByte(insn, &insn->vectorExtensionPrefix[2])) { + dbgprintf(insn, "Couldn't read third byte of EVEX prefix"); + return -1; + } + if (consumeByte(insn, &insn->vectorExtensionPrefix[3])) { + dbgprintf(insn, "Couldn't read fourth byte of EVEX prefix"); + return -1; + } + + /* We simulate the REX prefix for simplicity's sake */ + if (insn->mode == MODE_64BIT) { + insn->rexPrefix = 0x40 + | (wFromEVEX3of4(insn->vectorExtensionPrefix[2]) << 3) + | (rFromEVEX2of4(insn->vectorExtensionPrefix[1]) << 2) + | (xFromEVEX2of4(insn->vectorExtensionPrefix[1]) << 1) + | (bFromEVEX2of4(insn->vectorExtensionPrefix[1]) << 0); + } + + dbgprintf(insn, "Found EVEX prefix 0x%hhx 0x%hhx 0x%hhx 0x%hhx", + insn->vectorExtensionPrefix[0], insn->vectorExtensionPrefix[1], + insn->vectorExtensionPrefix[2], insn->vectorExtensionPrefix[3]); + } + } + else if (byte == 0xc4) { uint8_t byte1; if (lookAtByte(insn, &byte1)) { @@ -455,7 +492,7 @@ static int readPrefixes(struct InternalInstruction* insn) { } if (insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) { - insn->vexXopType = TYPE_VEX_3B; + insn->vectorExtensionType = TYPE_VEX_3B; insn->necessaryPrefixLocation = insn->readerCursor - 1; } else { @@ -463,33 +500,24 @@ static int readPrefixes(struct InternalInstruction* insn) { insn->necessaryPrefixLocation = insn->readerCursor - 1; } - if (insn->vexXopType == TYPE_VEX_3B) { - insn->vexXopPrefix[0] = byte; - consumeByte(insn, &insn->vexXopPrefix[1]); - consumeByte(insn, &insn->vexXopPrefix[2]); + if (insn->vectorExtensionType == TYPE_VEX_3B) { + insn->vectorExtensionPrefix[0] = byte; + consumeByte(insn, &insn->vectorExtensionPrefix[1]); + consumeByte(insn, &insn->vectorExtensionPrefix[2]); /* We simulate the REX prefix for simplicity's sake */ if (insn->mode == MODE_64BIT) { insn->rexPrefix = 0x40 - | (wFromVEX3of3(insn->vexXopPrefix[2]) << 3) - | (rFromVEX2of3(insn->vexXopPrefix[1]) << 2) - | (xFromVEX2of3(insn->vexXopPrefix[1]) << 1) - | (bFromVEX2of3(insn->vexXopPrefix[1]) << 0); - } - - switch (ppFromVEX3of3(insn->vexXopPrefix[2])) - { - default: - break; - case VEX_PREFIX_66: - hasOpSize = TRUE; - break; + | (wFromVEX3of3(insn->vectorExtensionPrefix[2]) << 3) + | (rFromVEX2of3(insn->vectorExtensionPrefix[1]) << 2) + | (xFromVEX2of3(insn->vectorExtensionPrefix[1]) << 1) + | (bFromVEX2of3(insn->vectorExtensionPrefix[1]) << 0); } dbgprintf(insn, "Found VEX prefix 0x%hhx 0x%hhx 0x%hhx", - insn->vexXopPrefix[0], insn->vexXopPrefix[1], - insn->vexXopPrefix[2]); + insn->vectorExtensionPrefix[0], insn->vectorExtensionPrefix[1], + insn->vectorExtensionPrefix[2]); } } else if (byte == 0xc5) { @@ -501,22 +529,22 @@ static int readPrefixes(struct InternalInstruction* insn) { } if (insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) { - insn->vexXopType = TYPE_VEX_2B; + insn->vectorExtensionType = TYPE_VEX_2B; } else { unconsumeByte(insn); } - if (insn->vexXopType == TYPE_VEX_2B) { - insn->vexXopPrefix[0] = byte; - consumeByte(insn, &insn->vexXopPrefix[1]); + if (insn->vectorExtensionType == TYPE_VEX_2B) { + insn->vectorExtensionPrefix[0] = byte; + consumeByte(insn, &insn->vectorExtensionPrefix[1]); if (insn->mode == MODE_64BIT) { insn->rexPrefix = 0x40 - | (rFromVEX2of2(insn->vexXopPrefix[1]) << 2); + | (rFromVEX2of2(insn->vectorExtensionPrefix[1]) << 2); } - switch (ppFromVEX2of2(insn->vexXopPrefix[1])) + switch (ppFromVEX2of2(insn->vectorExtensionPrefix[1])) { default: break; @@ -525,7 +553,9 @@ static int readPrefixes(struct InternalInstruction* insn) { break; } - dbgprintf(insn, "Found VEX prefix 0x%hhx 0x%hhx", insn->vexXopPrefix[0], insn->vexXopPrefix[1]); + dbgprintf(insn, "Found VEX prefix 0x%hhx 0x%hhx", + insn->vectorExtensionPrefix[0], + insn->vectorExtensionPrefix[1]); } } else if (byte == 0x8f) { @@ -537,7 +567,7 @@ static int readPrefixes(struct InternalInstruction* insn) { } if ((byte1 & 0x38) != 0x0) { /* 0 in these 3 bits is a POP instruction. */ - insn->vexXopType = TYPE_XOP; + insn->vectorExtensionType = TYPE_XOP; insn->necessaryPrefixLocation = insn->readerCursor - 1; } else { @@ -545,22 +575,22 @@ static int readPrefixes(struct InternalInstruction* insn) { insn->necessaryPrefixLocation = insn->readerCursor - 1; } - if (insn->vexXopType == TYPE_XOP) { - insn->vexXopPrefix[0] = byte; - consumeByte(insn, &insn->vexXopPrefix[1]); - consumeByte(insn, &insn->vexXopPrefix[2]); + if (insn->vectorExtensionType == TYPE_XOP) { + insn->vectorExtensionPrefix[0] = byte; + consumeByte(insn, &insn->vectorExtensionPrefix[1]); + consumeByte(insn, &insn->vectorExtensionPrefix[2]); /* We simulate the REX prefix for simplicity's sake */ if (insn->mode == MODE_64BIT) { insn->rexPrefix = 0x40 - | (wFromXOP3of3(insn->vexXopPrefix[2]) << 3) - | (rFromXOP2of3(insn->vexXopPrefix[1]) << 2) - | (xFromXOP2of3(insn->vexXopPrefix[1]) << 1) - | (bFromXOP2of3(insn->vexXopPrefix[1]) << 0); + | (wFromXOP3of3(insn->vectorExtensionPrefix[2]) << 3) + | (rFromXOP2of3(insn->vectorExtensionPrefix[1]) << 2) + | (xFromXOP2of3(insn->vectorExtensionPrefix[1]) << 1) + | (bFromXOP2of3(insn->vectorExtensionPrefix[1]) << 0); } - switch (ppFromXOP3of3(insn->vexXopPrefix[2])) + switch (ppFromXOP3of3(insn->vectorExtensionPrefix[2])) { default: break; @@ -570,8 +600,8 @@ static int readPrefixes(struct InternalInstruction* insn) { } dbgprintf(insn, "Found XOP prefix 0x%hhx 0x%hhx 0x%hhx", - insn->vexXopPrefix[0], insn->vexXopPrefix[1], - insn->vexXopPrefix[2]); + insn->vectorExtensionPrefix[0], insn->vectorExtensionPrefix[1], + insn->vectorExtensionPrefix[2]); } } else { @@ -646,13 +676,29 @@ static int readOpcode(struct InternalInstruction* insn) { insn->opcodeType = ONEBYTE; - if (insn->vexXopType == TYPE_VEX_3B) + if (insn->vectorExtensionType == TYPE_EVEX) { - switch (mmmmmFromVEX2of3(insn->vexXopPrefix[1])) - { + switch (mmFromEVEX2of4(insn->vectorExtensionPrefix[1])) { + default: + dbgprintf(insn, "Unhandled mm field for instruction (0x%hhx)", + mmFromEVEX2of4(insn->vectorExtensionPrefix[1])); + return -1; + case VEX_LOB_0F: + insn->opcodeType = TWOBYTE; + return consumeByte(insn, &insn->opcode); + case VEX_LOB_0F38: + insn->opcodeType = THREEBYTE_38; + return consumeByte(insn, &insn->opcode); + case VEX_LOB_0F3A: + insn->opcodeType = THREEBYTE_3A; + return consumeByte(insn, &insn->opcode); + } + } + else if (insn->vectorExtensionType == TYPE_VEX_3B) { + switch (mmmmmFromVEX2of3(insn->vectorExtensionPrefix[1])) { default: dbgprintf(insn, "Unhandled m-mmmm field for instruction (0x%hhx)", - mmmmmFromVEX2of3(insn->vexXopPrefix[1])); + mmmmmFromVEX2of3(insn->vectorExtensionPrefix[1])); return -1; case VEX_LOB_0F: insn->opcodeType = TWOBYTE; @@ -665,18 +711,15 @@ static int readOpcode(struct InternalInstruction* insn) { return consumeByte(insn, &insn->opcode); } } - else if (insn->vexXopType == TYPE_VEX_2B) - { + else if (insn->vectorExtensionType == TYPE_VEX_2B) { insn->opcodeType = TWOBYTE; return consumeByte(insn, &insn->opcode); } - else if (insn->vexXopType == TYPE_XOP) - { - switch (mmmmmFromXOP2of3(insn->vexXopPrefix[1])) - { + else if (insn->vectorExtensionType == TYPE_XOP) { + switch (mmmmmFromXOP2of3(insn->vectorExtensionPrefix[1])) { default: dbgprintf(insn, "Unhandled m-mmmm field for instruction (0x%hhx)", - mmmmmFromVEX2of3(insn->vexXopPrefix[1])); + mmmmmFromVEX2of3(insn->vectorExtensionPrefix[1])); return -1; case XOP_MAP_SELECT_8: insn->opcodeType = XOP8_MAP; @@ -713,20 +756,6 @@ static int readOpcode(struct InternalInstruction* insn) { return -1; insn->opcodeType = THREEBYTE_3A; - } else if (current == 0xa6) { - dbgprintf(insn, "Found a three-byte escape prefix (0x%hhx)", current); - - if (consumeByte(insn, ¤t)) - return -1; - - insn->opcodeType = THREEBYTE_A6; - } else if (current == 0xa7) { - dbgprintf(insn, "Found a three-byte escape prefix (0x%hhx)", current); - - if (consumeByte(insn, ¤t)) - return -1; - - insn->opcodeType = THREEBYTE_A7; } else { dbgprintf(insn, "Didn't find a three-byte escape prefix"); @@ -760,10 +789,10 @@ static int readModRM(struct InternalInstruction* insn); */ static int getIDWithAttrMask(uint16_t* instructionID, struct InternalInstruction* insn, - uint8_t attrMask) { + uint16_t attrMask) { BOOL hasModRMExtension; - uint8_t instructionClass; + uint16_t instructionClass; instructionClass = contextForAttrs(attrMask); @@ -826,7 +855,7 @@ static BOOL is16BitEquivalent(const char* orig, const char* equiv) { * nonzero otherwise. */ static int getID(struct InternalInstruction* insn, const void *miiArg) { - uint8_t attrMask; + uint16_t attrMask; uint16_t instructionID; dbgprintf(insn, "getID()"); @@ -836,11 +865,11 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { if (insn->mode == MODE_64BIT) attrMask |= ATTR_64BIT; - if (insn->vexXopType != TYPE_NO_VEX_XOP) { - attrMask |= ATTR_VEX; + if (insn->vectorExtensionType != TYPE_NO_VEX_XOP) { + attrMask |= (insn->vectorExtensionType == TYPE_EVEX) ? ATTR_EVEX : ATTR_VEX; - if (insn->vexXopType == TYPE_VEX_3B) { - switch (ppFromVEX3of3(insn->vexXopPrefix[2])) { + if (insn->vectorExtensionType == TYPE_EVEX) { + switch (ppFromEVEX3of4(insn->vectorExtensionPrefix[2])) { case VEX_PREFIX_66: attrMask |= ATTR_OPSIZE; break; @@ -852,11 +881,35 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { break; } - if (lFromVEX3of3(insn->vexXopPrefix[2])) + if (zFromEVEX4of4(insn->vectorExtensionPrefix[3])) + attrMask |= ATTR_EVEXKZ; + if (bFromEVEX4of4(insn->vectorExtensionPrefix[3])) + attrMask |= ATTR_EVEXB; + if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3])) + attrMask |= ATTR_EVEXK; + if (lFromEVEX4of4(insn->vectorExtensionPrefix[3])) + attrMask |= ATTR_EVEXL; + if (l2FromEVEX4of4(insn->vectorExtensionPrefix[3])) + attrMask |= ATTR_EVEXL2; + } + else if (insn->vectorExtensionType == TYPE_VEX_3B) { + switch (ppFromVEX3of3(insn->vectorExtensionPrefix[2])) { + case VEX_PREFIX_66: + attrMask |= ATTR_OPSIZE; + break; + case VEX_PREFIX_F3: + attrMask |= ATTR_XS; + break; + case VEX_PREFIX_F2: + attrMask |= ATTR_XD; + break; + } + + if (lFromVEX3of3(insn->vectorExtensionPrefix[2])) attrMask |= ATTR_VEXL; } - else if (insn->vexXopType == TYPE_VEX_2B) { - switch (ppFromVEX2of2(insn->vexXopPrefix[1])) { + else if (insn->vectorExtensionType == TYPE_VEX_2B) { + switch (ppFromVEX2of2(insn->vectorExtensionPrefix[1])) { case VEX_PREFIX_66: attrMask |= ATTR_OPSIZE; break; @@ -868,11 +921,11 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { break; } - if (lFromVEX2of2(insn->vexXopPrefix[1])) + if (lFromVEX2of2(insn->vectorExtensionPrefix[1])) attrMask |= ATTR_VEXL; } - else if (insn->vexXopType == TYPE_XOP) { - switch (ppFromXOP3of3(insn->vexXopPrefix[2])) { + else if (insn->vectorExtensionType == TYPE_XOP) { + switch (ppFromXOP3of3(insn->vectorExtensionPrefix[2])) { case VEX_PREFIX_66: attrMask |= ATTR_OPSIZE; break; @@ -884,7 +937,7 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { break; } - if (lFromXOP3of3(insn->vexXopPrefix[2])) + if (lFromXOP3of3(insn->vectorExtensionPrefix[2])) attrMask |= ATTR_VEXL; } else { @@ -892,7 +945,7 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { } } else { - if (isPrefixAtLocation(insn, 0x66, insn->necessaryPrefixLocation)) + if (insn->mode != MODE_16BIT && isPrefixAtLocation(insn, 0x66, insn->necessaryPrefixLocation)) attrMask |= ATTR_OPSIZE; else if (isPrefixAtLocation(insn, 0x67, insn->necessaryPrefixLocation)) attrMask |= ATTR_ADSIZE; @@ -908,9 +961,29 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { if (getIDWithAttrMask(&instructionID, insn, attrMask)) return -1; + /* + * JCXZ/JECXZ need special handling for 16-bit mode because the meaning + * of the AdSize prefix is inverted w.r.t. 32-bit mode. + */ + if (insn->mode == MODE_16BIT && insn->opcode == 0xE3) { + const struct InstructionSpecifier *spec; + spec = specifierForUID(instructionID); + + /* + * Check for Ii8PCRel instructions. We could alternatively do a + * string-compare on the names, but this is probably cheaper. + */ + if (x86OperandSets[spec->operands][0].type == TYPE_REL8) { + attrMask ^= ATTR_ADSIZE; + if (getIDWithAttrMask(&instructionID, insn, attrMask)) + return -1; + } + } + /* The following clauses compensate for limitations of the tables. */ - if (insn->prefixPresent[0x66] && !(attrMask & ATTR_OPSIZE)) { + if ((insn->mode == MODE_16BIT || insn->prefixPresent[0x66]) && + !(attrMask & ATTR_OPSIZE)) { /* * The instruction tables make no distinction between instructions that * allow OpSize anywhere (i.e., 16-bit operations) and that need it in a @@ -942,7 +1015,8 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) { specWithOpSizeName = x86DisassemblerGetInstrName(instructionIDWithOpsize, miiArg); - if (is16BitEquivalent(specName, specWithOpSizeName)) { + if (is16BitEquivalent(specName, specWithOpSizeName) && + (insn->mode == MODE_16BIT) ^ insn->prefixPresent[0x66]) { insn->instructionID = instructionIDWithOpsize; insn->spec = specifierForUID(instructionIDWithOpsize); } else { @@ -1018,7 +1092,6 @@ static int readSIB(struct InternalInstruction* insn) { case 2: dbgprintf(insn, "SIB-based addressing doesn't work in 16-bit mode"); return -1; - break; case 4: sibIndexBase = SIB_INDEX_EAX; sibBaseBase = SIB_BASE_EAX; @@ -1033,6 +1106,8 @@ static int readSIB(struct InternalInstruction* insn) { return -1; index = indexFromSIB(insn->sib) | (xFromREX(insn->rexPrefix) << 3); + if (insn->vectorExtensionType == TYPE_EVEX) + index |= v2FromEVEX4of4(insn->vectorExtensionPrefix[3]) << 4; switch (index) { case 0x4: @@ -1065,6 +1140,7 @@ static int readSIB(struct InternalInstruction* insn) { switch (base) { case 0x5: + case 0xd: switch (modFromModRM(insn->modRM)) { case 0x0: insn->eaDisplacement = EA_DISP_32; @@ -1072,13 +1148,11 @@ static int readSIB(struct InternalInstruction* insn) { break; case 0x1: insn->eaDisplacement = EA_DISP_8; - insn->sibBase = (insn->addressSize == 4 ? - SIB_BASE_EBP : SIB_BASE_RBP); + insn->sibBase = (SIBBase)(sibBaseBase + base); break; case 0x2: insn->eaDisplacement = EA_DISP_32; - insn->sibBase = (insn->addressSize == 4 ? - SIB_BASE_EBP : SIB_BASE_RBP); + insn->sibBase = (SIBBase)(sibBaseBase + base); break; case 0x3: debug("Cannot have Mod = 0b11 and a SIB byte"); @@ -1183,6 +1257,10 @@ static int readModRM(struct InternalInstruction* insn) { reg |= rFromREX(insn->rexPrefix) << 3; rm |= bFromREX(insn->rexPrefix) << 3; + if (insn->vectorExtensionType == TYPE_EVEX) { + reg |= r2FromEVEX2of4(insn->vectorExtensionPrefix[1]) << 4; + rm |= xFromEVEX2of4(insn->vectorExtensionPrefix[1]) << 4; + } insn->reg = (Reg)(insn->regBase + reg); @@ -1205,6 +1283,7 @@ static int readModRM(struct InternalInstruction* insn) { case 0x1: insn->eaBase = (EABase)(insn->eaBaseBase + rm); insn->eaDisplacement = EA_DISP_8; + insn->displacementSize = 1; if (readDisplacement(insn)) return -1; break; @@ -1229,12 +1308,12 @@ static int readModRM(struct InternalInstruction* insn) { case 0x0: insn->eaDisplacement = EA_DISP_NONE; /* readSIB may override this */ switch (rm) { + case 0x14: case 0x4: case 0xc: /* in case REXW.b is set */ insn->eaBase = (insn->addressSize == 4 ? EA_BASE_sib : EA_BASE_sib64); - readSIB(insn); - if (readDisplacement(insn)) + if (readSIB(insn) || readDisplacement(insn)) return -1; break; case 0x5: @@ -1249,14 +1328,16 @@ static int readModRM(struct InternalInstruction* insn) { } break; case 0x1: + insn->displacementSize = 1; + /* FALLTHROUGH */ case 0x2: insn->eaDisplacement = (mod == 0x1 ? EA_DISP_8 : EA_DISP_32); switch (rm) { + case 0x14: case 0x4: case 0xc: /* in case REXW.b is set */ insn->eaBase = EA_BASE_sib; - readSIB(insn); - if (readDisplacement(insn)) + if (readSIB(insn) || readDisplacement(insn)) return -1; break; default: @@ -1312,6 +1393,10 @@ static int readModRM(struct InternalInstruction* insn) { case TYPE_XMM32: \ case TYPE_XMM: \ return prefix##_XMM0 + index; \ + case TYPE_VK1: \ + case TYPE_VK8: \ + case TYPE_VK16: \ + return prefix##_K0 + index; \ case TYPE_MM64: \ case TYPE_MM32: \ case TYPE_MM: \ @@ -1400,44 +1485,11 @@ static int fixupReg(struct InternalInstruction *insn, } /* - * readOpcodeModifier - Reads an operand from the opcode field of an - * instruction. Handles AddRegFrm instructions. - * - * @param insn - The instruction whose opcode field is to be read. - * @param inModRM - Indicates that the opcode field is to be read from the - * ModR/M extension; useful for escape opcodes - * @return - 0 on success; nonzero otherwise. - */ -static int readOpcodeModifier(struct InternalInstruction* insn) { - dbgprintf(insn, "readOpcodeModifier()"); - - if (insn->consumedOpcodeModifier) - return 0; - - insn->consumedOpcodeModifier = TRUE; - - switch (insn->spec->modifierType) { - default: - debug("Unknown modifier type."); - return -1; - case MODIFIER_NONE: - debug("No modifier but an operand expects one."); - return -1; - case MODIFIER_OPCODE: - insn->opcodeModifier = insn->opcode - insn->spec->modifierBase; - return 0; - case MODIFIER_MODRM: - insn->opcodeModifier = insn->modRM - insn->spec->modifierBase; - return 0; - } -} - -/* * readOpcodeRegister - Reads an operand from the opcode field of an * instruction and interprets it appropriately given the operand width. * Handles AddRegFrm instructions. * - * @param insn - See readOpcodeModifier(). + * @param insn - the instruction whose opcode field is to be read. * @param size - The width (in bytes) of the register being specified. * 1 means AL and friends, 2 means AX, 4 means EAX, and 8 means * RAX. @@ -1446,16 +1498,13 @@ static int readOpcodeModifier(struct InternalInstruction* insn) { static int readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) { dbgprintf(insn, "readOpcodeRegister()"); - if (readOpcodeModifier(insn)) - return -1; - if (size == 0) size = insn->registerSize; switch (size) { case 1: insn->opcodeRegister = (Reg)(MODRM_REG_AL + ((bFromREX(insn->rexPrefix) << 3) - | insn->opcodeModifier)); + | (insn->opcode & 7))); if (insn->rexPrefix && insn->opcodeRegister >= MODRM_REG_AL + 0x4 && insn->opcodeRegister < MODRM_REG_AL + 0x8) { @@ -1467,17 +1516,17 @@ static int readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) { case 2: insn->opcodeRegister = (Reg)(MODRM_REG_AX + ((bFromREX(insn->rexPrefix) << 3) - | insn->opcodeModifier)); + | (insn->opcode & 7))); break; case 4: insn->opcodeRegister = (Reg)(MODRM_REG_EAX + ((bFromREX(insn->rexPrefix) << 3) - | insn->opcodeModifier)); + | (insn->opcode & 7))); break; case 8: insn->opcodeRegister = (Reg)(MODRM_REG_RAX + ((bFromREX(insn->rexPrefix) << 3) - | insn->opcodeModifier)); + | (insn->opcode & 7))); break; } @@ -1550,12 +1599,14 @@ static int readImmediate(struct InternalInstruction* insn, uint8_t size) { static int readVVVV(struct InternalInstruction* insn) { dbgprintf(insn, "readVVVV()"); - if (insn->vexXopType == TYPE_VEX_3B) - insn->vvvv = vvvvFromVEX3of3(insn->vexXopPrefix[2]); - else if (insn->vexXopType == TYPE_VEX_2B) - insn->vvvv = vvvvFromVEX2of2(insn->vexXopPrefix[1]); - else if (insn->vexXopType == TYPE_XOP) - insn->vvvv = vvvvFromXOP3of3(insn->vexXopPrefix[2]); + if (insn->vectorExtensionType == TYPE_EVEX) + insn->vvvv = vvvvFromEVEX3of4(insn->vectorExtensionPrefix[2]); + else if (insn->vectorExtensionType == TYPE_VEX_3B) + insn->vvvv = vvvvFromVEX3of3(insn->vectorExtensionPrefix[2]); + else if (insn->vectorExtensionType == TYPE_VEX_2B) + insn->vvvv = vvvvFromVEX2of2(insn->vectorExtensionPrefix[1]); + else if (insn->vectorExtensionType == TYPE_XOP) + insn->vvvv = vvvvFromXOP3of3(insn->vectorExtensionPrefix[2]); else return -1; @@ -1566,6 +1617,23 @@ static int readVVVV(struct InternalInstruction* insn) { } /* + * readMaskRegister - Reads an mask register from the opcode field of an + * instruction. + * + * @param insn - The instruction whose opcode field is to be read. + * @return - 0 on success; nonzero otherwise. + */ +static int readMaskRegister(struct InternalInstruction* insn) { + dbgprintf(insn, "readMaskRegister()"); + + if (insn->vectorExtensionType != TYPE_EVEX) + return -1; + + insn->writemask = aaaFromEVEX4of4(insn->vectorExtensionPrefix[3]); + return 0; +} + +/* * readOperands - Consults the specifier for an instruction and consumes all * operands for that instruction, interpreting them as it goes. * @@ -1587,6 +1655,8 @@ static int readOperands(struct InternalInstruction* insn) { for (index = 0; index < X86_MAX_OPERANDS; ++index) { switch (x86OperandSets[insn->spec->operands][index].encoding) { case ENCODING_NONE: + case ENCODING_SI: + case ENCODING_DI: break; case ENCODING_REG: case ENCODING_RM: @@ -1664,9 +1734,7 @@ static int readOperands(struct InternalInstruction* insn) { if (readOpcodeRegister(insn, 0)) return -1; break; - case ENCODING_I: - if (readOpcodeModifier(insn)) - return -1; + case ENCODING_FP: break; case ENCODING_VVVV: needVVVV = 0; /* Mark that we have found a VVVV operand. */ @@ -1675,6 +1743,10 @@ static int readOperands(struct InternalInstruction* insn) { if (fixupReg(insn, &x86OperandSets[insn->spec->operands][index])) return -1; break; + case ENCODING_WRITEMASK: + if (readMaskRegister(insn)) + return -1; + break; case ENCODING_DUP: break; default: diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h index 6d03d5c..ac3b39d 100644 --- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h +++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h @@ -45,6 +45,21 @@ extern "C" { #define xFromREX(rex) (((rex) & 0x2) >> 1) #define bFromREX(rex) ((rex) & 0x1) +#define rFromEVEX2of4(evex) (((~(evex)) & 0x80) >> 7) +#define xFromEVEX2of4(evex) (((~(evex)) & 0x40) >> 6) +#define bFromEVEX2of4(evex) (((~(evex)) & 0x20) >> 5) +#define r2FromEVEX2of4(evex) (((~(evex)) & 0x10) >> 4) +#define mmFromEVEX2of4(evex) ((evex) & 0x3) +#define wFromEVEX3of4(evex) (((evex) & 0x80) >> 7) +#define vvvvFromEVEX3of4(evex) (((~(evex)) & 0x78) >> 3) +#define ppFromEVEX3of4(evex) ((evex) & 0x3) +#define zFromEVEX4of4(evex) (((evex) & 0x80) >> 7) +#define l2FromEVEX4of4(evex) (((evex) & 0x40) >> 6) +#define lFromEVEX4of4(evex) (((evex) & 0x20) >> 5) +#define bFromEVEX4of4(evex) (((evex) & 0x10) >> 4) +#define v2FromEVEX4of4(evex) (((~evex) & 0x8) >> 3) +#define aaaFromEVEX4of4(evex) ((evex) & 0x7) + #define rFromVEX2of3(vex) (((~(vex)) & 0x80) >> 7) #define xFromVEX2of3(vex) (((~(vex)) & 0x40) >> 6) #define bFromVEX2of3(vex) (((~(vex)) & 0x20) >> 5) @@ -314,6 +329,16 @@ extern "C" { ENTRY(ZMM30) \ ENTRY(ZMM31) +#define REGS_MASKS \ + ENTRY(K0) \ + ENTRY(K1) \ + ENTRY(K2) \ + ENTRY(K3) \ + ENTRY(K4) \ + ENTRY(K5) \ + ENTRY(K6) \ + ENTRY(K7) + #define REGS_SEGMENT \ ENTRY(ES) \ ENTRY(CS) \ @@ -361,6 +386,7 @@ extern "C" { REGS_XMM \ REGS_YMM \ REGS_ZMM \ + REGS_MASKS \ REGS_SEGMENT \ REGS_DEBUG \ REGS_CONTROL \ @@ -463,7 +489,7 @@ typedef enum { } XOPMapSelect; /* - * VEXPrefixCode - Possible values for the VEX.pp field + * VEXPrefixCode - Possible values for the VEX.pp/EVEX.pp field */ typedef enum { @@ -474,11 +500,12 @@ typedef enum { } VEXPrefixCode; typedef enum { - TYPE_NO_VEX_XOP = 0x0, - TYPE_VEX_2B = 0x1, - TYPE_VEX_3B = 0x2, - TYPE_XOP = 0x3 -} VEXXOPType; + TYPE_NO_VEX_XOP = 0x0, + TYPE_VEX_2B = 0x1, + TYPE_VEX_3B = 0x2, + TYPE_EVEX = 0x3, + TYPE_XOP = 0x4 +} VectorExtensionType; typedef uint8_t BOOL; @@ -536,10 +563,10 @@ struct InternalInstruction { uint8_t prefixPresent[0x100]; /* contains the location (for use with the reader) of the prefix byte */ uint64_t prefixLocations[0x100]; - /* The value of the VEX/XOP prefix, if present */ - uint8_t vexXopPrefix[3]; - /* The length of the VEX prefix (0 if not present) */ - VEXXOPType vexXopType; + /* The value of the vector extension prefix(EVEX/VEX/XOP), if present */ + uint8_t vectorExtensionPrefix[4]; + /* The type of the vector extension prefix */ + VectorExtensionType vectorExtensionType; /* The value of the REX prefix, if present */ uint8_t rexPrefix; /* The location where a mandatory prefix would have to be (i.e., right before @@ -585,6 +612,9 @@ struct InternalInstruction { instructions */ Reg vvvv; + /* The writemask for AVX-512 instructions which is contained in EVEX.aaa */ + Reg writemask; + /* The ModR/M byte, which contains most register operands and some portion of all memory operands */ BOOL consumedModRM; @@ -604,8 +634,6 @@ struct InternalInstruction { uint64_t immediates[2]; /* A register or immediate operand encoded into the opcode */ - BOOL consumedOpcodeModifier; - uint8_t opcodeModifier; Reg opcodeRegister; /* Portions of the ModR/M byte */ diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h index dd1719c..523ae99 100644 --- a/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h +++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h @@ -30,8 +30,6 @@ #define TWOBYTE_SYM x86DisassemblerTwoByteOpcodes #define THREEBYTE38_SYM x86DisassemblerThreeByte38Opcodes #define THREEBYTE3A_SYM x86DisassemblerThreeByte3AOpcodes -#define THREEBYTEA6_SYM x86DisassemblerThreeByteA6Opcodes -#define THREEBYTEA7_SYM x86DisassemblerThreeByteA7Opcodes #define XOP8_MAP_SYM x86DisassemblerXOP8Opcodes #define XOP9_MAP_SYM x86DisassemblerXOP9Opcodes #define XOPA_MAP_SYM x86DisassemblerXOPAOpcodes @@ -42,8 +40,6 @@ #define TWOBYTE_STR "x86DisassemblerTwoByteOpcodes" #define THREEBYTE38_STR "x86DisassemblerThreeByte38Opcodes" #define THREEBYTE3A_STR "x86DisassemblerThreeByte3AOpcodes" -#define THREEBYTEA6_STR "x86DisassemblerThreeByteA6Opcodes" -#define THREEBYTEA7_STR "x86DisassemblerThreeByteA7Opcodes" #define XOP8_MAP_STR "x86DisassemblerXOP8Opcodes" #define XOP9_MAP_STR "x86DisassemblerXOP9Opcodes" #define XOPA_MAP_STR "x86DisassemblerXOPAOpcodes" @@ -53,16 +49,22 @@ * processed correctly. Most of these indicate the presence of particular * prefixes, but ATTR_64BIT is simply an attribute of the decoding context. */ -#define ATTRIBUTE_BITS \ - ENUM_ENTRY(ATTR_NONE, 0x00) \ - ENUM_ENTRY(ATTR_64BIT, 0x01) \ - ENUM_ENTRY(ATTR_XS, 0x02) \ - ENUM_ENTRY(ATTR_XD, 0x04) \ - ENUM_ENTRY(ATTR_REXW, 0x08) \ - ENUM_ENTRY(ATTR_OPSIZE, 0x10) \ - ENUM_ENTRY(ATTR_ADSIZE, 0x20) \ - ENUM_ENTRY(ATTR_VEX, 0x40) \ - ENUM_ENTRY(ATTR_VEXL, 0x80) +#define ATTRIBUTE_BITS \ + ENUM_ENTRY(ATTR_NONE, 0x00) \ + ENUM_ENTRY(ATTR_64BIT, (0x1 << 0)) \ + ENUM_ENTRY(ATTR_XS, (0x1 << 1)) \ + ENUM_ENTRY(ATTR_XD, (0x1 << 2)) \ + ENUM_ENTRY(ATTR_REXW, (0x1 << 3)) \ + ENUM_ENTRY(ATTR_OPSIZE, (0x1 << 4)) \ + ENUM_ENTRY(ATTR_ADSIZE, (0x1 << 5)) \ + ENUM_ENTRY(ATTR_VEX, (0x1 << 6)) \ + ENUM_ENTRY(ATTR_VEXL, (0x1 << 7)) \ + ENUM_ENTRY(ATTR_EVEX, (0x1 << 8)) \ + ENUM_ENTRY(ATTR_EVEXL, (0x1 << 9)) \ + ENUM_ENTRY(ATTR_EVEXL2, (0x1 << 10)) \ + ENUM_ENTRY(ATTR_EVEXK, (0x1 << 11)) \ + ENUM_ENTRY(ATTR_EVEXKZ, (0x1 << 12)) \ + ENUM_ENTRY(ATTR_EVEXB, (0x1 << 13)) #define ENUM_ENTRY(n, v) n = v, enum attributeBits { @@ -73,7 +75,7 @@ enum attributeBits { /* * Combinations of the above attributes that are relevant to instruction - * decode. Although other combinations are possible, they can be reduced to + * decode. Although other combinations are possible, they can be reduced to * these without affecting the ultimately decoded instruction. */ @@ -198,38 +200,38 @@ enum attributeBits { ENUM_ENTRY(IC_EVEX_L2_W_XS_B, 4, "requires EVEX_B, L2, W and XS prefix") \ ENUM_ENTRY(IC_EVEX_L2_W_XD_B, 4, "requires EVEX_B, L2, W and XD prefix") \ ENUM_ENTRY(IC_EVEX_L2_W_OPSIZE_B, 4, "requires EVEX_B, L2, W and OpSize") \ - ENUM_ENTRY(IC_EVEX_K_B, 1, "requires EVEX_B and EVEX_K prefix") \ - ENUM_ENTRY(IC_EVEX_XS_K_B, 2, "requires EVEX_B, EVEX_K and the XS prefix") \ - ENUM_ENTRY(IC_EVEX_XD_K_B, 2, "requires EVEX_B, EVEX_K and the XD prefix") \ - ENUM_ENTRY(IC_EVEX_OPSIZE_K_B, 2, "requires EVEX_B, EVEX_K and the OpSize prefix") \ - ENUM_ENTRY(IC_EVEX_W_K_B, 3, "requires EVEX_B, EVEX_K and the W prefix") \ - ENUM_ENTRY(IC_EVEX_W_XS_K_B, 4, "requires EVEX_B, EVEX_K, W, and XS prefix") \ - ENUM_ENTRY(IC_EVEX_W_XD_K_B, 4, "requires EVEX_B, EVEX_K, W, and XD prefix") \ - ENUM_ENTRY(IC_EVEX_W_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, W, and OpSize") \ - ENUM_ENTRY(IC_EVEX_L_K_B, 3, "requires EVEX_B, EVEX_K and the L prefix") \ - ENUM_ENTRY(IC_EVEX_L_XS_K_B, 4, "requires EVEX_B, EVEX_K and the L and XS prefix")\ - ENUM_ENTRY(IC_EVEX_L_XD_K_B, 4, "requires EVEX_B, EVEX_K and the L and XD prefix")\ - ENUM_ENTRY(IC_EVEX_L_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, L, and OpSize") \ - ENUM_ENTRY(IC_EVEX_L_W_K_B, 3, "requires EVEX_B, EVEX_K, L and W") \ - ENUM_ENTRY(IC_EVEX_L_W_XS_K_B, 4, "requires EVEX_B, EVEX_K, L, W and XS prefix") \ - ENUM_ENTRY(IC_EVEX_L_W_XD_K_B, 4, "requires EVEX_B, EVEX_K, L, W and XD prefix") \ - ENUM_ENTRY(IC_EVEX_L_W_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, L, W and OpSize") \ - ENUM_ENTRY(IC_EVEX_L2_K_B, 3, "requires EVEX_B, EVEX_K and the L2 prefix") \ - ENUM_ENTRY(IC_EVEX_L2_XS_K_B, 4, "requires EVEX_B, EVEX_K and the L2 and XS prefix")\ - ENUM_ENTRY(IC_EVEX_L2_XD_K_B, 4, "requires EVEX_B, EVEX_K and the L2 and XD prefix")\ - ENUM_ENTRY(IC_EVEX_L2_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, L2, and OpSize") \ - ENUM_ENTRY(IC_EVEX_L2_W_K_B, 3, "requires EVEX_B, EVEX_K, L2 and W") \ - ENUM_ENTRY(IC_EVEX_L2_W_XS_K_B, 4, "requires EVEX_B, EVEX_K, L2, W and XS prefix") \ - ENUM_ENTRY(IC_EVEX_L2_W_XD_K_B, 4, "requires EVEX_B, EVEX_K, L2, W and XD prefix") \ - ENUM_ENTRY(IC_EVEX_L2_W_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, L2, W and OpSize") \ - ENUM_ENTRY(IC_EVEX_KZ_B, 1, "requires EVEX_B and EVEX_KZ prefix") \ - ENUM_ENTRY(IC_EVEX_XS_KZ_B, 2, "requires EVEX_B, EVEX_KZ and the XS prefix") \ - ENUM_ENTRY(IC_EVEX_XD_KZ_B, 2, "requires EVEX_B, EVEX_KZ and the XD prefix") \ - ENUM_ENTRY(IC_EVEX_OPSIZE_KZ_B, 2, "requires EVEX_B, EVEX_KZ and the OpSize prefix") \ - ENUM_ENTRY(IC_EVEX_W_KZ_B, 3, "requires EVEX_B, EVEX_KZ and the W prefix") \ - ENUM_ENTRY(IC_EVEX_W_XS_KZ_B, 4, "requires EVEX_B, EVEX_KZ, W, and XS prefix") \ - ENUM_ENTRY(IC_EVEX_W_XD_KZ_B, 4, "requires EVEX_B, EVEX_KZ, W, and XD prefix") \ - ENUM_ENTRY(IC_EVEX_W_OPSIZE_KZ_B, 4, "requires EVEX_B, EVEX_KZ, W, and OpSize") \ + ENUM_ENTRY(IC_EVEX_K_B, 1, "requires EVEX_B and EVEX_K prefix") \ + ENUM_ENTRY(IC_EVEX_XS_K_B, 2, "requires EVEX_B, EVEX_K and the XS prefix") \ + ENUM_ENTRY(IC_EVEX_XD_K_B, 2, "requires EVEX_B, EVEX_K and the XD prefix") \ + ENUM_ENTRY(IC_EVEX_OPSIZE_K_B, 2, "requires EVEX_B, EVEX_K and the OpSize prefix") \ + ENUM_ENTRY(IC_EVEX_W_K_B, 3, "requires EVEX_B, EVEX_K and the W prefix") \ + ENUM_ENTRY(IC_EVEX_W_XS_K_B, 4, "requires EVEX_B, EVEX_K, W, and XS prefix") \ + ENUM_ENTRY(IC_EVEX_W_XD_K_B, 4, "requires EVEX_B, EVEX_K, W, and XD prefix") \ + ENUM_ENTRY(IC_EVEX_W_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, W, and OpSize") \ + ENUM_ENTRY(IC_EVEX_L_K_B, 3, "requires EVEX_B, EVEX_K and the L prefix") \ + ENUM_ENTRY(IC_EVEX_L_XS_K_B, 4, "requires EVEX_B, EVEX_K and the L and XS prefix")\ + ENUM_ENTRY(IC_EVEX_L_XD_K_B, 4, "requires EVEX_B, EVEX_K and the L and XD prefix")\ + ENUM_ENTRY(IC_EVEX_L_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, L, and OpSize") \ + ENUM_ENTRY(IC_EVEX_L_W_K_B, 3, "requires EVEX_B, EVEX_K, L and W") \ + ENUM_ENTRY(IC_EVEX_L_W_XS_K_B, 4, "requires EVEX_B, EVEX_K, L, W and XS prefix") \ + ENUM_ENTRY(IC_EVEX_L_W_XD_K_B, 4, "requires EVEX_B, EVEX_K, L, W and XD prefix") \ + ENUM_ENTRY(IC_EVEX_L_W_OPSIZE_K_B,4, "requires EVEX_B, EVEX_K, L, W and OpSize") \ + ENUM_ENTRY(IC_EVEX_L2_K_B, 3, "requires EVEX_B, EVEX_K and the L2 prefix") \ + ENUM_ENTRY(IC_EVEX_L2_XS_K_B, 4, "requires EVEX_B, EVEX_K and the L2 and XS prefix")\ + ENUM_ENTRY(IC_EVEX_L2_XD_K_B, 4, "requires EVEX_B, EVEX_K and the L2 and XD prefix")\ + ENUM_ENTRY(IC_EVEX_L2_OPSIZE_K_B, 4, "requires EVEX_B, EVEX_K, L2, and OpSize") \ + ENUM_ENTRY(IC_EVEX_L2_W_K_B, 3, "requires EVEX_B, EVEX_K, L2 and W") \ + ENUM_ENTRY(IC_EVEX_L2_W_XS_K_B, 4, "requires EVEX_B, EVEX_K, L2, W and XS prefix") \ + ENUM_ENTRY(IC_EVEX_L2_W_XD_K_B, 4, "requires EVEX_B, EVEX_K, L2, W and XD prefix") \ + ENUM_ENTRY(IC_EVEX_L2_W_OPSIZE_K_B,4, "requires EVEX_B, EVEX_K, L2, W and OpSize") \ + ENUM_ENTRY(IC_EVEX_KZ_B, 1, "requires EVEX_B and EVEX_KZ prefix") \ + ENUM_ENTRY(IC_EVEX_XS_KZ_B, 2, "requires EVEX_B, EVEX_KZ and the XS prefix") \ + ENUM_ENTRY(IC_EVEX_XD_KZ_B, 2, "requires EVEX_B, EVEX_KZ and the XD prefix") \ + ENUM_ENTRY(IC_EVEX_OPSIZE_KZ_B, 2, "requires EVEX_B, EVEX_KZ and the OpSize prefix") \ + ENUM_ENTRY(IC_EVEX_W_KZ_B, 3, "requires EVEX_B, EVEX_KZ and the W prefix") \ + ENUM_ENTRY(IC_EVEX_W_XS_KZ_B, 4, "requires EVEX_B, EVEX_KZ, W, and XS prefix") \ + ENUM_ENTRY(IC_EVEX_W_XD_KZ_B, 4, "requires EVEX_B, EVEX_KZ, W, and XD prefix") \ + ENUM_ENTRY(IC_EVEX_W_OPSIZE_KZ_B, 4, "requires EVEX_B, EVEX_KZ, W, and OpSize") \ ENUM_ENTRY(IC_EVEX_L_KZ_B, 3, "requires EVEX_B, EVEX_KZ and the L prefix") \ ENUM_ENTRY(IC_EVEX_L_XS_KZ_B, 4, "requires EVEX_B, EVEX_KZ and the L and XS prefix")\ ENUM_ENTRY(IC_EVEX_L_XD_KZ_B, 4, "requires EVEX_B, EVEX_KZ and the L and XD prefix")\ @@ -287,11 +289,9 @@ typedef enum { TWOBYTE = 1, THREEBYTE_38 = 2, THREEBYTE_3A = 3, - THREEBYTE_A6 = 4, - THREEBYTE_A7 = 5, - XOP8_MAP = 6, - XOP9_MAP = 7, - XOPA_MAP = 8 + XOP8_MAP = 4, + XOP9_MAP = 5, + XOPA_MAP = 6 } OpcodeType; /* @@ -395,15 +395,17 @@ struct ContextDecision { ENUM_ENTRY(ENCODING_RW, "(AX..DI, R8W..R15W)") \ ENUM_ENTRY(ENCODING_RD, "(EAX..EDI, R8D..R15D)") \ ENUM_ENTRY(ENCODING_RO, "(RAX..RDI, R8..R15)") \ - ENUM_ENTRY(ENCODING_I, "Position on floating-point stack added to the " \ - "opcode byte") \ + ENUM_ENTRY(ENCODING_FP, "Position on floating-point stack in ModR/M " \ + "byte.") \ \ ENUM_ENTRY(ENCODING_Iv, "Immediate of operand size") \ ENUM_ENTRY(ENCODING_Ia, "Immediate of address size") \ ENUM_ENTRY(ENCODING_Rv, "Register code of operand size added to the " \ "opcode byte") \ ENUM_ENTRY(ENCODING_DUP, "Duplicate of another operand; ID is encoded " \ - "in type") + "in type") \ + ENUM_ENTRY(ENCODING_SI, "Source index; encoded in OpSize/Adsize prefix") \ + ENUM_ENTRY(ENCODING_DI, "Destination index; encoded in prefixes") #define ENUM_ENTRY(n, d) n, typedef enum { @@ -454,6 +456,14 @@ struct ContextDecision { ENUM_ENTRY(TYPE_M16_16, "2+2-byte (BOUND)") \ ENUM_ENTRY(TYPE_M32_32, "4+4-byte (BOUND)") \ ENUM_ENTRY(TYPE_M16_64, "2+8-byte (LIDT, LGDT)") \ + ENUM_ENTRY(TYPE_SRCIDX8, "1-byte memory at source index") \ + ENUM_ENTRY(TYPE_SRCIDX16, "2-byte memory at source index") \ + ENUM_ENTRY(TYPE_SRCIDX32, "4-byte memory at source index") \ + ENUM_ENTRY(TYPE_SRCIDX64, "8-byte memory at source index") \ + ENUM_ENTRY(TYPE_DSTIDX8, "1-byte memory at destination index") \ + ENUM_ENTRY(TYPE_DSTIDX16, "2-byte memory at destination index") \ + ENUM_ENTRY(TYPE_DSTIDX32, "4-byte memory at destination index") \ + ENUM_ENTRY(TYPE_DSTIDX64, "8-byte memory at destination index") \ ENUM_ENTRY(TYPE_MOFFS8, "1-byte memory offset (relative to segment " \ "base)") \ ENUM_ENTRY(TYPE_MOFFS16, "2-byte") \ @@ -478,6 +488,7 @@ struct ContextDecision { ENUM_ENTRY(TYPE_XMM128, "16-byte") \ ENUM_ENTRY(TYPE_XMM256, "32-byte") \ ENUM_ENTRY(TYPE_XMM512, "64-byte") \ + ENUM_ENTRY(TYPE_VK1, "1-bit") \ ENUM_ENTRY(TYPE_VK8, "8-bit") \ ENUM_ENTRY(TYPE_VK16, "16-bit") \ ENUM_ENTRY(TYPE_XMM0, "Implicit use of XMM0") \ @@ -518,9 +529,7 @@ struct OperandSpecifier { */ #define MODIFIER_TYPES \ - ENUM_ENTRY(MODIFIER_NONE) \ - ENUM_ENTRY(MODIFIER_OPCODE) \ - ENUM_ENTRY(MODIFIER_MODRM) + ENUM_ENTRY(MODIFIER_NONE) #define ENUM_ENTRY(n) n, typedef enum { @@ -536,9 +545,6 @@ typedef enum { * its operands. */ struct InstructionSpecifier { - uint8_t modifierType; - uint8_t modifierBase; - /* The macro below must be defined wherever this file is included. */ INSTRUCTION_SPECIFIER_FIELDS }; diff --git a/lib/Target/X86/InstPrinter/Android.mk b/lib/Target/X86/InstPrinter/Android.mk index 633df62..9881beb 100644 --- a/lib/Target/X86/InstPrinter/Android.mk +++ b/lib/Target/X86/InstPrinter/Android.mk @@ -14,6 +14,7 @@ x86_instprinter_SRC_FILES := \ # For the device # ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) @@ -32,6 +33,7 @@ LOCAL_MODULE_TAGS := optional include $(LLVM_DEVICE_BUILD_MK) include $(LLVM_TBLGEN_RULES_MK) include $(BUILD_STATIC_LIBRARY) +endif # For the host # ===================================================== diff --git a/lib/Target/X86/InstPrinter/CMakeLists.txt b/lib/Target/X86/InstPrinter/CMakeLists.txt index 28e2460..686a37e 100644 --- a/lib/Target/X86/InstPrinter/CMakeLists.txt +++ b/lib/Target/X86/InstPrinter/CMakeLists.txt @@ -1,9 +1,5 @@ -include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. ) - add_llvm_library(LLVMX86AsmPrinter X86ATTInstPrinter.cpp X86IntelInstPrinter.cpp X86InstComments.cpp ) - -add_dependencies(LLVMX86AsmPrinter X86CommonTableGen) diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp index 4439311..eea0a76 100644 --- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp +++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp @@ -123,6 +123,16 @@ void X86ATTInstPrinter::printAVXCC(const MCInst *MI, unsigned Op, } } +void X86ATTInstPrinter::printRoundingControl(const MCInst *MI, unsigned Op, + raw_ostream &O) { + int64_t Imm = MI->getOperand(Op).getImm() & 0x3; + switch (Imm) { + case 0: O << "{rn-sae}"; break; + case 1: O << "{rd-sae}"; break; + case 2: O << "{ru-sae}"; break; + case 3: O << "{rz-sae}"; break; + } +} /// printPCRelImm - This is used to print an immediate value that ends up /// being encoded as a pc-relative value (e.g. for jumps and calls). These /// print slightly differently than normal immediates. For example, a $ is not @@ -172,16 +182,16 @@ void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo, void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op, raw_ostream &O) { - const MCOperand &BaseReg = MI->getOperand(Op); - const MCOperand &IndexReg = MI->getOperand(Op+2); - const MCOperand &DispSpec = MI->getOperand(Op+3); - const MCOperand &SegReg = MI->getOperand(Op+4); + const MCOperand &BaseReg = MI->getOperand(Op+X86::AddrBaseReg); + const MCOperand &IndexReg = MI->getOperand(Op+X86::AddrIndexReg); + const MCOperand &DispSpec = MI->getOperand(Op+X86::AddrDisp); + const MCOperand &SegReg = MI->getOperand(Op+X86::AddrSegmentReg); O << markup("<mem:"); // If this has a segment register, print it. if (SegReg.getReg()) { - printOperand(MI, Op+4, O); + printOperand(MI, Op+X86::AddrSegmentReg, O); O << ':'; } @@ -197,12 +207,12 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op, if (IndexReg.getReg() || BaseReg.getReg()) { O << '('; if (BaseReg.getReg()) - printOperand(MI, Op, O); + printOperand(MI, Op+X86::AddrBaseReg, O); if (IndexReg.getReg()) { O << ','; - printOperand(MI, Op+2, O); - unsigned ScaleVal = MI->getOperand(Op+1).getImm(); + printOperand(MI, Op+X86::AddrIndexReg, O); + unsigned ScaleVal = MI->getOperand(Op+X86::AddrScaleAmt).getImm(); if (ScaleVal != 1) { O << ',' << markup("<imm:") @@ -216,12 +226,49 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op, O << markup(">"); } +void X86ATTInstPrinter::printSrcIdx(const MCInst *MI, unsigned Op, + raw_ostream &O) { + const MCOperand &SegReg = MI->getOperand(Op+1); + + O << markup("<mem:"); + + // If this has a segment register, print it. + if (SegReg.getReg()) { + printOperand(MI, Op+1, O); + O << ':'; + } + + O << "("; + printOperand(MI, Op, O); + O << ")"; + + O << markup(">"); +} + +void X86ATTInstPrinter::printDstIdx(const MCInst *MI, unsigned Op, + raw_ostream &O) { + O << markup("<mem:"); + + O << "%es:("; + printOperand(MI, Op, O); + O << ")"; + + O << markup(">"); +} + void X86ATTInstPrinter::printMemOffset(const MCInst *MI, unsigned Op, raw_ostream &O) { const MCOperand &DispSpec = MI->getOperand(Op); + const MCOperand &SegReg = MI->getOperand(Op+1); O << markup("<mem:"); + // If this has a segment register, print it. + if (SegReg.getReg()) { + printOperand(MI, Op+1, O); + O << ':'; + } + if (DispSpec.isImm()) { O << formatImm(DispSpec.getImm()); } else { diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h index a8fab72..f34e633 100644 --- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h +++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h @@ -19,15 +19,15 @@ namespace llvm { class MCOperand; - -class X86ATTInstPrinter : public MCInstPrinter { + +class X86ATTInstPrinter final : public MCInstPrinter { public: X86ATTInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII, const MCRegisterInfo &MRI) : MCInstPrinter(MAI, MII, MRI) {} - virtual void printRegName(raw_ostream &OS, unsigned RegNo) const; - virtual void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot); + void printRegName(raw_ostream &OS, unsigned RegNo) const override; + void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot) override; // Autogenerated by tblgen, returns true if we successfully printed an // alias. @@ -42,7 +42,10 @@ public: void printSSECC(const MCInst *MI, unsigned Op, raw_ostream &OS); void printAVXCC(const MCInst *MI, unsigned Op, raw_ostream &OS); void printPCRelImm(const MCInst *MI, unsigned OpNo, raw_ostream &OS); + void printSrcIdx(const MCInst *MI, unsigned OpNo, raw_ostream &OS); + void printDstIdx(const MCInst *MI, unsigned OpNo, raw_ostream &OS); void printMemOffset(const MCInst *MI, unsigned OpNo, raw_ostream &OS); + void printRoundingControl(const MCInst *MI, unsigned Op, raw_ostream &OS); void printopaquemem(const MCInst *MI, unsigned OpNo, raw_ostream &O) { printMemReference(MI, OpNo, O); @@ -88,6 +91,30 @@ public: printMemReference(MI, OpNo, O); } + void printSrcIdx8(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printSrcIdx(MI, OpNo, O); + } + void printSrcIdx16(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printSrcIdx(MI, OpNo, O); + } + void printSrcIdx32(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printSrcIdx(MI, OpNo, O); + } + void printSrcIdx64(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printSrcIdx(MI, OpNo, O); + } + void printDstIdx8(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printDstIdx(MI, OpNo, O); + } + void printDstIdx16(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printDstIdx(MI, OpNo, O); + } + void printDstIdx32(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printDstIdx(MI, OpNo, O); + } + void printDstIdx64(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + printDstIdx(MI, OpNo, O); + } void printMemOffs8(const MCInst *MI, unsigned OpNo, raw_ostream &O) { printMemOffset(MI, OpNo, O); } diff --git a/lib/Target/X86/InstPrinter/X86InstComments.cpp b/lib/Target/X86/InstPrinter/X86InstComments.cpp index 0f6eeb1..db61fb0 100644 --- a/lib/Target/X86/InstPrinter/X86InstComments.cpp +++ b/lib/Target/X86/InstPrinter/X86InstComments.cpp @@ -16,7 +16,9 @@ #include "MCTargetDesc/X86MCTargetDesc.h" #include "Utils/X86ShuffleDecode.h" #include "llvm/MC/MCInst.h" +#include "llvm/CodeGen/MachineValueType.h" #include "llvm/Support/raw_ostream.h" + using namespace llvm; //===----------------------------------------------------------------------===// @@ -38,7 +40,8 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, DestName = getRegName(MI->getOperand(0).getReg()); Src1Name = getRegName(MI->getOperand(1).getReg()); Src2Name = getRegName(MI->getOperand(2).getReg()); - DecodeINSERTPSMask(MI->getOperand(3).getImm(), ShuffleMask); + if(MI->getOperand(3).isImm()) + DecodeINSERTPSMask(MI->getOperand(3).getImm(), ShuffleMask); break; case X86::MOVLHPSrr: @@ -65,9 +68,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::VPALIGNR128rm: Src2Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); - DecodePALIGNRMask(MVT::v16i8, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePALIGNRMask(MVT::v16i8, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::VPALIGNR256rr: Src1Name = getRegName(MI->getOperand(2).getReg()); @@ -75,9 +79,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::VPALIGNR256rm: Src2Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); - DecodePALIGNRMask(MVT::v32i8, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePALIGNRMask(MVT::v32i8, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::PSHUFDri: @@ -87,16 +92,20 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::PSHUFDmi: case X86::VPSHUFDmi: DestName = getRegName(MI->getOperand(0).getReg()); - DecodePSHUFMask(MVT::v4i32, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFMask(MVT::v4i32, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::VPSHUFDYri: Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPSHUFDYmi: DestName = getRegName(MI->getOperand(0).getReg()); - DecodePSHUFMask(MVT::v8i32, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFMask(MVT::v8i32, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; @@ -107,18 +116,20 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::PSHUFHWmi: case X86::VPSHUFHWmi: DestName = getRegName(MI->getOperand(0).getReg()); - DecodePSHUFHWMask(MVT::v8i16, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFHWMask(MVT::v8i16, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::VPSHUFHWYri: Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPSHUFHWYmi: DestName = getRegName(MI->getOperand(0).getReg()); - DecodePSHUFHWMask(MVT::v16i16, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFHWMask(MVT::v16i16, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::PSHUFLWri: case X86::VPSHUFLWri: @@ -127,18 +138,20 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::PSHUFLWmi: case X86::VPSHUFLWmi: DestName = getRegName(MI->getOperand(0).getReg()); - DecodePSHUFLWMask(MVT::v8i16, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFLWMask(MVT::v8i16, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::VPSHUFLWYri: Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPSHUFLWYmi: DestName = getRegName(MI->getOperand(0).getReg()); - DecodePSHUFLWMask(MVT::v16i16, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFLWMask(MVT::v16i16, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); break; case X86::PUNPCKHBWrr: @@ -293,8 +306,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, // FALL THROUGH. case X86::SHUFPDrmi: case X86::VSHUFPDrmi: - DecodeSHUFPMask(MVT::v2f64, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodeSHUFPMask(MVT::v2f64, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); Src1Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); break; @@ -302,8 +317,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, Src2Name = getRegName(MI->getOperand(2).getReg()); // FALL THROUGH. case X86::VSHUFPDYrmi: - DecodeSHUFPMask(MVT::v4f64, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodeSHUFPMask(MVT::v4f64, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); Src1Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); break; @@ -314,8 +331,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, // FALL THROUGH. case X86::SHUFPSrmi: case X86::VSHUFPSrmi: - DecodeSHUFPMask(MVT::v4f32, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodeSHUFPMask(MVT::v4f32, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); Src1Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); break; @@ -323,8 +342,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, Src2Name = getRegName(MI->getOperand(2).getReg()); // FALL THROUGH. case X86::VSHUFPSYrmi: - DecodeSHUFPMask(MVT::v8f32, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodeSHUFPMask(MVT::v8f32, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); Src1Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); break; @@ -405,32 +426,40 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPERMILPSmi: - DecodePSHUFMask(MVT::v4f32, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFMask(MVT::v4f32, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); DestName = getRegName(MI->getOperand(0).getReg()); break; case X86::VPERMILPSYri: Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPERMILPSYmi: - DecodePSHUFMask(MVT::v8f32, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFMask(MVT::v8f32, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); DestName = getRegName(MI->getOperand(0).getReg()); break; case X86::VPERMILPDri: Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPERMILPDmi: - DecodePSHUFMask(MVT::v2f64, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFMask(MVT::v2f64, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); DestName = getRegName(MI->getOperand(0).getReg()); break; case X86::VPERMILPDYri: Src1Name = getRegName(MI->getOperand(1).getReg()); // FALL THROUGH. case X86::VPERMILPDYmi: - DecodePSHUFMask(MVT::v4f64, MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodePSHUFMask(MVT::v4f64, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); DestName = getRegName(MI->getOperand(0).getReg()); break; case X86::VPERM2F128rr: @@ -440,9 +469,10 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::VPERM2F128rm: case X86::VPERM2I128rm: // For instruction comments purpose, assume the 256-bit vector is v4i64. - DecodeVPERM2X128Mask(MVT::v4i64, - MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodeVPERM2X128Mask(MVT::v4i64, + MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); Src1Name = getRegName(MI->getOperand(1).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); break; @@ -452,8 +482,9 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, // FALL THROUGH. case X86::VPERMQYmi: case X86::VPERMPDYmi: - DecodeVPERMMask(MI->getOperand(MI->getNumOperands()-1).getImm(), - ShuffleMask); + if(MI->getOperand(MI->getNumOperands()-1).isImm()) + DecodeVPERMMask(MI->getOperand(MI->getNumOperands()-1).getImm(), + ShuffleMask); DestName = getRegName(MI->getOperand(0).getReg()); break; } diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp index e7e7b15..1c95d37 100644 --- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp +++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp @@ -113,6 +113,17 @@ void X86IntelInstPrinter::printAVXCC(const MCInst *MI, unsigned Op, } } +void X86IntelInstPrinter::printRoundingControl(const MCInst *MI, unsigned Op, + raw_ostream &O) { + int64_t Imm = MI->getOperand(Op).getImm() & 0x3; + switch (Imm) { + case 0: O << "{rn-sae}"; break; + case 1: O << "{rd-sae}"; break; + case 2: O << "{ru-sae}"; break; + case 3: O << "{rz-sae}"; break; + } +} + /// printPCRelImm - This is used to print an immediate value that ends up /// being encoded as a pc-relative value. void X86IntelInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo, @@ -151,15 +162,15 @@ void X86IntelInstPrinter::printOperand(const MCInst *MI, unsigned OpNo, void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op, raw_ostream &O) { - const MCOperand &BaseReg = MI->getOperand(Op); - unsigned ScaleVal = MI->getOperand(Op+1).getImm(); - const MCOperand &IndexReg = MI->getOperand(Op+2); - const MCOperand &DispSpec = MI->getOperand(Op+3); - const MCOperand &SegReg = MI->getOperand(Op+4); + const MCOperand &BaseReg = MI->getOperand(Op+X86::AddrBaseReg); + unsigned ScaleVal = MI->getOperand(Op+X86::AddrScaleAmt).getImm(); + const MCOperand &IndexReg = MI->getOperand(Op+X86::AddrIndexReg); + const MCOperand &DispSpec = MI->getOperand(Op+X86::AddrDisp); + const MCOperand &SegReg = MI->getOperand(Op+X86::AddrSegmentReg); // If this has a segment register, print it. if (SegReg.getReg()) { - printOperand(MI, Op+4, O); + printOperand(MI, Op+X86::AddrSegmentReg, O); O << ':'; } @@ -167,7 +178,7 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op, bool NeedPlus = false; if (BaseReg.getReg()) { - printOperand(MI, Op, O); + printOperand(MI, Op+X86::AddrBaseReg, O); NeedPlus = true; } @@ -175,7 +186,7 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op, if (NeedPlus) O << " + "; if (ScaleVal != 1) O << ScaleVal << '*'; - printOperand(MI, Op+2, O); + printOperand(MI, Op+X86::AddrIndexReg, O); NeedPlus = true; } @@ -201,9 +212,38 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op, O << ']'; } +void X86IntelInstPrinter::printSrcIdx(const MCInst *MI, unsigned Op, + raw_ostream &O) { + const MCOperand &SegReg = MI->getOperand(Op+1); + + // If this has a segment register, print it. + if (SegReg.getReg()) { + printOperand(MI, Op+1, O); + O << ':'; + } + O << '['; + printOperand(MI, Op, O); + O << ']'; +} + +void X86IntelInstPrinter::printDstIdx(const MCInst *MI, unsigned Op, + raw_ostream &O) { + // DI accesses are always ES-based. + O << "es:["; + printOperand(MI, Op, O); + O << ']'; +} + void X86IntelInstPrinter::printMemOffset(const MCInst *MI, unsigned Op, raw_ostream &O) { const MCOperand &DispSpec = MI->getOperand(Op); + const MCOperand &SegReg = MI->getOperand(Op+1); + + // If this has a segment register, print it. + if (SegReg.getReg()) { + printOperand(MI, Op+1, O); + O << ':'; + } O << '['; diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h index 590bf68..4d9b481 100644 --- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h +++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h @@ -20,16 +20,16 @@ namespace llvm { class MCOperand; - -class X86IntelInstPrinter : public MCInstPrinter { + +class X86IntelInstPrinter final : public MCInstPrinter { public: X86IntelInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII, const MCRegisterInfo &MRI) : MCInstPrinter(MAI, MII, MRI) {} - virtual void printRegName(raw_ostream &OS, unsigned RegNo) const; - virtual void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot); - + void printRegName(raw_ostream &OS, unsigned RegNo) const override; + void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot) override; + // Autogenerated by tblgen. void printInstruction(const MCInst *MI, raw_ostream &O); static const char *getRegisterName(unsigned RegNo); @@ -40,6 +40,9 @@ public: void printAVXCC(const MCInst *MI, unsigned Op, raw_ostream &O); void printPCRelImm(const MCInst *MI, unsigned OpNo, raw_ostream &O); void printMemOffset(const MCInst *MI, unsigned OpNo, raw_ostream &O); + void printSrcIdx(const MCInst *MI, unsigned OpNo, raw_ostream &O); + void printDstIdx(const MCInst *MI, unsigned OpNo, raw_ostream &O); + void printRoundingControl(const MCInst *MI, unsigned Op, raw_ostream &OS); void printopaquemem(const MCInst *MI, unsigned OpNo, raw_ostream &O) { O << "opaque ptr "; @@ -99,6 +102,39 @@ public: printMemReference(MI, OpNo, O); } + + void printSrcIdx8(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "byte ptr "; + printSrcIdx(MI, OpNo, O); + } + void printSrcIdx16(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "word ptr "; + printSrcIdx(MI, OpNo, O); + } + void printSrcIdx32(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "dword ptr "; + printSrcIdx(MI, OpNo, O); + } + void printSrcIdx64(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "qword ptr "; + printSrcIdx(MI, OpNo, O); + } + void printDstIdx8(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "byte ptr "; + printDstIdx(MI, OpNo, O); + } + void printDstIdx16(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "word ptr "; + printDstIdx(MI, OpNo, O); + } + void printDstIdx32(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "dword ptr "; + printDstIdx(MI, OpNo, O); + } + void printDstIdx64(const MCInst *MI, unsigned OpNo, raw_ostream &O) { + O << "qword ptr "; + printDstIdx(MI, OpNo, O); + } void printMemOffs8(const MCInst *MI, unsigned OpNo, raw_ostream &O) { O << "byte ptr "; printMemOffset(MI, OpNo, O); diff --git a/lib/Target/X86/MCTargetDesc/Android.mk b/lib/Target/X86/MCTargetDesc/Android.mk index 65cd8df..ee37c27 100644 --- a/lib/Target/X86/MCTargetDesc/Android.mk +++ b/lib/Target/X86/MCTargetDesc/Android.mk @@ -36,6 +36,7 @@ include $(BUILD_HOST_STATIC_LIBRARY) # For the device only # ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) @@ -52,3 +53,4 @@ include $(LLVM_DEVICE_BUILD_MK) include $(LLVM_TBLGEN_RULES_MK) include $(LLVM_GEN_INTRINSICS_MK) include $(BUILD_STATIC_LIBRARY) +endif diff --git a/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/lib/Target/X86/MCTargetDesc/CMakeLists.txt index 2eb5f25..3f5a0e2 100644 --- a/lib/Target/X86/MCTargetDesc/CMakeLists.txt +++ b/lib/Target/X86/MCTargetDesc/CMakeLists.txt @@ -9,8 +9,3 @@ add_llvm_library(LLVMX86Desc X86MachORelocationInfo.cpp X86ELFRelocationInfo.cpp ) - -add_dependencies(LLVMX86Desc X86CommonTableGen) - -# Hack: we need to include 'main' target directory to grab private headers -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/..) diff --git a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index f8e359b..23763f7 100644 --- a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -79,11 +79,11 @@ public: CPU != "c3" && CPU != "c3-2"; } - unsigned getNumFixupKinds() const { + unsigned getNumFixupKinds() const override { return X86::NumTargetFixupKinds; } - const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const { + const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override { const static MCFixupKindInfo Infos[X86::NumTargetFixupKinds] = { { "reloc_riprel_4byte", 0, 4 * 8, MCFixupKindInfo::FKF_IsPCRel }, { "reloc_riprel_4byte_movq_load", 0, 4 * 8, MCFixupKindInfo::FKF_IsPCRel}, @@ -100,7 +100,7 @@ public: } void applyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize, - uint64_t Value) const { + uint64_t Value, bool IsPCRel) const override { unsigned Size = 1 << getFixupKindLog2Size(Fixup.getKind()); assert(Fixup.getOffset() + Size <= DataSize && @@ -117,16 +117,15 @@ public: Data[Fixup.getOffset() + i] = uint8_t(Value >> (i * 8)); } - bool mayNeedRelaxation(const MCInst &Inst) const; + bool mayNeedRelaxation(const MCInst &Inst) const override; - bool fixupNeedsRelaxation(const MCFixup &Fixup, - uint64_t Value, + bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, - const MCAsmLayout &Layout) const; + const MCAsmLayout &Layout) const override; - void relaxInstruction(const MCInst &Inst, MCInst &Res) const; + void relaxInstruction(const MCInst &Inst, MCInst &Res) const override; - bool writeNopData(uint64_t Count, MCObjectWriter *OW) const; + bool writeNopData(uint64_t Count, MCObjectWriter *OW) const override; }; } // end anonymous namespace @@ -217,9 +216,9 @@ static unsigned getRelaxedOpcodeArith(unsigned Op) { case X86::CMP64mi8: return X86::CMP64mi32; // PUSH - case X86::PUSHi8: return X86::PUSHi32; - case X86::PUSHi16: return X86::PUSHi32; - case X86::PUSH64i8: return X86::PUSH64i32; + case X86::PUSH32i8: return X86::PUSHi32; + case X86::PUSH16i8: return X86::PUSHi16; + case X86::PUSH64i8: return X86::PUSH64i32; case X86::PUSH64i16: return X86::PUSH64i32; } } @@ -314,7 +313,7 @@ bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const { {0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, }; - // This CPU doesnt support long nops. If needed add more. + // This CPU doesn't support long nops. If needed add more. // FIXME: Can we get this from the subtarget somehow? // FIXME: We could generated something better than plain 0x90. if (!HasNopl) { @@ -347,14 +346,7 @@ class ELFX86AsmBackend : public X86AsmBackend { public: uint8_t OSABI; ELFX86AsmBackend(const Target &T, uint8_t _OSABI, StringRef CPU) - : X86AsmBackend(T, CPU), OSABI(_OSABI) { - HasReliableSymbolDifference = true; - } - - virtual bool doesSectionRequireSymbols(const MCSection &Section) const { - const MCSectionELF &ES = static_cast<const MCSectionELF&>(Section); - return ES.getFlags() & ELF::SHF_MERGE; - } + : X86AsmBackend(T, CPU), OSABI(_OSABI) {} }; class ELFX86_32AsmBackend : public ELFX86AsmBackend { @@ -362,7 +354,7 @@ public: ELFX86_32AsmBackend(const Target &T, uint8_t OSABI, StringRef CPU) : ELFX86AsmBackend(T, OSABI, CPU) {} - MCObjectWriter *createObjectWriter(raw_ostream &OS) const { + MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { return createX86ELFObjectWriter(OS, /*IsELF64*/ false, OSABI, ELF::EM_386); } }; @@ -372,7 +364,7 @@ public: ELFX86_64AsmBackend(const Target &T, uint8_t OSABI, StringRef CPU) : ELFX86AsmBackend(T, OSABI, CPU) {} - MCObjectWriter *createObjectWriter(raw_ostream &OS) const { + MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { return createX86ELFObjectWriter(OS, /*IsELF64*/ true, OSABI, ELF::EM_X86_64); } }; @@ -386,7 +378,7 @@ public: , Is64Bit(is64Bit) { } - MCObjectWriter *createObjectWriter(raw_ostream &OS) const { + MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { return createX86WinCOFFObjectWriter(OS, Is64Bit); } }; @@ -725,15 +717,15 @@ public: StringRef CPU, bool SupportsCU) : DarwinX86AsmBackend(T, MRI, CPU, false), SupportsCU(SupportsCU) {} - MCObjectWriter *createObjectWriter(raw_ostream &OS) const { + MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { return createX86MachObjectWriter(OS, /*Is64Bit=*/false, MachO::CPU_TYPE_I386, MachO::CPU_SUBTYPE_I386_ALL); } /// \brief Generate the compact unwind encoding for the CFI instructions. - virtual uint32_t - generateCompactUnwindEncoding(ArrayRef<MCCFIInstruction> Instrs) const { + uint32_t generateCompactUnwindEncoding( + ArrayRef<MCCFIInstruction> Instrs) const override { return SupportsCU ? generateCompactUnwindEncodingImpl(Instrs) : 0; } }; @@ -747,15 +739,14 @@ public: MachO::CPUSubTypeX86 st) : DarwinX86AsmBackend(T, MRI, CPU, true), SupportsCU(SupportsCU), Subtype(st) { - HasReliableSymbolDifference = true; } - MCObjectWriter *createObjectWriter(raw_ostream &OS) const { + MCObjectWriter *createObjectWriter(raw_ostream &OS) const override { return createX86MachObjectWriter(OS, /*Is64Bit=*/true, MachO::CPU_TYPE_X86_64, Subtype); } - virtual bool doesSectionRequireSymbols(const MCSection &Section) const { + bool doesSectionRequireSymbols(const MCSection &Section) const override { // Temporary labels in the string literals sections require symbols. The // issue is that the x86_64 relocation format does not allow symbol + // offset, and so the linker does not have enough information to resolve the @@ -765,32 +756,32 @@ public: // // See <rdar://problem/4765733>. const MCSectionMachO &SMO = static_cast<const MCSectionMachO&>(Section); - return SMO.getType() == MCSectionMachO::S_CSTRING_LITERALS; + return SMO.getType() == MachO::S_CSTRING_LITERALS; } - virtual bool isSectionAtomizable(const MCSection &Section) const { + bool isSectionAtomizable(const MCSection &Section) const override { const MCSectionMachO &SMO = static_cast<const MCSectionMachO&>(Section); // Fixed sized data sections are uniqued, they cannot be diced into atoms. switch (SMO.getType()) { default: return true; - case MCSectionMachO::S_4BYTE_LITERALS: - case MCSectionMachO::S_8BYTE_LITERALS: - case MCSectionMachO::S_16BYTE_LITERALS: - case MCSectionMachO::S_LITERAL_POINTERS: - case MCSectionMachO::S_NON_LAZY_SYMBOL_POINTERS: - case MCSectionMachO::S_LAZY_SYMBOL_POINTERS: - case MCSectionMachO::S_MOD_INIT_FUNC_POINTERS: - case MCSectionMachO::S_MOD_TERM_FUNC_POINTERS: - case MCSectionMachO::S_INTERPOSING: + case MachO::S_4BYTE_LITERALS: + case MachO::S_8BYTE_LITERALS: + case MachO::S_16BYTE_LITERALS: + case MachO::S_LITERAL_POINTERS: + case MachO::S_NON_LAZY_SYMBOL_POINTERS: + case MachO::S_LAZY_SYMBOL_POINTERS: + case MachO::S_MOD_INIT_FUNC_POINTERS: + case MachO::S_MOD_TERM_FUNC_POINTERS: + case MachO::S_INTERPOSING: return false; } } /// \brief Generate the compact unwind encoding for the CFI instructions. - virtual uint32_t - generateCompactUnwindEncoding(ArrayRef<MCCFIInstruction> Instrs) const { + uint32_t generateCompactUnwindEncoding( + ArrayRef<MCCFIInstruction> Instrs) const override { return SupportsCU ? generateCompactUnwindEncodingImpl(Instrs) : 0; } }; @@ -803,12 +794,12 @@ MCAsmBackend *llvm::createX86_32AsmBackend(const Target &T, StringRef CPU) { Triple TheTriple(TT); - if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO) + if (TheTriple.isOSBinFormatMachO()) return new DarwinX86_32AsmBackend(T, MRI, CPU, TheTriple.isMacOSX() && !TheTriple.isMacOSXVersionLT(10, 7)); - if (TheTriple.isOSWindows() && TheTriple.getEnvironment() != Triple::ELF) + if (TheTriple.isOSWindows() && !TheTriple.isOSBinFormatELF()) return new WindowsX86AsmBackend(T, false, CPU); uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS()); @@ -821,7 +812,7 @@ MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T, StringRef CPU) { Triple TheTriple(TT); - if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO) { + if (TheTriple.isOSBinFormatMachO()) { MachO::CPUSubTypeX86 CS = StringSwitch<MachO::CPUSubTypeX86>(TheTriple.getArchName()) .Case("x86_64h", MachO::CPU_SUBTYPE_X86_64_H) @@ -831,7 +822,7 @@ MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T, !TheTriple.isMacOSXVersionLT(10, 7), CS); } - if (TheTriple.isOSWindows() && TheTriple.getEnvironment() != Triple::ELF) + if (TheTriple.isOSWindows() && !TheTriple.isOSBinFormatELF()) return new WindowsX86AsmBackend(T, true, CPU); uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS()); diff --git a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h index 1ef9814..38fab15 100644 --- a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -18,9 +18,9 @@ #define X86BASEINFO_H #include "X86MCTargetDesc.h" +#include "llvm/MC/MCInstrDesc.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/MC/MCInstrInfo.h" namespace llvm { @@ -255,6 +255,38 @@ namespace X86II { /// MRMSrcMem = 6, + /// RawFrmMemOffs - This form is for instructions that store an absolute + /// memory offset as an immediate with a possible segment override. + RawFrmMemOffs = 7, + + /// RawFrmSrc - This form is for instructions that use the source index + /// register SI/ESI/RSI with a possible segment override. + RawFrmSrc = 8, + + /// RawFrmDst - This form is for instructions that use the destination index + /// register DI/EDI/ESI. + RawFrmDst = 9, + + /// RawFrmSrc - This form is for instructions that use the the source index + /// register SI/ESI/ERI with a possible segment override, and also the + /// destination index register DI/ESI/RDI. + RawFrmDstSrc = 10, + + /// RawFrmImm8 - This is used for the ENTER instruction, which has two + /// immediates, the first of which is a 16-bit immediate (specified by + /// the imm encoding) and the second is a 8-bit fixed value. + RawFrmImm8 = 11, + + /// RawFrmImm16 - This is used for CALL FAR instructions, which have two + /// immediates, the first of which is a 16 or 32-bit immediate (specified by + /// the imm encoding) and the second is a 16-bit fixed value. In the AMD + /// manual, this operand is described as pntr16:32 and pntr16:16 + RawFrmImm16 = 12, + + /// MRMX[rm] - The forms are used to represent instructions that use a + /// Mod/RM byte, and don't use the middle field for anything. + MRMXr = 14, MRMXm = 15, + /// MRM[0-7][rm] - These forms are used to represent instructions that use /// a Mod/RM byte, and use the middle field to hold extended opcode /// information. In the intel manual these are represented as /0, /1, ... @@ -268,94 +300,83 @@ namespace X86II { MRM0m = 24, MRM1m = 25, MRM2m = 26, MRM3m = 27, // Format /0 /1 /2 /3 MRM4m = 28, MRM5m = 29, MRM6m = 30, MRM7m = 31, // Format /4 /5 /6 /7 - // MRMInitReg - This form is used for instructions whose source and - // destinations are the same register. - MRMInitReg = 32, - //// MRM_XX - A mod/rm byte of exactly 0xXX. - MRM_C1 = 33, MRM_C2 = 34, MRM_C3 = 35, MRM_C4 = 36, - MRM_C8 = 37, MRM_C9 = 38, MRM_CA = 39, MRM_CB = 40, - MRM_E8 = 41, MRM_F0 = 42, MRM_F8 = 45, MRM_F9 = 46, - MRM_D0 = 47, MRM_D1 = 48, MRM_D4 = 49, MRM_D5 = 50, - MRM_D6 = 51, MRM_D8 = 52, MRM_D9 = 53, MRM_DA = 54, - MRM_DB = 55, MRM_DC = 56, MRM_DD = 57, MRM_DE = 58, - MRM_DF = 59, - - /// RawFrmImm8 - This is used for the ENTER instruction, which has two - /// immediates, the first of which is a 16-bit immediate (specified by - /// the imm encoding) and the second is a 8-bit fixed value. - RawFrmImm8 = 43, - - /// RawFrmImm16 - This is used for CALL FAR instructions, which have two - /// immediates, the first of which is a 16 or 32-bit immediate (specified by - /// the imm encoding) and the second is a 16-bit fixed value. In the AMD - /// manual, this operand is described as pntr16:32 and pntr16:16 - RawFrmImm16 = 44, - - FormMask = 63, + MRM_C0 = 32, MRM_C1 = 33, MRM_C2 = 34, MRM_C3 = 35, + MRM_C4 = 36, MRM_C8 = 37, MRM_C9 = 38, MRM_CA = 39, + MRM_CB = 40, MRM_D0 = 41, MRM_D1 = 42, MRM_D4 = 43, + MRM_D5 = 44, MRM_D6 = 45, MRM_D8 = 46, MRM_D9 = 47, + MRM_DA = 48, MRM_DB = 49, MRM_DC = 50, MRM_DD = 51, + MRM_DE = 52, MRM_DF = 53, MRM_E0 = 54, MRM_E1 = 55, + MRM_E2 = 56, MRM_E3 = 57, MRM_E4 = 58, MRM_E5 = 59, + MRM_E8 = 60, MRM_E9 = 61, MRM_EA = 62, MRM_EB = 63, + MRM_EC = 64, MRM_ED = 65, MRM_EE = 66, MRM_F0 = 67, + MRM_F1 = 68, MRM_F2 = 69, MRM_F3 = 70, MRM_F4 = 71, + MRM_F5 = 72, MRM_F6 = 73, MRM_F7 = 74, MRM_F8 = 75, + MRM_F9 = 76, MRM_FA = 77, MRM_FB = 78, MRM_FC = 79, + MRM_FD = 80, MRM_FE = 81, MRM_FF = 82, + + FormMask = 127, //===------------------------------------------------------------------===// // Actual flags... - // OpSize - Set if this instruction requires an operand size prefix (0x66), - // which most often indicates that the instruction operates on 16 bit data - // instead of 32 bit data. - OpSize = 1 << 6, + // OpSize - OpSizeFixed implies instruction never needs a 0x66 prefix. + // OpSize16 means this is a 16-bit instruction and needs 0x66 prefix in + // 32-bit mode. OpSize32 means this is a 32-bit instruction needs a 0x66 + // prefix in 16-bit mode. + OpSizeShift = 7, + OpSizeMask = 0x3 << OpSizeShift, + + OpSize16 = 1, + OpSize32 = 2, // AsSize - Set if this instruction requires an operand size prefix (0x67), // which most often indicates that the instruction address 16 bit address // instead of 32 bit address (or 32 bit address in 64 bit mode). - AdSize = 1 << 7, + AdSizeShift = OpSizeShift + 2, + AdSize = 1 << AdSizeShift, //===------------------------------------------------------------------===// - // Op0Mask - There are several prefix bytes that are used to form two byte - // opcodes. These are currently 0x0F, 0xF3, and 0xD8-0xDF. This mask is - // used to obtain the setting of this field. If no bits in this field is - // set, there is no prefix byte for obtaining a multibyte opcode. + // OpPrefix - There are several prefix bytes that are used as opcode + // extensions. These are 0x66, 0xF3, and 0xF2. If this field is 0 there is + // no prefix. // - Op0Shift = 8, - Op0Mask = 0x1F << Op0Shift, - - // TB - TwoByte - Set if this instruction has a two byte opcode, which - // starts with a 0x0F byte before the real opcode. - TB = 1 << Op0Shift, - - // REP - The 0xF3 prefix byte indicating repetition of the following - // instruction. - REP = 2 << Op0Shift, + OpPrefixShift = AdSizeShift + 1, + OpPrefixMask = 0x7 << OpPrefixShift, - // D8-DF - These escape opcodes are used by the floating point unit. These - // values must remain sequential. - D8 = 3 << Op0Shift, D9 = 4 << Op0Shift, - DA = 5 << Op0Shift, DB = 6 << Op0Shift, - DC = 7 << Op0Shift, DD = 8 << Op0Shift, - DE = 9 << Op0Shift, DF = 10 << Op0Shift, + // PS, PD - Prefix code for packed single and double precision vector + // floating point operations performed in the SSE registers. + PS = 1 << OpPrefixShift, PD = 2 << OpPrefixShift, // XS, XD - These prefix codes are for single and double precision scalar // floating point operations performed in the SSE registers. - XD = 11 << Op0Shift, XS = 12 << Op0Shift, + XS = 3 << OpPrefixShift, XD = 4 << OpPrefixShift, - // T8, TA, A6, A7 - Prefix after the 0x0F prefix. - T8 = 13 << Op0Shift, TA = 14 << Op0Shift, - A6 = 15 << Op0Shift, A7 = 16 << Op0Shift, + //===------------------------------------------------------------------===// + // OpMap - This field determines which opcode map this instruction + // belongs to. i.e. one-byte, two-byte, 0x0f 0x38, 0x0f 0x3a, etc. + // + OpMapShift = OpPrefixShift + 3, + OpMapMask = 0x7 << OpMapShift, - // T8XD - Prefix before and after 0x0F. Combination of T8 and XD. - T8XD = 17 << Op0Shift, + // OB - OneByte - Set if this instruction has a one byte opcode. + OB = 0 << OpMapShift, - // T8XS - Prefix before and after 0x0F. Combination of T8 and XS. - T8XS = 18 << Op0Shift, + // TB - TwoByte - Set if this instruction has a two byte opcode, which + // starts with a 0x0F byte before the real opcode. + TB = 1 << OpMapShift, - // TAXD - Prefix before and after 0x0F. Combination of TA and XD. - TAXD = 19 << Op0Shift, + // T8, TA - Prefix after the 0x0F prefix. + T8 = 2 << OpMapShift, TA = 3 << OpMapShift, // XOP8 - Prefix to include use of imm byte. - XOP8 = 20 << Op0Shift, + XOP8 = 4 << OpMapShift, // XOP9 - Prefix to exclude use of imm byte. - XOP9 = 21 << Op0Shift, + XOP9 = 5 << OpMapShift, // XOPA - Prefix to encode 0xA in VEX.MMMM of XOP instructions. - XOPA = 22 << Op0Shift, + XOPA = 6 << OpMapShift, //===------------------------------------------------------------------===// // REX_W - REX prefixes are instruction prefixes used in 64-bit mode. @@ -363,27 +384,28 @@ namespace X86II { // etc. We only cares about REX.W and REX.R bits and only the former is // statically determined. // - REXShift = Op0Shift + 5, + REXShift = OpMapShift + 3, REX_W = 1 << REXShift, //===------------------------------------------------------------------===// // This three-bit field describes the size of an immediate operand. Zero is // unused so that we can tell if we forgot to set a value. ImmShift = REXShift + 1, - ImmMask = 7 << ImmShift, + ImmMask = 15 << ImmShift, Imm8 = 1 << ImmShift, Imm8PCRel = 2 << ImmShift, Imm16 = 3 << ImmShift, Imm16PCRel = 4 << ImmShift, Imm32 = 5 << ImmShift, Imm32PCRel = 6 << ImmShift, - Imm64 = 7 << ImmShift, + Imm32S = 7 << ImmShift, + Imm64 = 8 << ImmShift, //===------------------------------------------------------------------===// // FP Instruction Classification... Zero is non-fp instruction. // FPTypeMask - Mask for all of the FP types... - FPTypeShift = ImmShift + 3, + FPTypeShift = ImmShift + 4, FPTypeMask = 7 << FPTypeShift, // NotFP - The default, set for instructions that do not use FP registers. @@ -419,51 +441,64 @@ namespace X86II { LOCKShift = FPTypeShift + 3, LOCK = 1 << LOCKShift, - // Segment override prefixes. Currently we just need ability to address - // stuff in gs and fs segments. - SegOvrShift = LOCKShift + 1, - SegOvrMask = 3 << SegOvrShift, - FS = 1 << SegOvrShift, - GS = 2 << SegOvrShift, + // REP prefix + REPShift = LOCKShift + 1, + REP = 1 << REPShift, + + // Execution domain for SSE instructions. + // 0 means normal, non-SSE instruction. + SSEDomainShift = REPShift + 1, + + // Encoding + EncodingShift = SSEDomainShift + 2, + EncodingMask = 0x3 << EncodingShift, + + // VEX - encoding using 0xC4/0xC5 + VEX = 1, + + /// XOP - Opcode prefix used by XOP instructions. + XOP = 2, - // Execution domain for SSE instructions in bits 23, 24. - // 0 in bits 23-24 means normal, non-SSE instruction. - SSEDomainShift = SegOvrShift + 2, + // VEX_EVEX - Specifies that this instruction use EVEX form which provides + // syntax support up to 32 512-bit register operands and up to 7 16-bit + // mask operands as well as source operand data swizzling/memory operand + // conversion, eviction hint, and rounding mode. + EVEX = 3, - OpcodeShift = SSEDomainShift + 2, + // Opcode + OpcodeShift = EncodingShift + 2, //===------------------------------------------------------------------===// /// VEX - The opcode prefix used by AVX instructions VEXShift = OpcodeShift + 8, - VEX = 1U << 0, /// VEX_W - Has a opcode specific functionality, but is used in the same /// way as REX_W is for regular SSE instructions. - VEX_W = 1U << 1, + VEX_W = 1U << 0, /// VEX_4V - Used to specify an additional AVX/SSE register. Several 2 /// address instructions in SSE are represented as 3 address ones in AVX /// and the additional register is encoded in VEX_VVVV prefix. - VEX_4V = 1U << 2, + VEX_4V = 1U << 1, /// VEX_4VOp3 - Similar to VEX_4V, but used on instructions that encode /// operand 3 with VEX.vvvv. - VEX_4VOp3 = 1U << 3, + VEX_4VOp3 = 1U << 2, /// VEX_I8IMM - Specifies that the last register used in a AVX instruction, /// must be encoded in the i8 immediate field. This usually happens in /// instructions with 4 operands. - VEX_I8IMM = 1U << 4, + VEX_I8IMM = 1U << 3, /// VEX_L - Stands for a bit in the VEX opcode prefix meaning the current /// instruction uses 256-bit wide registers. This is usually auto detected /// if a VR256 register is used, but some AVX instructions also have this /// field marked when using a f256 memory references. - VEX_L = 1U << 5, + VEX_L = 1U << 4, // VEX_LIG - Specifies that this instruction ignores the L-bit in the VEX // prefix. Usually used for scalar instructions. Needed by disassembler. - VEX_LIG = 1U << 6, + VEX_LIG = 1U << 5, // TODO: we should combine VEX_L and VEX_LIG together to form a 2-bit field // with following encoding: @@ -473,26 +508,20 @@ namespace X86II { // - 11 LIG (but, in insn encoding, leave VEX.L and EVEX.L in zeros. // this will save 1 tsflag bit - // VEX_EVEX - Specifies that this instruction use EVEX form which provides - // syntax support up to 32 512-bit register operands and up to 7 16-bit - // mask operands as well as source operand data swizzling/memory operand - // conversion, eviction hint, and rounding mode. - EVEX = 1U << 7, - // EVEX_K - Set if this instruction requires masking - EVEX_K = 1U << 8, + EVEX_K = 1U << 6, // EVEX_Z - Set if this instruction has EVEX.Z field set. - EVEX_Z = 1U << 9, + EVEX_Z = 1U << 7, // EVEX_L2 - Set if this instruction has EVEX.L' field set. - EVEX_L2 = 1U << 10, + EVEX_L2 = 1U << 8, // EVEX_B - Set if this instruction has EVEX.B field set. - EVEX_B = 1U << 11, + EVEX_B = 1U << 9, // EVEX_CD8E - compressed disp8 form, element-size - EVEX_CD8EShift = VEXShift + 12, + EVEX_CD8EShift = VEXShift + 10, EVEX_CD8EMask = 3, // EVEX_CD8V - compressed disp8 form, vector-width @@ -505,15 +534,14 @@ namespace X86II { /// storing a classifier in the imm8 field. To simplify our implementation, /// we handle this by storeing the classifier in the opcode field and using /// this flag to indicate that the encoder should do the wacky 3DNow! thing. - Has3DNow0F0FOpcode = 1U << 17, + Has3DNow0F0FOpcode = 1U << 15, /// MemOp4 - Used to indicate swapping of operand 3 and 4 to be encoded in /// ModRM or I8IMM. This is used for FMA4 and XOP instructions. - MemOp4 = 1U << 18, - - /// XOP - Opcode prefix used by XOP instructions. - XOP = 1U << 19 + MemOp4 = 1U << 16, + /// Explicitly specified rounding control + EVEX_RC = 1U << 17 }; // getBaseOpcodeFor - This function returns the "base" X86 opcode for the @@ -537,6 +565,7 @@ namespace X86II { case X86II::Imm16: case X86II::Imm16PCRel: return 2; case X86II::Imm32: + case X86II::Imm32S: case X86II::Imm32PCRel: return 4; case X86II::Imm64: return 8; } @@ -554,6 +583,25 @@ namespace X86II { case X86II::Imm8: case X86II::Imm16: case X86II::Imm32: + case X86II::Imm32S: + case X86II::Imm64: + return false; + } + } + + /// isImmSigned - Return true if the immediate of the specified instruction's + /// TSFlags indicates that it is signed. + inline unsigned isImmSigned(uint64_t TSFlags) { + switch (TSFlags & X86II::ImmMask) { + default: llvm_unreachable("Unknown immediate signedness"); + case X86II::Imm32S: + return true; + case X86II::Imm8: + case X86II::Imm8PCRel: + case X86II::Imm16: + case X86II::Imm16PCRel: + case X86II::Imm32: + case X86II::Imm32PCRel: case X86II::Imm64: return false; } @@ -596,9 +644,6 @@ namespace X86II { /// inline int getMemoryOperandNo(uint64_t TSFlags, unsigned Opcode) { switch (TSFlags & X86II::FormMask) { - case X86II::MRMInitReg: - // FIXME: Remove this form. - return -1; default: llvm_unreachable("Unknown FormMask value in getMemoryOperandNo!"); case X86II::Pseudo: case X86II::RawFrm: @@ -607,14 +652,17 @@ namespace X86II { case X86II::MRMSrcReg: case X86II::RawFrmImm8: case X86II::RawFrmImm16: + case X86II::RawFrmMemOffs: + case X86II::RawFrmSrc: + case X86II::RawFrmDst: + case X86II::RawFrmDstSrc: return -1; case X86II::MRMDestMem: return 0; case X86II::MRMSrcMem: { bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V; bool HasMemOp4 = (TSFlags >> X86II::VEXShift) & X86II::MemOp4; - bool HasEVEX = (TSFlags >> X86II::VEXShift) & X86II::EVEX; - bool HasEVEX_K = HasEVEX && ((TSFlags >> X86II::VEXShift) & X86II::EVEX_K); + bool HasEVEX_K = ((TSFlags >> X86II::VEXShift) & X86II::EVEX_K); unsigned FirstMemOp = 1; if (HasVEX_4V) ++FirstMemOp;// Skip the register source (which is encoded in VEX_VVVV). @@ -627,11 +675,13 @@ namespace X86II { // Opcode == X86::LEA16r || Opcode == X86::LEA32r) return FirstMemOp; } + case X86II::MRMXr: case X86II::MRM0r: case X86II::MRM1r: case X86II::MRM2r: case X86II::MRM3r: case X86II::MRM4r: case X86II::MRM5r: case X86II::MRM6r: case X86II::MRM7r: return -1; + case X86II::MRMXm: case X86II::MRM0m: case X86II::MRM1m: case X86II::MRM2m: case X86II::MRM3m: case X86II::MRM4m: case X86II::MRM5m: @@ -642,15 +692,23 @@ namespace X86II { ++FirstMemOp;// Skip the register dest (which is encoded in VEX_VVVV). return FirstMemOp; } - case X86II::MRM_C1: case X86II::MRM_C2: case X86II::MRM_C3: - case X86II::MRM_C4: case X86II::MRM_C8: case X86II::MRM_C9: - case X86II::MRM_CA: case X86II::MRM_CB: case X86II::MRM_E8: - case X86II::MRM_F0: case X86II::MRM_F8: case X86II::MRM_F9: + case X86II::MRM_C0: case X86II::MRM_C1: case X86II::MRM_C2: + case X86II::MRM_C3: case X86II::MRM_C4: case X86II::MRM_C8: + case X86II::MRM_C9: case X86II::MRM_CA: case X86II::MRM_CB: case X86II::MRM_D0: case X86II::MRM_D1: case X86II::MRM_D4: case X86II::MRM_D5: case X86II::MRM_D6: case X86II::MRM_D8: case X86II::MRM_D9: case X86II::MRM_DA: case X86II::MRM_DB: case X86II::MRM_DC: case X86II::MRM_DD: case X86II::MRM_DE: - case X86II::MRM_DF: + case X86II::MRM_DF: case X86II::MRM_E0: case X86II::MRM_E1: + case X86II::MRM_E2: case X86II::MRM_E3: case X86II::MRM_E4: + case X86II::MRM_E5: case X86II::MRM_E8: case X86II::MRM_E9: + case X86II::MRM_EA: case X86II::MRM_EB: case X86II::MRM_EC: + case X86II::MRM_ED: case X86II::MRM_EE: case X86II::MRM_F0: + case X86II::MRM_F1: case X86II::MRM_F2: case X86II::MRM_F3: + case X86II::MRM_F4: case X86II::MRM_F5: case X86II::MRM_F6: + case X86II::MRM_F7: case X86II::MRM_F8: case X86II::MRM_F9: + case X86II::MRM_FA: case X86II::MRM_FB: case X86II::MRM_FC: + case X86II::MRM_FD: case X86II::MRM_FE: case X86II::MRM_FF: return -1; } } diff --git a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp index 3ddd865..c44d88d 100644 --- a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp +++ b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp @@ -24,9 +24,8 @@ namespace { virtual ~X86ELFObjectWriter(); protected: - virtual unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup, - bool IsPCRel, bool IsRelocWithSymbol, - int64_t Addend) const; + unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup, + bool IsPCRel) const override; }; } @@ -41,13 +40,10 @@ X86ELFObjectWriter::~X86ELFObjectWriter() unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target, const MCFixup &Fixup, - bool IsPCRel, - bool IsRelocWithSymbol, - int64_t Addend) const { + bool IsPCRel) const { // determine the type of the relocation - MCSymbolRefExpr::VariantKind Modifier = Target.isAbsolute() ? - MCSymbolRefExpr::VK_None : Target.getSymA()->getKind(); + MCSymbolRefExpr::VariantKind Modifier = Fixup.getAccessVariant(); unsigned Type; if (getEMachine() == ELF::EM_X86_64) { if (IsPCRel) { @@ -57,6 +53,7 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target, case FK_Data_8: Type = ELF::R_X86_64_PC64; break; case FK_Data_4: Type = ELF::R_X86_64_PC32; break; case FK_Data_2: Type = ELF::R_X86_64_PC16; break; + case FK_Data_1: Type = ELF::R_X86_64_PC8; break; case FK_PCRel_8: assert(Modifier == MCSymbolRefExpr::VK_None); @@ -160,6 +157,28 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target, Type = ELF::R_386_GOTPC; break; + case FK_PCRel_1: + case FK_Data_1: + switch (Modifier) { + default: + llvm_unreachable("Unimplemented"); + case MCSymbolRefExpr::VK_None: + Type = ELF::R_386_PC8; + break; + } + break; + + case FK_PCRel_2: + case FK_Data_2: + switch (Modifier) { + default: + llvm_unreachable("Unimplemented"); + case MCSymbolRefExpr::VK_None: + Type = ELF::R_386_PC16; + break; + } + break; + case X86::reloc_signed_4byte: case FK_PCRel_4: case FK_Data_4: diff --git a/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp b/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp index a3eb4fb..4fa519c 100644 --- a/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp +++ b/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp @@ -11,8 +11,8 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCRelocationInfo.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/ELF.h" @@ -25,7 +25,7 @@ class X86_64ELFRelocationInfo : public MCRelocationInfo { public: X86_64ELFRelocationInfo(MCContext &Ctx) : MCRelocationInfo(Ctx) {} - const MCExpr *createExprForRelocation(RelocationRef Rel) { + const MCExpr *createExprForRelocation(RelocationRef Rel) override { uint64_t RelType; Rel.getType(RelType); symbol_iterator SymI = Rel.getSymbol(); diff --git a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp index 3861e1c..6561804 100644 --- a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp +++ b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp @@ -65,6 +65,19 @@ X86MCAsmInfoDarwin::X86MCAsmInfoDarwin(const Triple &T) { // Exceptions handling ExceptionsType = ExceptionHandling::DwarfCFI; + + // old assembler lacks some directives + // FIXME: this should really be a check on the assembler characteristics + // rather than OS version + if (T.isMacOSX() && T.isMacOSXVersionLT(10, 6)) + HasWeakDefCanBeHiddenDirective = false; + + // FIXME: this should not depend on the target OS version, but on the ld64 + // version in use. From at least >= ld64-97.17 (Xcode 3.2.6) the abs-ified + // FDE relocs may be used. + DwarfFDESymbolsUseAbsDiff = T.isMacOSX() && !T.isMacOSXVersionLT(10, 6); + + UseIntegratedAssembler = true; } X86_64MCAsmInfoDarwin::X86_64MCAsmInfoDarwin(const Triple &Triple) @@ -89,8 +102,6 @@ X86ELFMCAsmInfo::X86ELFMCAsmInfo(const Triple &T) { TextAlignFillValue = 0x90; - PrivateGlobalPrefix = ".L"; - // Set up DWARF directives HasLEB128 = true; // Target asm supports leb128 directives (little-endian) @@ -105,6 +116,10 @@ X86ELFMCAsmInfo::X86ELFMCAsmInfo(const Triple &T) { if ((T.getOS() == Triple::OpenBSD || T.getOS() == Triple::Bitrig) && T.getArch() == Triple::x86) Data64bitsDirective = 0; + + // Always enable the integrated assembler by default. + // Clang also enabled it when the OS is Solaris but that is redundant here. + UseIntegratedAssembler = true; } const MCExpr * @@ -127,25 +142,23 @@ getNonexecutableStackSection(MCContext &Ctx) const { void X86MCAsmInfoMicrosoft::anchor() { } X86MCAsmInfoMicrosoft::X86MCAsmInfoMicrosoft(const Triple &Triple) { - if (Triple.getArch() == Triple::x86_64) { - GlobalPrefix = ""; + if (Triple.getArch() == Triple::x86_64) PrivateGlobalPrefix = ".L"; - } AssemblerDialect = AsmWriterFlavor; TextAlignFillValue = 0x90; AllowAtInName = true; + + UseIntegratedAssembler = true; } void X86MCAsmInfoGNUCOFF::anchor() { } X86MCAsmInfoGNUCOFF::X86MCAsmInfoGNUCOFF(const Triple &Triple) { - if (Triple.getArch() == Triple::x86_64) { - GlobalPrefix = ""; + if (Triple.getArch() == Triple::x86_64) PrivateGlobalPrefix = ".L"; - } AssemblerDialect = AsmWriterFlavor; @@ -153,4 +166,6 @@ X86MCAsmInfoGNUCOFF::X86MCAsmInfoGNUCOFF(const Triple &Triple) { // Exceptions handling ExceptionsType = ExceptionHandling::DwarfCFI; + + UseIntegratedAssembler = true; } diff --git a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h index 80979dd..a7509b0 100644 --- a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h +++ b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h @@ -23,34 +23,34 @@ namespace llvm { class Triple; class X86MCAsmInfoDarwin : public MCAsmInfoDarwin { - virtual void anchor(); + void anchor() override; public: explicit X86MCAsmInfoDarwin(const Triple &Triple); }; struct X86_64MCAsmInfoDarwin : public X86MCAsmInfoDarwin { explicit X86_64MCAsmInfoDarwin(const Triple &Triple); - virtual const MCExpr * - getExprForPersonalitySymbol(const MCSymbol *Sym, - unsigned Encoding, - MCStreamer &Streamer) const; + const MCExpr * + getExprForPersonalitySymbol(const MCSymbol *Sym, unsigned Encoding, + MCStreamer &Streamer) const override; }; class X86ELFMCAsmInfo : public MCAsmInfoELF { - virtual void anchor(); + void anchor() override; public: explicit X86ELFMCAsmInfo(const Triple &Triple); - virtual const MCSection *getNonexecutableStackSection(MCContext &Ctx) const; + const MCSection * + getNonexecutableStackSection(MCContext &Ctx) const override; }; class X86MCAsmInfoMicrosoft : public MCAsmInfoMicrosoft { - virtual void anchor(); + void anchor() override; public: explicit X86MCAsmInfoMicrosoft(const Triple &Triple); }; class X86MCAsmInfoGNUCOFF : public MCAsmInfoGNUCOFF { - virtual void anchor(); + void anchor() override; public: explicit X86MCAsmInfoGNUCOFF(const Triple &Triple); }; diff --git a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp index 7952607..e6fb037 100644 --- a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -32,24 +32,43 @@ class X86MCCodeEmitter : public MCCodeEmitter { X86MCCodeEmitter(const X86MCCodeEmitter &) LLVM_DELETED_FUNCTION; void operator=(const X86MCCodeEmitter &) LLVM_DELETED_FUNCTION; const MCInstrInfo &MCII; - const MCSubtargetInfo &STI; MCContext &Ctx; public: - X86MCCodeEmitter(const MCInstrInfo &mcii, const MCSubtargetInfo &sti, - MCContext &ctx) - : MCII(mcii), STI(sti), Ctx(ctx) { + X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) + : MCII(mcii), Ctx(ctx) { } ~X86MCCodeEmitter() {} - bool is64BitMode() const { - // FIXME: Can tablegen auto-generate this? + bool is64BitMode(const MCSubtargetInfo &STI) const { return (STI.getFeatureBits() & X86::Mode64Bit) != 0; } - bool is32BitMode() const { - // FIXME: Can tablegen auto-generate this? - return (STI.getFeatureBits() & X86::Mode64Bit) == 0; + bool is32BitMode(const MCSubtargetInfo &STI) const { + return (STI.getFeatureBits() & X86::Mode32Bit) != 0; + } + + bool is16BitMode(const MCSubtargetInfo &STI) const { + return (STI.getFeatureBits() & X86::Mode16Bit) != 0; + } + + /// Is16BitMemOperand - Return true if the specified instruction has + /// a 16-bit memory operand. Op specifies the operand # of the memoperand. + bool Is16BitMemOperand(const MCInst &MI, unsigned Op, + const MCSubtargetInfo &STI) const { + const MCOperand &BaseReg = MI.getOperand(Op+X86::AddrBaseReg); + const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg); + const MCOperand &Disp = MI.getOperand(Op+X86::AddrDisp); + + if (is16BitMode(STI) && BaseReg.getReg() == 0 && + Disp.isImm() && Disp.getImm() < 0x10000) + return true; + if ((BaseReg.getReg() != 0 && + X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) || + (IndexReg.getReg() != 0 && + X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg()))) + return true; + return false; } unsigned GetX86RegNum(const MCOperand &MO) const { @@ -126,21 +145,23 @@ public: void EmitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField, uint64_t TSFlags, unsigned &CurByte, raw_ostream &OS, - SmallVectorImpl<MCFixup> &Fixups) const; + SmallVectorImpl<MCFixup> &Fixups, + const MCSubtargetInfo &STI) const; void EncodeInstruction(const MCInst &MI, raw_ostream &OS, - SmallVectorImpl<MCFixup> &Fixups) const; + SmallVectorImpl<MCFixup> &Fixups, + const MCSubtargetInfo &STI) const override; void EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand, const MCInst &MI, const MCInstrDesc &Desc, raw_ostream &OS) const; - void EmitSegmentOverridePrefix(uint64_t TSFlags, unsigned &CurByte, - int MemOperand, const MCInst &MI, - raw_ostream &OS) const; + void EmitSegmentOverridePrefix(unsigned &CurByte, unsigned SegOperand, + const MCInst &MI, raw_ostream &OS) const; void EmitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand, const MCInst &MI, const MCInstrDesc &Desc, + const MCSubtargetInfo &STI, raw_ostream &OS) const; }; @@ -151,7 +172,7 @@ MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI, const MCSubtargetInfo &STI, MCContext &Ctx) { - return new X86MCCodeEmitter(MCII, STI, Ctx); + return new X86MCCodeEmitter(MCII, Ctx); } /// isDisp8 - Return true if this signed displacement fits in a 8-bit @@ -163,7 +184,7 @@ static bool isDisp8(int Value) { /// isCDisp8 - Return true if this signed displacement fits in a 8-bit /// compressed dispacement field. static bool isCDisp8(uint64_t TSFlags, int Value, int& CValue) { - assert(((TSFlags >> X86II::VEXShift) & X86II::EVEX) && + assert((TSFlags & X86II::EncodingMask) >> X86II::EncodingShift == X86II::EVEX && "Compressed 8-bit displacement is only valid for EVEX inst."); unsigned CD8E = (TSFlags >> X86II::EVEX_CD8EShift) & X86II::EVEX_CD8EMask; @@ -198,7 +219,7 @@ static bool isCDisp8(uint64_t TSFlags, int Value, int& CValue) { if (Value & MemObjMask) // Unaligned offset return false; - Value /= MemObjSize; + Value /= (int)MemObjSize; bool Ret = (Value == (signed char)Value); if (Ret) @@ -212,6 +233,12 @@ static MCFixupKind getImmFixupKind(uint64_t TSFlags) { unsigned Size = X86II::getSizeOfImm(TSFlags); bool isPCRel = X86II::isImmPCRel(TSFlags); + if (X86II::isImmSigned(TSFlags)) { + switch (Size) { + default: llvm_unreachable("Unsupported signed fixup size!"); + case 4: return MCFixupKind(X86::reloc_signed_4byte); + } + } return MCFixup::getKindForSize(Size, isPCRel); } @@ -245,20 +272,6 @@ static bool Is64BitMemOperand(const MCInst &MI, unsigned Op) { } #endif -/// Is16BitMemOperand - Return true if the specified instruction has -/// a 16-bit memory operand. Op specifies the operand # of the memoperand. -static bool Is16BitMemOperand(const MCInst &MI, unsigned Op) { - const MCOperand &BaseReg = MI.getOperand(Op+X86::AddrBaseReg); - const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg); - - if ((BaseReg.getReg() != 0 && - X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) || - (IndexReg.getReg() != 0 && - X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg()))) - return true; - return false; -} - /// StartsWithGlobalOffsetTable - Check if this expression starts with /// _GLOBAL_OFFSET_TABLE_ and if it is of the form /// _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on ELF @@ -366,17 +379,20 @@ void X86MCCodeEmitter::EmitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField, uint64_t TSFlags, unsigned &CurByte, raw_ostream &OS, - SmallVectorImpl<MCFixup> &Fixups) const{ + SmallVectorImpl<MCFixup> &Fixups, + const MCSubtargetInfo &STI) const{ const MCOperand &Disp = MI.getOperand(Op+X86::AddrDisp); const MCOperand &Base = MI.getOperand(Op+X86::AddrBaseReg); const MCOperand &Scale = MI.getOperand(Op+X86::AddrScaleAmt); const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg); unsigned BaseReg = Base.getReg(); - bool HasEVEX = (TSFlags >> X86II::VEXShift) & X86II::EVEX; + unsigned char Encoding = (TSFlags & X86II::EncodingMask) >> + X86II::EncodingShift; + bool HasEVEX = (Encoding == X86II::EVEX); // Handle %rip relative addressing. if (BaseReg == X86::RIP) { // [disp32+RIP] in X86-64 mode - assert(is64BitMode() && "Rip-relative addressing requires 64-bit mode"); + assert(is64BitMode(STI) && "Rip-relative addressing requires 64-bit mode"); assert(IndexReg.getReg() == 0 && "Invalid rip-relative address"); EmitByte(ModRMByte(0, RegOpcodeField, 5), CurByte, OS); @@ -402,6 +418,66 @@ void X86MCCodeEmitter::EmitMemModRMByte(const MCInst &MI, unsigned Op, unsigned BaseRegNo = BaseReg ? GetX86RegNum(Base) : -1U; + // 16-bit addressing forms of the ModR/M byte have a different encoding for + // the R/M field and are far more limited in which registers can be used. + if (Is16BitMemOperand(MI, Op, STI)) { + if (BaseReg) { + // For 32-bit addressing, the row and column values in Table 2-2 are + // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with + // some special cases. And GetX86RegNum reflects that numbering. + // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A, + // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only + // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order, + // while values 0-3 indicate the allowed combinations (base+index) of + // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI. + // + // R16Table[] is a lookup from the normal RegNo, to the row values from + // Table 2-1 for 16-bit addressing modes. Where zero means disallowed. + static const unsigned R16Table[] = { 0, 0, 0, 7, 0, 6, 4, 5 }; + unsigned RMfield = R16Table[BaseRegNo]; + + assert(RMfield && "invalid 16-bit base register"); + + if (IndexReg.getReg()) { + unsigned IndexReg16 = R16Table[GetX86RegNum(IndexReg)]; + + assert(IndexReg16 && "invalid 16-bit index register"); + // We must have one of SI/DI (4,5), and one of BP/BX (6,7). + assert(((IndexReg16 ^ RMfield) & 2) && + "invalid 16-bit base/index register combination"); + assert(Scale.getImm() == 1 && + "invalid scale for 16-bit memory reference"); + + // Allow base/index to appear in either order (although GAS doesn't). + if (IndexReg16 & 2) + RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1); + else + RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1); + } + + if (Disp.isImm() && isDisp8(Disp.getImm())) { + if (Disp.getImm() == 0 && BaseRegNo != N86::EBP) { + // There is no displacement; just the register. + EmitByte(ModRMByte(0, RegOpcodeField, RMfield), CurByte, OS); + return; + } + // Use the [REG]+disp8 form, including for [BP] which cannot be encoded. + EmitByte(ModRMByte(1, RegOpcodeField, RMfield), CurByte, OS); + EmitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups); + return; + } + // This is the [REG]+disp16 case. + EmitByte(ModRMByte(2, RegOpcodeField, RMfield), CurByte, OS); + } else { + // There is no BaseReg; this is the plain [disp16] case. + EmitByte(ModRMByte(0, RegOpcodeField, 6), CurByte, OS); + } + + // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases. + EmitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, CurByte, OS, Fixups); + return; + } + // Determine whether a SIB byte is needed. // If no BaseReg, issue a RIP relative instruction only if the MCE can // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table @@ -415,7 +491,7 @@ void X86MCCodeEmitter::EmitMemModRMByte(const MCInst &MI, unsigned Op, BaseRegNo != N86::ESP && // If there is no base register and we're in 64-bit mode, we need a SIB // byte to emit an addr that is just 'disp32' (the non-RIP relative form). - (!is64BitMode() || BaseReg != 0)) { + (!is64BitMode(STI) || BaseReg != 0)) { if (BaseReg == 0) { // [disp32] in X86-32 mode EmitByte(ModRMByte(0, RegOpcodeField, 5), CurByte, OS); @@ -530,11 +606,13 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand, const MCInst &MI, const MCInstrDesc &Desc, raw_ostream &OS) const { - bool HasEVEX = (TSFlags >> X86II::VEXShift) & X86II::EVEX; - bool HasEVEX_K = HasEVEX && ((TSFlags >> X86II::VEXShift) & X86II::EVEX_K); + unsigned char Encoding = (TSFlags & X86II::EncodingMask) >> + X86II::EncodingShift; + bool HasEVEX_K = ((TSFlags >> X86II::VEXShift) & X86II::EVEX_K); bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V; bool HasVEX_4VOp3 = (TSFlags >> X86II::VEXShift) & X86II::VEX_4VOp3; bool HasMemOp4 = (TSFlags >> X86II::VEXShift) & X86II::MemOp4; + bool HasEVEX_RC = (TSFlags >> X86II::VEXShift) & X86II::EVEX_RC; // VEX_R: opcode externsion equivalent to REX.R in // 1's complement (inverted) form @@ -563,9 +641,6 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // opcode extension, or ignored, depending on the opcode byte) unsigned char VEX_W = 0; - // XOP: Use XOP prefix byte 0x8f instead of VEX. - bool XOP = false; - // VEX_5M (VEX m-mmmmm field): // // 0b00000: Reserved for future use @@ -576,7 +651,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // 0b01000: XOP map select - 08h instructions with imm byte // 0b01001: XOP map select - 09h instructions with no imm byte // 0b01010: XOP map select - 0Ah instructions with imm dword - unsigned char VEX_5M = 0x1; + unsigned char VEX_5M = 0; // VEX_4V (VEX vvvv field): a register specifier // (in 1's complement form) or 1111 if unused. @@ -610,91 +685,53 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // EVEX_b unsigned char EVEX_b = 0; + // EVEX_rc + unsigned char EVEX_rc = 0; + // EVEX_aaa unsigned char EVEX_aaa = 0; - // Encode the operand size opcode prefix as needed. - if (TSFlags & X86II::OpSize) - VEX_PP = 0x01; + bool EncodeRC = false; if ((TSFlags >> X86II::VEXShift) & X86II::VEX_W) VEX_W = 1; - if ((TSFlags >> X86II::VEXShift) & X86II::XOP) - XOP = true; - if ((TSFlags >> X86II::VEXShift) & X86II::VEX_L) VEX_L = 1; - if (HasEVEX && ((TSFlags >> X86II::VEXShift) & X86II::EVEX_L2)) + if (((TSFlags >> X86II::VEXShift) & X86II::EVEX_L2)) EVEX_L2 = 1; if (HasEVEX_K && ((TSFlags >> X86II::VEXShift) & X86II::EVEX_Z)) EVEX_z = 1; - if (HasEVEX && ((TSFlags >> X86II::VEXShift) & X86II::EVEX_B)) + if (((TSFlags >> X86II::VEXShift) & X86II::EVEX_B)) EVEX_b = 1; - switch (TSFlags & X86II::Op0Mask) { - default: llvm_unreachable("Invalid prefix!"); - case X86II::T8: // 0F 38 - VEX_5M = 0x2; - break; - case X86II::TA: // 0F 3A - VEX_5M = 0x3; - break; - case X86II::T8XS: // F3 0F 38 - VEX_PP = 0x2; - VEX_5M = 0x2; - break; - case X86II::T8XD: // F2 0F 38 - VEX_PP = 0x3; - VEX_5M = 0x2; - break; - case X86II::TAXD: // F2 0F 3A - VEX_PP = 0x3; - VEX_5M = 0x3; - break; - case X86II::XS: // F3 0F - VEX_PP = 0x2; - break; - case X86II::XD: // F2 0F - VEX_PP = 0x3; - break; - case X86II::XOP8: - VEX_5M = 0x8; - break; - case X86II::XOP9: - VEX_5M = 0x9; - break; - case X86II::XOPA: - VEX_5M = 0xA; - break; - case X86II::TB: // VEX_5M/VEX_PP already correct - break; + switch (TSFlags & X86II::OpPrefixMask) { + default: break; // VEX_PP already correct + case X86II::PD: VEX_PP = 0x1; break; // 66 + case X86II::XS: VEX_PP = 0x2; break; // F3 + case X86II::XD: VEX_PP = 0x3; break; // F2 } + switch (TSFlags & X86II::OpMapMask) { + default: llvm_unreachable("Invalid prefix!"); + case X86II::TB: VEX_5M = 0x1; break; // 0F + case X86II::T8: VEX_5M = 0x2; break; // 0F 38 + case X86II::TA: VEX_5M = 0x3; break; // 0F 3A + case X86II::XOP8: VEX_5M = 0x8; break; + case X86II::XOP9: VEX_5M = 0x9; break; + case X86II::XOPA: VEX_5M = 0xA; break; + } // Classify VEX_B, VEX_4V, VEX_R, VEX_X unsigned NumOps = Desc.getNumOperands(); - unsigned CurOp = 0; - if (NumOps > 1 && Desc.getOperandConstraint(1, MCOI::TIED_TO) == 0) - ++CurOp; - else if (NumOps > 3 && Desc.getOperandConstraint(2, MCOI::TIED_TO) == 0 && - Desc.getOperandConstraint(3, MCOI::TIED_TO) == 1) - // Special case for AVX-512 GATHER with 2 TIED_TO operands - // Skip the first 2 operands: dst, mask_wb - CurOp += 2; - else if (NumOps > 3 && Desc.getOperandConstraint(2, MCOI::TIED_TO) == 0 && - Desc.getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1) - // Special case for GATHER with 2 TIED_TO operands - // Skip the first 2 operands: dst, mask_wb - CurOp += 2; - else if (NumOps > 2 && Desc.getOperandConstraint(NumOps - 2, MCOI::TIED_TO) == 0) - // SCATTER - ++CurOp; + unsigned CurOp = X86II::getOperandBias(Desc); switch (TSFlags & X86II::FormMask) { - case X86II::MRMInitReg: llvm_unreachable("FIXME: Remove this!"); + default: llvm_unreachable("Unexpected form in EmitVEXOpcodePrefix!"); + case X86II::RawFrm: + break; case X86II::MRMDestMem: { // MRMDestMem instructions forms: // MemAddr, src1(ModR/M) @@ -707,7 +744,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (X86II::isX86_64ExtendedReg(MI.getOperand(MemOperand + X86::AddrIndexReg).getReg())) VEX_X = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(MemOperand + + if (X86II::is32ExtendedReg(MI.getOperand(MemOperand + X86::AddrIndexReg).getReg())) EVEX_V2 = 0x0; @@ -718,7 +755,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (HasVEX_4V) { VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_V2 = 0x0; CurOp++; } @@ -727,7 +764,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (MO.isReg()) { if (X86II::isX86_64ExtendedReg(MO.getReg())) VEX_R = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MO.getReg())) + if (X86II::is32ExtendedReg(MO.getReg())) EVEX_R2 = 0x0; } break; @@ -744,7 +781,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // dst(ModR/M.reg), src1(VEX_4V), src2(VEX_I8IMM), src3(ModR/M), if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_R = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_R2 = 0x0; CurOp++; @@ -753,7 +790,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (HasVEX_4V) { VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_V2 = 0x0; CurOp++; } @@ -764,8 +801,8 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (X86II::isX86_64ExtendedReg( MI.getOperand(MemOperand+X86::AddrIndexReg).getReg())) VEX_X = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(MemOperand + - X86::AddrIndexReg).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(MemOperand + + X86::AddrIndexReg).getReg())) EVEX_V2 = 0x0; if (HasVEX_4VOp3) @@ -785,7 +822,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // src1(VEX_4V), MemAddr if (HasVEX_4V) { VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_V2 = 0x0; CurOp++; } @@ -812,7 +849,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // dst(ModR/M.reg), src1(VEX_4V), src2(VEX_I8IMM), src3(ModR/M), if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_R = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_R2 = 0x0; CurOp++; @@ -821,7 +858,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (HasVEX_4V) { VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_V2 = 0x0; CurOp++; } @@ -831,11 +868,19 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_B = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_X = 0x0; CurOp++; if (HasVEX_4VOp3) - VEX_4V = getVEXRegisterEncoding(MI, CurOp); + VEX_4V = getVEXRegisterEncoding(MI, CurOp++); + if (EVEX_b) { + if (HasEVEX_RC) { + unsigned RcOperand = NumOps-1; + assert(RcOperand >= CurOp); + EVEX_rc = MI.getOperand(RcOperand).getImm() & 0x3; + } + EncodeRC = true; + } break; case X86II::MRMDestReg: // MRMDestReg instructions forms: @@ -844,7 +889,7 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // dst(ModR/M), src1(VEX_4V), src2(ModR/M) if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_B = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_X = 0x0; CurOp++; @@ -853,15 +898,17 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (HasVEX_4V) { VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_V2 = 0x0; CurOp++; } if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_R = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_R2 = 0x0; + if (EVEX_b) + EncodeRC = true; break; case X86II::MRM0r: case X86II::MRM1r: case X86II::MRM2r: case X86II::MRM3r: @@ -871,26 +918,21 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // dst(VEX_4V), src(ModR/M), imm8 if (HasVEX_4V) { VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) EVEX_V2 = 0x0; CurOp++; - } + } if (HasEVEX_K) EVEX_aaa = getWriteMaskRegisterEncoding(MI, CurOp++); if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_B = 0x0; - if (HasEVEX && X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) + if (X86II::is32ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_X = 0x0; break; - default: // RawFrm - break; } - // Emit segment override opcode prefix as needed. - EmitSegmentOverridePrefix(TSFlags, CurByte, MemOperand, MI, OS); - - if (!HasEVEX) { + if (Encoding == X86II::VEX || Encoding == X86II::XOP) { // VEX opcode prefix can have 2 or 3 bytes // // 3 bytes: @@ -902,19 +944,25 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, // | C5h | | R | vvvv | L | pp | // +-----+ +-------------------+ // + // XOP uses a similar prefix: + // +-----+ +--------------+ +-------------------+ + // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp | + // +-----+ +--------------+ +-------------------+ unsigned char LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3); - if (VEX_B && VEX_X && !VEX_W && !XOP && (VEX_5M == 1)) { // 2 byte VEX prefix + // Can we use the 2 byte VEX prefix? + if (Encoding == X86II::VEX && VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) { EmitByte(0xC5, CurByte, OS); EmitByte(LastByte | (VEX_R << 7), CurByte, OS); return; } // 3 byte VEX prefix - EmitByte(XOP ? 0x8F : 0xC4, CurByte, OS); + EmitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, CurByte, OS); EmitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, CurByte, OS); EmitByte(LastByte | (VEX_W << 7), CurByte, OS); } else { + assert(Encoding == X86II::EVEX && "unknown encoding!"); // EVEX opcode prefix can have 4 bytes // // +-----+ +--------------+ +-------------------+ +------------------------+ @@ -935,12 +983,19 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, CurByte, OS); - EmitByte((EVEX_z << 7) | - (EVEX_L2 << 6) | - (VEX_L << 5) | - (EVEX_b << 4) | - (EVEX_V2 << 3) | - EVEX_aaa, CurByte, OS); + if (EncodeRC) + EmitByte((EVEX_z << 7) | + (EVEX_rc << 5) | + (EVEX_b << 4) | + (EVEX_V2 << 3) | + EVEX_aaa, CurByte, OS); + else + EmitByte((EVEX_z << 7) | + (EVEX_L2 << 6) | + (VEX_L << 5) | + (EVEX_b << 4) | + (EVEX_V2 << 3) | + EVEX_aaa, CurByte, OS); } } @@ -974,7 +1029,6 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags, } switch (TSFlags & X86II::FormMask) { - case X86II::MRMInitReg: llvm_unreachable("FIXME: Remove this!"); case X86II::MRMSrcReg: if (MI.getOperand(0).isReg() && X86II::isX86_64ExtendedReg(MI.getOperand(0).getReg())) @@ -1002,6 +1056,7 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags, } break; } + case X86II::MRMXm: case X86II::MRM0m: case X86II::MRM1m: case X86II::MRM2m: case X86II::MRM3m: case X86II::MRM4m: case X86II::MRM5m: @@ -1039,33 +1094,20 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags, } /// EmitSegmentOverridePrefix - Emit segment override opcode prefix as needed -void X86MCCodeEmitter::EmitSegmentOverridePrefix(uint64_t TSFlags, - unsigned &CurByte, int MemOperand, - const MCInst &MI, - raw_ostream &OS) const { - switch (TSFlags & X86II::SegOvrMask) { - default: llvm_unreachable("Invalid segment!"); - case 0: - // No segment override, check for explicit one on memory operand. - if (MemOperand != -1) { // If the instruction has a memory operand. - switch (MI.getOperand(MemOperand+X86::AddrSegmentReg).getReg()) { - default: llvm_unreachable("Unknown segment register!"); - case 0: break; - case X86::CS: EmitByte(0x2E, CurByte, OS); break; - case X86::SS: EmitByte(0x36, CurByte, OS); break; - case X86::DS: EmitByte(0x3E, CurByte, OS); break; - case X86::ES: EmitByte(0x26, CurByte, OS); break; - case X86::FS: EmitByte(0x64, CurByte, OS); break; - case X86::GS: EmitByte(0x65, CurByte, OS); break; - } - } - break; - case X86II::FS: - EmitByte(0x64, CurByte, OS); - break; - case X86II::GS: - EmitByte(0x65, CurByte, OS); - break; +void X86MCCodeEmitter::EmitSegmentOverridePrefix(unsigned &CurByte, + unsigned SegOperand, + const MCInst &MI, + raw_ostream &OS) const { + // Check for explicit segment override on memory operand. + switch (MI.getOperand(SegOperand).getReg()) { + default: llvm_unreachable("Unknown segment register!"); + case 0: break; + case X86::CS: EmitByte(0x2E, CurByte, OS); break; + case X86::SS: EmitByte(0x36, CurByte, OS); break; + case X86::DS: EmitByte(0x3E, CurByte, OS); break; + case X86::ES: EmitByte(0x26, CurByte, OS); break; + case X86::FS: EmitByte(0x64, CurByte, OS); break; + case X86::GS: EmitByte(0x65, CurByte, OS); break; } } @@ -1076,118 +1118,56 @@ void X86MCCodeEmitter::EmitSegmentOverridePrefix(uint64_t TSFlags, void X86MCCodeEmitter::EmitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand, const MCInst &MI, const MCInstrDesc &Desc, + const MCSubtargetInfo &STI, raw_ostream &OS) const { - // Emit the lock opcode prefix as needed. - if (TSFlags & X86II::LOCK) - EmitByte(0xF0, CurByte, OS); - - // Emit segment override opcode prefix as needed. - EmitSegmentOverridePrefix(TSFlags, CurByte, MemOperand, MI, OS); - - // Emit the repeat opcode prefix as needed. - if ((TSFlags & X86II::Op0Mask) == X86II::REP) - EmitByte(0xF3, CurByte, OS); - - // Emit the address size opcode prefix as needed. - bool need_address_override; - if (TSFlags & X86II::AdSize) { - need_address_override = true; - } else if (MemOperand == -1) { - need_address_override = false; - } else if (is64BitMode()) { - assert(!Is16BitMemOperand(MI, MemOperand)); - need_address_override = Is32BitMemOperand(MI, MemOperand); - } else if (is32BitMode()) { - assert(!Is64BitMemOperand(MI, MemOperand)); - need_address_override = Is16BitMemOperand(MI, MemOperand); - } else { - need_address_override = false; - } - - if (need_address_override) - EmitByte(0x67, CurByte, OS); - // Emit the operand size opcode prefix as needed. - if (TSFlags & X86II::OpSize) + unsigned char OpSize = (TSFlags & X86II::OpSizeMask) >> X86II::OpSizeShift; + if (OpSize == (is16BitMode(STI) ? X86II::OpSize32 : X86II::OpSize16)) EmitByte(0x66, CurByte, OS); - bool Need0FPrefix = false; - switch (TSFlags & X86II::Op0Mask) { - default: llvm_unreachable("Invalid prefix!"); - case 0: break; // No prefix! - case X86II::REP: break; // already handled. - case X86II::TB: // Two-byte opcode prefix - case X86II::T8: // 0F 38 - case X86II::TA: // 0F 3A - case X86II::A6: // 0F A6 - case X86II::A7: // 0F A7 - Need0FPrefix = true; - break; - case X86II::T8XS: // F3 0F 38 - EmitByte(0xF3, CurByte, OS); - Need0FPrefix = true; - break; - case X86II::T8XD: // F2 0F 38 - EmitByte(0xF2, CurByte, OS); - Need0FPrefix = true; - break; - case X86II::TAXD: // F2 0F 3A - EmitByte(0xF2, CurByte, OS); - Need0FPrefix = true; + switch (TSFlags & X86II::OpPrefixMask) { + case X86II::PD: // 66 + EmitByte(0x66, CurByte, OS); break; - case X86II::XS: // F3 0F + case X86II::XS: // F3 EmitByte(0xF3, CurByte, OS); - Need0FPrefix = true; break; - case X86II::XD: // F2 0F + case X86II::XD: // F2 EmitByte(0xF2, CurByte, OS); - Need0FPrefix = true; break; - case X86II::D8: EmitByte(0xD8, CurByte, OS); break; - case X86II::D9: EmitByte(0xD9, CurByte, OS); break; - case X86II::DA: EmitByte(0xDA, CurByte, OS); break; - case X86II::DB: EmitByte(0xDB, CurByte, OS); break; - case X86II::DC: EmitByte(0xDC, CurByte, OS); break; - case X86II::DD: EmitByte(0xDD, CurByte, OS); break; - case X86II::DE: EmitByte(0xDE, CurByte, OS); break; - case X86II::DF: EmitByte(0xDF, CurByte, OS); break; } // Handle REX prefix. // FIXME: Can this come before F2 etc to simplify emission? - if (is64BitMode()) { + if (is64BitMode(STI)) { if (unsigned REX = DetermineREXPrefix(MI, TSFlags, Desc)) EmitByte(0x40 | REX, CurByte, OS); } // 0x0F escape code must be emitted just before the opcode. - if (Need0FPrefix) + switch (TSFlags & X86II::OpMapMask) { + case X86II::TB: // Two-byte opcode map + case X86II::T8: // 0F 38 + case X86II::TA: // 0F 3A EmitByte(0x0F, CurByte, OS); + break; + } - // FIXME: Pull this up into previous switch if REX can be moved earlier. - switch (TSFlags & X86II::Op0Mask) { - case X86II::T8XS: // F3 0F 38 - case X86II::T8XD: // F2 0F 38 + switch (TSFlags & X86II::OpMapMask) { case X86II::T8: // 0F 38 EmitByte(0x38, CurByte, OS); break; - case X86II::TAXD: // F2 0F 3A case X86II::TA: // 0F 3A EmitByte(0x3A, CurByte, OS); break; - case X86II::A6: // 0F A6 - EmitByte(0xA6, CurByte, OS); - break; - case X86II::A7: // 0F A7 - EmitByte(0xA7, CurByte, OS); - break; } } void X86MCCodeEmitter:: EncodeInstruction(const MCInst &MI, raw_ostream &OS, - SmallVectorImpl<MCFixup> &Fixups) const { + SmallVectorImpl<MCFixup> &Fixups, + const MCSubtargetInfo &STI) const { unsigned Opcode = MI.getOpcode(); const MCInstrDesc &Desc = MCII.get(Opcode); uint64_t TSFlags = Desc.TSFlags; @@ -1202,8 +1182,9 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, // Keep track of the current byte being emitted. unsigned CurByte = 0; - // Is this instruction encoded using the AVX VEX prefix? - bool HasVEXPrefix = (TSFlags >> X86II::VEXShift) & X86II::VEX; + // Encoding type for this instruction. + unsigned char Encoding = (TSFlags & X86II::EncodingMask) >> + X86II::EncodingShift; // It uses the VEX.VVVV field? bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V; @@ -1212,15 +1193,58 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, const unsigned MemOp4_I8IMMOperand = 2; // It uses the EVEX.aaa field? - bool HasEVEX = (TSFlags >> X86II::VEXShift) & X86II::EVEX; - bool HasEVEX_K = HasEVEX && ((TSFlags >> X86II::VEXShift) & X86II::EVEX_K); - + bool HasEVEX_K = ((TSFlags >> X86II::VEXShift) & X86II::EVEX_K); + bool HasEVEX_RC = ((TSFlags >> X86II::VEXShift) & X86II::EVEX_RC); + // Determine where the memory operand starts, if present. int MemoryOperand = X86II::getMemoryOperandNo(TSFlags, Opcode); if (MemoryOperand != -1) MemoryOperand += CurOp; - if (!HasVEXPrefix) - EmitOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, OS); + // Emit the lock opcode prefix as needed. + if (TSFlags & X86II::LOCK) + EmitByte(0xF0, CurByte, OS); + + // Emit segment override opcode prefix as needed. + if (MemoryOperand >= 0) + EmitSegmentOverridePrefix(CurByte, MemoryOperand+X86::AddrSegmentReg, + MI, OS); + + // Emit the repeat opcode prefix as needed. + if (TSFlags & X86II::REP) + EmitByte(0xF3, CurByte, OS); + + // Emit the address size opcode prefix as needed. + bool need_address_override; + // The AdSize prefix is only for 32-bit and 64-bit modes. Hm, perhaps we + // should introduce an AdSize16 bit instead of having seven special cases? + if ((!is16BitMode(STI) && TSFlags & X86II::AdSize) || + (is16BitMode(STI) && (MI.getOpcode() == X86::JECXZ_32 || + MI.getOpcode() == X86::MOV8o8a || + MI.getOpcode() == X86::MOV16o16a || + MI.getOpcode() == X86::MOV32o32a || + MI.getOpcode() == X86::MOV8ao8 || + MI.getOpcode() == X86::MOV16ao16 || + MI.getOpcode() == X86::MOV32ao32))) { + need_address_override = true; + } else if (MemoryOperand < 0) { + need_address_override = false; + } else if (is64BitMode(STI)) { + assert(!Is16BitMemOperand(MI, MemoryOperand, STI)); + need_address_override = Is32BitMemOperand(MI, MemoryOperand); + } else if (is32BitMode(STI)) { + assert(!Is64BitMemOperand(MI, MemoryOperand)); + need_address_override = Is16BitMemOperand(MI, MemoryOperand, STI); + } else { + assert(is16BitMode(STI)); + assert(!Is64BitMemOperand(MI, MemoryOperand)); + need_address_override = !Is16BitMemOperand(MI, MemoryOperand, STI); + } + + if (need_address_override) + EmitByte(0x67, CurByte, OS); + + if (Encoding == 0) + EmitOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, STI, OS); else EmitVEXOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, OS); @@ -1231,15 +1255,62 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, unsigned SrcRegNum = 0; switch (TSFlags & X86II::FormMask) { - case X86II::MRMInitReg: - llvm_unreachable("FIXME: Remove this form when the JIT moves to MCCodeEmitter!"); default: errs() << "FORM: " << (TSFlags & X86II::FormMask) << "\n"; llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!"); case X86II::Pseudo: llvm_unreachable("Pseudo instruction shouldn't be emitted"); + case X86II::RawFrmDstSrc: { + unsigned siReg = MI.getOperand(1).getReg(); + assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) || + (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) || + (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) && + "SI and DI register sizes do not match"); + // Emit segment override opcode prefix as needed (not for %ds). + if (MI.getOperand(2).getReg() != X86::DS) + EmitSegmentOverridePrefix(CurByte, 2, MI, OS); + // Emit AdSize prefix as needed. + if ((!is32BitMode(STI) && siReg == X86::ESI) || + (is32BitMode(STI) && siReg == X86::SI)) + EmitByte(0x67, CurByte, OS); + CurOp += 3; // Consume operands. + EmitByte(BaseOpcode, CurByte, OS); + break; + } + case X86II::RawFrmSrc: { + unsigned siReg = MI.getOperand(0).getReg(); + // Emit segment override opcode prefix as needed (not for %ds). + if (MI.getOperand(1).getReg() != X86::DS) + EmitSegmentOverridePrefix(CurByte, 1, MI, OS); + // Emit AdSize prefix as needed. + if ((!is32BitMode(STI) && siReg == X86::ESI) || + (is32BitMode(STI) && siReg == X86::SI)) + EmitByte(0x67, CurByte, OS); + CurOp += 2; // Consume operands. + EmitByte(BaseOpcode, CurByte, OS); + break; + } + case X86II::RawFrmDst: { + unsigned siReg = MI.getOperand(0).getReg(); + // Emit AdSize prefix as needed. + if ((!is32BitMode(STI) && siReg == X86::EDI) || + (is32BitMode(STI) && siReg == X86::DI)) + EmitByte(0x67, CurByte, OS); + ++CurOp; // Consume operand. + EmitByte(BaseOpcode, CurByte, OS); + break; + } case X86II::RawFrm: EmitByte(BaseOpcode, CurByte, OS); break; + case X86II::RawFrmMemOffs: + // Emit segment override opcode prefix as needed. + EmitSegmentOverridePrefix(CurByte, 1, MI, OS); + EmitByte(BaseOpcode, CurByte, OS); + EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(), + X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), + CurByte, OS, Fixups); + ++CurOp; // skip segment operand + break; case X86II::RawFrmImm8: EmitByte(BaseOpcode, CurByte, OS); EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(), @@ -1288,7 +1359,7 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, EmitMemModRMByte(MI, CurOp, GetX86RegNum(MI.getOperand(SrcRegNum)), - TSFlags, CurByte, OS, Fixups); + TSFlags, CurByte, OS, Fixups, STI); CurOp = SrcRegNum + 1; break; @@ -1312,6 +1383,9 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, CurOp = HasMemOp4 ? SrcRegNum : SrcRegNum + 1; if (HasVEX_4VOp3) ++CurOp; + // do not count the rounding control operand + if (HasEVEX_RC) + NumOps--; break; case X86II::MRMSrcMem: { @@ -1333,49 +1407,65 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, EmitByte(BaseOpcode, CurByte, OS); EmitMemModRMByte(MI, FirstMemOp, GetX86RegNum(MI.getOperand(CurOp)), - TSFlags, CurByte, OS, Fixups); + TSFlags, CurByte, OS, Fixups, STI); CurOp += AddrOperands + 1; if (HasVEX_4VOp3) ++CurOp; break; } + case X86II::MRMXr: case X86II::MRM0r: case X86II::MRM1r: case X86II::MRM2r: case X86II::MRM3r: case X86II::MRM4r: case X86II::MRM5r: - case X86II::MRM6r: case X86II::MRM7r: + case X86II::MRM6r: case X86II::MRM7r: { if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). ++CurOp; EmitByte(BaseOpcode, CurByte, OS); + uint64_t Form = TSFlags & X86II::FormMask; EmitRegModRMByte(MI.getOperand(CurOp++), - (TSFlags & X86II::FormMask)-X86II::MRM0r, + (Form == X86II::MRMXr) ? 0 : Form-X86II::MRM0r, CurByte, OS); break; + } + + case X86II::MRMXm: case X86II::MRM0m: case X86II::MRM1m: case X86II::MRM2m: case X86II::MRM3m: case X86II::MRM4m: case X86II::MRM5m: - case X86II::MRM6m: case X86II::MRM7m: + case X86II::MRM6m: case X86II::MRM7m: { if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). ++CurOp; EmitByte(BaseOpcode, CurByte, OS); - EmitMemModRMByte(MI, CurOp, (TSFlags & X86II::FormMask)-X86II::MRM0m, - TSFlags, CurByte, OS, Fixups); + uint64_t Form = TSFlags & X86II::FormMask; + EmitMemModRMByte(MI, CurOp, (Form == X86II::MRMXm) ? 0 : Form-X86II::MRM0m, + TSFlags, CurByte, OS, Fixups, STI); CurOp += X86::AddrNumOperands; break; - case X86II::MRM_C1: case X86II::MRM_C2: case X86II::MRM_C3: - case X86II::MRM_C4: case X86II::MRM_C8: case X86II::MRM_C9: - case X86II::MRM_CA: case X86II::MRM_CB: case X86II::MRM_D0: - case X86II::MRM_D1: case X86II::MRM_D4: case X86II::MRM_D5: - case X86II::MRM_D6: case X86II::MRM_D8: case X86II::MRM_D9: - case X86II::MRM_DA: case X86II::MRM_DB: case X86II::MRM_DC: - case X86II::MRM_DD: case X86II::MRM_DE: case X86II::MRM_DF: - case X86II::MRM_E8: case X86II::MRM_F0: case X86II::MRM_F8: - case X86II::MRM_F9: + } + case X86II::MRM_C0: case X86II::MRM_C1: case X86II::MRM_C2: + case X86II::MRM_C3: case X86II::MRM_C4: case X86II::MRM_C8: + case X86II::MRM_C9: case X86II::MRM_CA: case X86II::MRM_CB: + case X86II::MRM_D0: case X86II::MRM_D1: case X86II::MRM_D4: + case X86II::MRM_D5: case X86II::MRM_D6: case X86II::MRM_D8: + case X86II::MRM_D9: case X86II::MRM_DA: case X86II::MRM_DB: + case X86II::MRM_DC: case X86II::MRM_DD: case X86II::MRM_DE: + case X86II::MRM_DF: case X86II::MRM_E0: case X86II::MRM_E1: + case X86II::MRM_E2: case X86II::MRM_E3: case X86II::MRM_E4: + case X86II::MRM_E5: case X86II::MRM_E8: case X86II::MRM_E9: + case X86II::MRM_EA: case X86II::MRM_EB: case X86II::MRM_EC: + case X86II::MRM_ED: case X86II::MRM_EE: case X86II::MRM_F0: + case X86II::MRM_F1: case X86II::MRM_F2: case X86II::MRM_F3: + case X86II::MRM_F4: case X86II::MRM_F5: case X86II::MRM_F6: + case X86II::MRM_F7: case X86II::MRM_F8: case X86II::MRM_F9: + case X86II::MRM_FA: case X86II::MRM_FB: case X86II::MRM_FC: + case X86II::MRM_FD: case X86II::MRM_FE: case X86II::MRM_FF: EmitByte(BaseOpcode, CurByte, OS); unsigned char MRM; switch (TSFlags & X86II::FormMask) { default: llvm_unreachable("Invalid Form"); + case X86II::MRM_C0: MRM = 0xC0; break; case X86II::MRM_C1: MRM = 0xC1; break; case X86II::MRM_C2: MRM = 0xC2; break; case X86II::MRM_C3: MRM = 0xC3; break; @@ -1397,10 +1487,35 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, case X86II::MRM_DD: MRM = 0xDD; break; case X86II::MRM_DE: MRM = 0xDE; break; case X86II::MRM_DF: MRM = 0xDF; break; + case X86II::MRM_E0: MRM = 0xE0; break; + case X86II::MRM_E1: MRM = 0xE1; break; + case X86II::MRM_E2: MRM = 0xE2; break; + case X86II::MRM_E3: MRM = 0xE3; break; + case X86II::MRM_E4: MRM = 0xE4; break; + case X86II::MRM_E5: MRM = 0xE5; break; case X86II::MRM_E8: MRM = 0xE8; break; + case X86II::MRM_E9: MRM = 0xE9; break; + case X86II::MRM_EA: MRM = 0xEA; break; + case X86II::MRM_EB: MRM = 0xEB; break; + case X86II::MRM_EC: MRM = 0xEC; break; + case X86II::MRM_ED: MRM = 0xED; break; + case X86II::MRM_EE: MRM = 0xEE; break; case X86II::MRM_F0: MRM = 0xF0; break; + case X86II::MRM_F1: MRM = 0xF1; break; + case X86II::MRM_F2: MRM = 0xF2; break; + case X86II::MRM_F3: MRM = 0xF3; break; + case X86II::MRM_F4: MRM = 0xF4; break; + case X86II::MRM_F5: MRM = 0xF5; break; + case X86II::MRM_F6: MRM = 0xF6; break; + case X86II::MRM_F7: MRM = 0xF7; break; case X86II::MRM_F8: MRM = 0xF8; break; case X86II::MRM_F9: MRM = 0xF9; break; + case X86II::MRM_FA: MRM = 0xFA; break; + case X86II::MRM_FB: MRM = 0xFB; break; + case X86II::MRM_FC: MRM = 0xFC; break; + case X86II::MRM_FD: MRM = 0xFD; break; + case X86II::MRM_FE: MRM = 0xFE; break; + case X86II::MRM_FF: MRM = 0xFF; break; } EmitByte(MRM, CurByte, OS); break; @@ -1432,17 +1547,8 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS, EmitImmediate(MCOperand::CreateImm(RegNum), MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups); } else { - unsigned FixupKind; - // FIXME: Is there a better way to know that we need a signed relocation? - if (MI.getOpcode() == X86::ADD64ri32 || - MI.getOpcode() == X86::MOV64ri32 || - MI.getOpcode() == X86::MOV64mi32 || - MI.getOpcode() == X86::PUSH64i32) - FixupKind = X86::reloc_signed_4byte; - else - FixupKind = getImmFixupKind(TSFlags); EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(), - X86II::getSizeOfImm(TSFlags), MCFixupKind(FixupKind), + X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), CurByte, OS, Fixups); } } diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp index 1cbdafd..09fdb9c 100644 --- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp +++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp @@ -47,9 +47,12 @@ std::string X86_MC::ParseX86Triple(StringRef TT) { Triple TheTriple(TT); std::string FS; if (TheTriple.getArch() == Triple::x86_64) - FS = "+64bit-mode"; + FS = "+64bit-mode,-32bit-mode,-16bit-mode"; + else if (TheTriple.getEnvironment() != Triple::CODE16) + FS = "-64bit-mode,+32bit-mode,-16bit-mode"; else - FS = "-64bit-mode"; + FS = "-64bit-mode,-32bit-mode,+16bit-mode"; + return FS; } @@ -202,8 +205,7 @@ unsigned X86_MC::getDwarfRegFlavour(StringRef TT, bool isEH) { if (TheTriple.isOSDarwin()) return isEH ? DWARFFlavour::X86_32_DarwinEH : DWARFFlavour::X86_32_Generic; - if (TheTriple.getOS() == Triple::MinGW32 || - TheTriple.getOS() == Triple::Cygwin) + if (TheTriple.isOSCygMing()) // Unsupported by now, just quick fallback return DWARFFlavour::X86_32_Generic; return DWARFFlavour::X86_32_Generic; @@ -268,17 +270,17 @@ static MCAsmInfo *createX86MCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) { bool is64Bit = TheTriple.getArch() == Triple::x86_64; MCAsmInfo *MAI; - if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO) { + if (TheTriple.isOSBinFormatMachO()) { if (is64Bit) MAI = new X86_64MCAsmInfoDarwin(TheTriple); else MAI = new X86MCAsmInfoDarwin(TheTriple); - } else if (TheTriple.getEnvironment() == Triple::ELF) { + } else if (TheTriple.isOSBinFormatELF()) { // Force the use of an ELF container. MAI = new X86ELFMCAsmInfo(TheTriple); - } else if (TheTriple.getOS() == Triple::Win32) { + } else if (TheTriple.isWindowsMSVCEnvironment()) { MAI = new X86MCAsmInfoMicrosoft(TheTriple); - } else if (TheTriple.getOS() == Triple::MinGW32 || TheTriple.getOS() == Triple::Cygwin) { + } else if (TheTriple.isOSCygMing()) { MAI = new X86MCAsmInfoGNUCOFF(TheTriple); } else { // The default is ELF. @@ -358,17 +360,18 @@ static MCStreamer *createMCStreamer(const Target &T, StringRef TT, MCContext &Ctx, MCAsmBackend &MAB, raw_ostream &_OS, MCCodeEmitter *_Emitter, + const MCSubtargetInfo &STI, bool RelaxAll, bool NoExecStack) { Triple TheTriple(TT); - if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO) + if (TheTriple.isOSBinFormatMachO()) return createMachOStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll); - if (TheTriple.isOSWindows() && TheTriple.getEnvironment() != Triple::ELF) + if (TheTriple.isOSWindows() && !TheTriple.isOSBinFormatELF()) return createWinCOFFStreamer(Ctx, MAB, *_Emitter, _OS, RelaxAll); - return createELFStreamer(Ctx, 0, MAB, _OS, _Emitter, RelaxAll, NoExecStack); + return createELFStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll, NoExecStack); } static MCInstPrinter *createX86MCInstPrinter(const Target &T, @@ -387,7 +390,7 @@ static MCInstPrinter *createX86MCInstPrinter(const Target &T, static MCRelocationInfo *createX86MCRelocationInfo(StringRef TT, MCContext &Ctx) { Triple TheTriple(TT); - if (TheTriple.isEnvironmentMachO() && TheTriple.getArch() == Triple::x86_64) + if (TheTriple.isOSBinFormatMachO() && TheTriple.getArch() == Triple::x86_64) return createX86_64MachORelocationInfo(Ctx); else if (TheTriple.isOSBinFormatELF()) return createX86_64ELFRelocationInfo(Ctx); diff --git a/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp b/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp index 209b1d0..f2023e3 100644 --- a/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp +++ b/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp @@ -11,8 +11,8 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCRelocationInfo.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Object/MachO.h" using namespace llvm; @@ -24,7 +24,7 @@ class X86_64MachORelocationInfo : public MCRelocationInfo { public: X86_64MachORelocationInfo(MCContext &Ctx) : MCRelocationInfo(Ctx) {} - const MCExpr *createExprForRelocation(RelocationRef Rel) { + const MCExpr *createExprForRelocation(RelocationRef Rel) override { const MachOObjectFile *Obj = cast<MachOObjectFile>(Rel.getObjectFile()); uint64_t RelType; Rel.getType(RelType); @@ -72,9 +72,9 @@ public: break; case X86_64_RELOC_SUBTRACTOR: { - RelocationRef RelNext; - Obj->getRelocationNext(Rel.getRawDataRefImpl(), RelNext); - any_relocation_info RENext = Obj->getRelocation(RelNext.getRawDataRefImpl()); + Rel.moveNext(); + any_relocation_info RENext = + Obj->getRelocation(Rel.getRawDataRefImpl()); // X86_64_SUBTRACTOR must be followed by a relocation of type // X86_64_RELOC_UNSIGNED. @@ -86,7 +86,7 @@ public: const MCExpr *LHS = MCSymbolRefExpr::Create(Sym, Ctx); - symbol_iterator RSymI = RelNext.getSymbol(); + symbol_iterator RSymI = Rel.getSymbol(); uint64_t RSymAddr; RSymI->getAddress(RSymAddr); StringRef RSymName; diff --git a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp index eb7c0b1..1a35ced 100644 --- a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp +++ b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp @@ -63,7 +63,7 @@ public: void RecordRelocation(MachObjectWriter *Writer, const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, - MCValue Target, uint64_t &FixedValue) { + MCValue Target, uint64_t &FixedValue) override { if (Writer->is64Bit()) RecordX86_64Relocation(Writer, Asm, Layout, Fragment, Fixup, Target, FixedValue); @@ -230,7 +230,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(MachObjectWriter *Writer, if (Symbol->isInSection()) { const MCSectionMachO &Section = static_cast<const MCSectionMachO&>( Fragment->getParent()->getSection()); - if (Section.hasAttribute(MCSectionMachO::S_ATTR_DEBUG)) + if (Section.hasAttribute(MachO::S_ATTR_DEBUG)) Base = 0; } @@ -362,6 +362,7 @@ bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer, MCValue Target, unsigned Log2Size, uint64_t &FixedValue) { + uint64_t OriginalFixedValue = FixedValue; uint32_t FixupOffset = Layout.getFragmentOffset(Fragment)+Fixup.getOffset(); unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind()); unsigned Type = MachO::GENERIC_RELOC_VANILLA; @@ -431,8 +432,10 @@ bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer, // symbol, things can go badly. // // Required for 'as' compatibility. - if (FixupOffset > 0xffffff) + if (FixupOffset > 0xffffff) { + FixedValue = OriginalFixedValue; return false; + } } MachO::any_relocation_info MRE; diff --git a/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp index 6da4142..ffc9e8d 100644 --- a/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp +++ b/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp @@ -29,9 +29,8 @@ namespace { X86WinCOFFObjectWriter(bool Is64Bit_); virtual ~X86WinCOFFObjectWriter(); - virtual unsigned getRelocType(const MCValue &Target, - const MCFixup &Fixup, - bool IsCrossSection) const LLVM_OVERRIDE; + unsigned getRelocType(const MCValue &Target, const MCFixup &Fixup, + bool IsCrossSection) const override; }; } @@ -65,6 +64,9 @@ unsigned X86WinCOFFObjectWriter::getRelocType(const MCValue &Target, if (Is64Bit) return COFF::IMAGE_REL_AMD64_ADDR64; llvm_unreachable("unsupported relocation type"); + case FK_SecRel_2: + return Is64Bit ? COFF::IMAGE_REL_AMD64_SECTION + : COFF::IMAGE_REL_I386_SECTION; case FK_SecRel_4: return Is64Bit ? COFF::IMAGE_REL_AMD64_SECREL : COFF::IMAGE_REL_I386_SECREL; default: diff --git a/lib/Target/X86/README-SSE.txt b/lib/Target/X86/README-SSE.txt index adfa7fa..71329b0 100644 --- a/lib/Target/X86/README-SSE.txt +++ b/lib/Target/X86/README-SSE.txt @@ -494,11 +494,6 @@ is memory. //===---------------------------------------------------------------------===// -SSE4 extract-to-mem ops aren't being pattern matched because of the AssertZext -sitting between the truncate and the extract. - -//===---------------------------------------------------------------------===// - INSERTPS can match any insert (extract, imm1), imm2 for 4 x float, and insert any number of 0.0 simultaneously. Currently we only use it for simple insertions. diff --git a/lib/Target/X86/README.txt b/lib/Target/X86/README.txt index b4285a0..52d3c01 100644 --- a/lib/Target/X86/README.txt +++ b/lib/Target/X86/README.txt @@ -123,7 +123,7 @@ flags. The instruction selector sometimes misses folding a load into a compare. The pattern is written as (cmp reg, (load p)). Because the compare isn't commutative, it is not matched with the load on both sides. The dag combiner -should be made smart enough to cannonicalize the load into the RHS of a compare +should be made smart enough to canonicalize the load into the RHS of a compare when it can invert the result of the compare for free. //===---------------------------------------------------------------------===// @@ -1444,54 +1444,6 @@ it would be nice to produce "into" someday. //===---------------------------------------------------------------------===// -This code: - -void vec_mpys1(int y[], const int x[], int scaler) { -int i; -for (i = 0; i < 150; i++) - y[i] += (((long long)scaler * (long long)x[i]) >> 31); -} - -Compiles to this loop with GCC 3.x: - -.L5: - movl %ebx, %eax - imull (%edi,%ecx,4) - shrdl $31, %edx, %eax - addl %eax, (%esi,%ecx,4) - incl %ecx - cmpl $149, %ecx - jle .L5 - -llvm-gcc compiles it to the much uglier: - -LBB1_1: ## bb1 - movl 24(%esp), %eax - movl (%eax,%edi,4), %ebx - movl %ebx, %ebp - imull %esi, %ebp - movl %ebx, %eax - mull %ecx - addl %ebp, %edx - sarl $31, %ebx - imull %ecx, %ebx - addl %edx, %ebx - shldl $1, %eax, %ebx - movl 20(%esp), %eax - addl %ebx, (%eax,%edi,4) - incl %edi - cmpl $150, %edi - jne LBB1_1 ## bb1 - -The issue is that we hoist the cast of "scaler" to long long outside of the -loop, the value comes into the loop as two values, and -RegsForValue::getCopyFromRegs doesn't know how to put an AssertSext on the -constructed BUILD_PAIR which represents the cast value. - -This can be handled by making CodeGenPrepare sink the cast. - -//===---------------------------------------------------------------------===// - Test instructions can be eliminated by using EFLAGS values from arithmetic instructions. This is currently not done for mul, and, or, xor, neg, shl, sra, srl, shld, shrd, atomic ops, and others. It is also currently not done diff --git a/lib/Target/X86/TargetInfo/Android.mk b/lib/Target/X86/TargetInfo/Android.mk index 1c53475..1a6c902 100644 --- a/lib/Target/X86/TargetInfo/Android.mk +++ b/lib/Target/X86/TargetInfo/Android.mk @@ -32,6 +32,7 @@ include $(BUILD_HOST_STATIC_LIBRARY) # For the device # ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) include $(CLEAR_TBLGEN_VARS) @@ -51,3 +52,4 @@ LOCAL_MODULE_TAGS := optional include $(LLVM_DEVICE_BUILD_MK) include $(LLVM_TBLGEN_RULES_MK) include $(BUILD_STATIC_LIBRARY) +endif diff --git a/lib/Target/X86/TargetInfo/CMakeLists.txt b/lib/Target/X86/TargetInfo/CMakeLists.txt index b1d0b9f..1d8a8c1 100644 --- a/lib/Target/X86/TargetInfo/CMakeLists.txt +++ b/lib/Target/X86/TargetInfo/CMakeLists.txt @@ -1,7 +1,3 @@ -include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. ) - add_llvm_library(LLVMX86Info X86TargetInfo.cpp ) - -add_dependencies(LLVMX86Info X86CommonTableGen) diff --git a/lib/Target/X86/TargetInfo/LLVMBuild.txt b/lib/Target/X86/TargetInfo/LLVMBuild.txt index 3c64a22..6a52ea6 100644 --- a/lib/Target/X86/TargetInfo/LLVMBuild.txt +++ b/lib/Target/X86/TargetInfo/LLVMBuild.txt @@ -19,5 +19,5 @@ type = Library name = X86Info parent = X86 -required_libraries = MC Support Target +required_libraries = Support add_to_library_groups = X86 diff --git a/lib/Target/X86/TargetInfo/X86TargetInfo.cpp b/lib/Target/X86/TargetInfo/X86TargetInfo.cpp index 815d235..1ea8798 100644 --- a/lib/Target/X86/TargetInfo/X86TargetInfo.cpp +++ b/lib/Target/X86/TargetInfo/X86TargetInfo.cpp @@ -7,8 +7,7 @@ // //===----------------------------------------------------------------------===// -#include "X86.h" -#include "llvm/IR/Module.h" +#include "MCTargetDesc/X86MCTargetDesc.h" #include "llvm/Support/TargetRegistry.h" using namespace llvm; diff --git a/lib/Target/X86/Utils/Android.mk b/lib/Target/X86/Utils/Android.mk index d9dd670..e0c4797 100644 --- a/lib/Target/X86/Utils/Android.mk +++ b/lib/Target/X86/Utils/Android.mk @@ -5,6 +5,7 @@ x86_utils_SRC_FILES := \ # For the device # ===================================================== +ifneq (true,$(DISABLE_LLVM_DEVICE_BUILDS)) include $(CLEAR_VARS) LOCAL_SRC_FILES := $(x86_utils_SRC_FILES) @@ -17,6 +18,7 @@ LOCAL_MODULE_TAGS := optional include $(LLVM_DEVICE_BUILD_MK) include $(BUILD_STATIC_LIBRARY) +endif # For the host # ===================================================== diff --git a/lib/Target/X86/Utils/CMakeLists.txt b/lib/Target/X86/Utils/CMakeLists.txt index 2e72c34..b269746 100644 --- a/lib/Target/X86/Utils/CMakeLists.txt +++ b/lib/Target/X86/Utils/CMakeLists.txt @@ -1,7 +1,3 @@ -include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. ) - add_llvm_library(LLVMX86Utils X86ShuffleDecode.cpp ) - -add_dependencies(LLVMX86Utils X86CommonTableGen) diff --git a/lib/Target/X86/Utils/LLVMBuild.txt b/lib/Target/X86/Utils/LLVMBuild.txt index de0a30f..fdb886f 100644 --- a/lib/Target/X86/Utils/LLVMBuild.txt +++ b/lib/Target/X86/Utils/LLVMBuild.txt @@ -19,5 +19,5 @@ type = Library name = X86Utils parent = X86 -required_libraries = Core Support +required_libraries = Support add_to_library_groups = X86 diff --git a/lib/Target/X86/Utils/X86ShuffleDecode.cpp b/lib/Target/X86/Utils/X86ShuffleDecode.cpp index bbd4904..5f2441c 100644 --- a/lib/Target/X86/Utils/X86ShuffleDecode.cpp +++ b/lib/Target/X86/Utils/X86ShuffleDecode.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "X86ShuffleDecode.h" +#include "llvm/CodeGen/MachineValueType.h" //===----------------------------------------------------------------------===// // Vector Mask Decoding diff --git a/lib/Target/X86/Utils/X86ShuffleDecode.h b/lib/Target/X86/Utils/X86ShuffleDecode.h index 017ab32..9e75b6b 100644 --- a/lib/Target/X86/Utils/X86ShuffleDecode.h +++ b/lib/Target/X86/Utils/X86ShuffleDecode.h @@ -16,13 +16,14 @@ #define X86_SHUFFLE_DECODE_H #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/ValueTypes.h" //===----------------------------------------------------------------------===// // Vector Mask Decoding //===----------------------------------------------------------------------===// namespace llvm { +class MVT; + enum { SM_SentinelZero = -1 }; diff --git a/lib/Target/X86/X86.h b/lib/Target/X86/X86.h index 947002f..18e6845 100644 --- a/lib/Target/X86/X86.h +++ b/lib/Target/X86/X86.h @@ -15,14 +15,12 @@ #ifndef TARGET_X86_H #define TARGET_X86_H -#include "MCTargetDesc/X86BaseInfo.h" -#include "MCTargetDesc/X86MCTargetDesc.h" -#include "llvm/Support/DataTypes.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/Support/CodeGen.h" namespace llvm { class FunctionPass; +class ImmutablePass; class JITCodeEmitter; class X86TargetMachine; diff --git a/lib/Target/X86/X86.td b/lib/Target/X86/X86.td index 65c5552..78edcf0 100644 --- a/lib/Target/X86/X86.td +++ b/lib/Target/X86/X86.td @@ -22,6 +22,10 @@ include "llvm/Target/Target.td" def Mode64Bit : SubtargetFeature<"64bit-mode", "In64BitMode", "true", "64-bit mode (x86_64)">; +def Mode32Bit : SubtargetFeature<"32bit-mode", "In32BitMode", "true", + "32-bit mode (80386)">; +def Mode16Bit : SubtargetFeature<"16bit-mode", "In16BitMode", "true", + "16-bit mode (i8086)">; //===----------------------------------------------------------------------===// // X86 Subtarget features @@ -73,6 +77,8 @@ def FeatureCMPXCHG16B : SubtargetFeature<"cx16", "HasCmpxchg16b", "true", [Feature64Bit]>; def FeatureSlowBTMem : SubtargetFeature<"slow-bt-mem", "IsBTMemSlow", "true", "Bit testing of memory is slow">; +def FeatureSlowSHLD : SubtargetFeature<"slow-shld", "IsSHLDSlow", "true", + "SHLD instruction is slow">; def FeatureFastUAMem : SubtargetFeature<"fast-unaligned-mem", "IsUAMemFast", "true", "Fast unaligned memory access">; @@ -221,7 +227,7 @@ def : ProcessorModel<"slm", SLMModel, [ProcIntelSLM, FeaturePCLMUL, FeatureAES, FeatureCallRegIndirect, FeaturePRFCHW, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureFastUAMem]>; // "Arrandale" along with corei3 and corei5 def : ProcessorModel<"corei7", SandyBridgeModel, [FeatureSSE42, FeatureCMPXCHG16B, FeatureSlowBTMem, @@ -268,46 +274,53 @@ def : ProcessorModel<"knl", HaswellModel, def : Proc<"k6", [FeatureMMX]>; def : Proc<"k6-2", [Feature3DNow]>; def : Proc<"k6-3", [Feature3DNow]>; -def : Proc<"athlon", [Feature3DNowA, FeatureSlowBTMem]>; -def : Proc<"athlon-tbird", [Feature3DNowA, FeatureSlowBTMem]>; -def : Proc<"athlon-4", [FeatureSSE1, Feature3DNowA, FeatureSlowBTMem]>; -def : Proc<"athlon-xp", [FeatureSSE1, Feature3DNowA, FeatureSlowBTMem]>; -def : Proc<"athlon-mp", [FeatureSSE1, Feature3DNowA, FeatureSlowBTMem]>; +def : Proc<"athlon", [Feature3DNowA, FeatureSlowBTMem, + FeatureSlowSHLD]>; +def : Proc<"athlon-tbird", [Feature3DNowA, FeatureSlowBTMem, + FeatureSlowSHLD]>; +def : Proc<"athlon-4", [FeatureSSE1, Feature3DNowA, FeatureSlowBTMem, + FeatureSlowSHLD]>; +def : Proc<"athlon-xp", [FeatureSSE1, Feature3DNowA, FeatureSlowBTMem, + FeatureSlowSHLD]>; +def : Proc<"athlon-mp", [FeatureSSE1, Feature3DNowA, FeatureSlowBTMem, + FeatureSlowSHLD]>; def : Proc<"k8", [FeatureSSE2, Feature3DNowA, Feature64Bit, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"opteron", [FeatureSSE2, Feature3DNowA, Feature64Bit, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"athlon64", [FeatureSSE2, Feature3DNowA, Feature64Bit, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"athlon-fx", [FeatureSSE2, Feature3DNowA, Feature64Bit, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"k8-sse3", [FeatureSSE3, Feature3DNowA, FeatureCMPXCHG16B, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"opteron-sse3", [FeatureSSE3, Feature3DNowA, FeatureCMPXCHG16B, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"athlon64-sse3", [FeatureSSE3, Feature3DNowA, FeatureCMPXCHG16B, - FeatureSlowBTMem]>; + FeatureSlowBTMem, FeatureSlowSHLD]>; def : Proc<"amdfam10", [FeatureSSE4A, Feature3DNowA, FeatureCMPXCHG16B, FeatureLZCNT, - FeaturePOPCNT, FeatureSlowBTMem]>; + FeaturePOPCNT, FeatureSlowBTMem, + FeatureSlowSHLD]>; // Bobcat def : Proc<"btver1", [FeatureSSSE3, FeatureSSE4A, FeatureCMPXCHG16B, - FeaturePRFCHW, FeatureLZCNT, FeaturePOPCNT]>; + FeaturePRFCHW, FeatureLZCNT, FeaturePOPCNT, + FeatureSlowSHLD]>; // Jaguar def : Proc<"btver2", [FeatureAVX, FeatureSSE4A, FeatureCMPXCHG16B, FeaturePRFCHW, FeatureAES, FeaturePCLMUL, FeatureBMI, FeatureF16C, FeatureMOVBE, - FeatureLZCNT, FeaturePOPCNT]>; + FeatureLZCNT, FeaturePOPCNT, FeatureSlowSHLD]>; // Bulldozer def : Proc<"bdver1", [FeatureXOP, FeatureFMA4, FeatureCMPXCHG16B, FeatureAES, FeaturePRFCHW, FeaturePCLMUL, - FeatureLZCNT, FeaturePOPCNT]>; + FeatureLZCNT, FeaturePOPCNT, FeatureSlowSHLD]>; // Piledriver def : Proc<"bdver2", [FeatureXOP, FeatureFMA4, FeatureCMPXCHG16B, FeatureAES, FeaturePRFCHW, FeaturePCLMUL, FeatureF16C, FeatureLZCNT, - FeaturePOPCNT, FeatureBMI, FeatureTBM, - FeatureFMA]>; + FeaturePOPCNT, FeatureBMI, FeatureTBM, + FeatureFMA, FeatureSlowSHLD]>; // Steamroller def : Proc<"bdver3", [FeatureXOP, FeatureFMA4, FeatureCMPXCHG16B, @@ -387,12 +400,10 @@ def IntelAsmParserVariant : AsmParserVariant { def ATTAsmWriter : AsmWriter { string AsmWriterClassName = "ATTInstPrinter"; int Variant = 0; - bit isMCAsmWriter = 1; } def IntelAsmWriter : AsmWriter { string AsmWriterClassName = "IntelInstPrinter"; int Variant = 1; - bit isMCAsmWriter = 1; } def X86 : Target { diff --git a/lib/Target/X86/X86AsmPrinter.cpp b/lib/Target/X86/X86AsmPrinter.cpp index 1258441..fb66acc 100644 --- a/lib/Target/X86/X86AsmPrinter.cpp +++ b/lib/Target/X86/X86AsmPrinter.cpp @@ -14,18 +14,17 @@ #include "X86AsmPrinter.h" #include "InstPrinter/X86ATTInstPrinter.h" -#include "X86.h" +#include "MCTargetDesc/X86BaseInfo.h" #include "X86COFFMachineModuleInfo.h" +#include "X86InstrInfo.h" #include "X86MachineFunctionInfo.h" -#include "X86TargetMachine.h" #include "llvm/ADT/SmallString.h" -#include "llvm/Assembly/Writer.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" +#include "llvm/CodeGen/MachineValueType.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" -#include "llvm/DebugInfo.h" -#include "llvm/IR/CallingConv.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/MC/MCAsmInfo.h" @@ -38,8 +37,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" -#include "llvm/Target/Mangler.h" -#include "llvm/Target/TargetOptions.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -51,7 +48,7 @@ using namespace llvm; bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) { SetupMachineFunction(MF); - if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho()) { + if (Subtarget->isTargetCOFF()) { bool Intrn = MF.getFunction()->hasInternalLinkage(); OutStreamer.BeginCOFFSymbolDef(CurrentFnSym); OutStreamer.EmitCOFFSymbolStorageClass(Intrn ? COFF::IMAGE_SYM_CLASS_STATIC @@ -74,56 +71,55 @@ bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) { /// printSymbolOperand - Print a raw symbol reference operand. This handles /// jump tables, constant pools, global address and external symbols, all of /// which print to a label with various suffixes for relocation types etc. -void X86AsmPrinter::printSymbolOperand(const MachineOperand &MO, - raw_ostream &O) { +static void printSymbolOperand(X86AsmPrinter &P, const MachineOperand &MO, + raw_ostream &O) { switch (MO.getType()) { default: llvm_unreachable("unknown symbol type!"); - case MachineOperand::MO_JumpTableIndex: - O << *GetJTISymbol(MO.getIndex()); - break; case MachineOperand::MO_ConstantPoolIndex: - O << *GetCPISymbol(MO.getIndex()); - printOffset(MO.getOffset(), O); + O << *P.GetCPISymbol(MO.getIndex()); + P.printOffset(MO.getOffset(), O); break; case MachineOperand::MO_GlobalAddress: { const GlobalValue *GV = MO.getGlobal(); MCSymbol *GVSym; if (MO.getTargetFlags() == X86II::MO_DARWIN_STUB) - GVSym = GetSymbolWithGlobalValueBase(GV, "$stub"); + GVSym = P.getSymbolWithGlobalValueBase(GV, "$stub"); else if (MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY || MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY_PIC_BASE || MO.getTargetFlags() == X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE) - GVSym = GetSymbolWithGlobalValueBase(GV, "$non_lazy_ptr"); + GVSym = P.getSymbolWithGlobalValueBase(GV, "$non_lazy_ptr"); else - GVSym = getSymbol(GV); + GVSym = P.getSymbol(GV); // Handle dllimport linkage. if (MO.getTargetFlags() == X86II::MO_DLLIMPORT) - GVSym = OutContext.GetOrCreateSymbol(Twine("__imp_") + GVSym->getName()); + GVSym = + P.OutContext.GetOrCreateSymbol(Twine("__imp_") + GVSym->getName()); if (MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY || MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY_PIC_BASE) { - MCSymbol *Sym = GetSymbolWithGlobalValueBase(GV, "$non_lazy_ptr"); + MCSymbol *Sym = P.getSymbolWithGlobalValueBase(GV, "$non_lazy_ptr"); MachineModuleInfoImpl::StubValueTy &StubSym = - MMI->getObjFileInfo<MachineModuleInfoMachO>().getGVStubEntry(Sym); + P.MMI->getObjFileInfo<MachineModuleInfoMachO>().getGVStubEntry(Sym); if (StubSym.getPointer() == 0) StubSym = MachineModuleInfoImpl:: - StubValueTy(getSymbol(GV), !GV->hasInternalLinkage()); + StubValueTy(P.getSymbol(GV), !GV->hasInternalLinkage()); } else if (MO.getTargetFlags() == X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE){ - MCSymbol *Sym = GetSymbolWithGlobalValueBase(GV, "$non_lazy_ptr"); + MCSymbol *Sym = P.getSymbolWithGlobalValueBase(GV, "$non_lazy_ptr"); MachineModuleInfoImpl::StubValueTy &StubSym = - MMI->getObjFileInfo<MachineModuleInfoMachO>().getHiddenGVStubEntry(Sym); + P.MMI->getObjFileInfo<MachineModuleInfoMachO>().getHiddenGVStubEntry( + Sym); if (StubSym.getPointer() == 0) StubSym = MachineModuleInfoImpl:: - StubValueTy(getSymbol(GV), !GV->hasInternalLinkage()); + StubValueTy(P.getSymbol(GV), !GV->hasInternalLinkage()); } else if (MO.getTargetFlags() == X86II::MO_DARWIN_STUB) { - MCSymbol *Sym = GetSymbolWithGlobalValueBase(GV, "$stub"); + MCSymbol *Sym = P.getSymbolWithGlobalValueBase(GV, "$stub"); MachineModuleInfoImpl::StubValueTy &StubSym = - MMI->getObjFileInfo<MachineModuleInfoMachO>().getFnStubEntry(Sym); + P.MMI->getObjFileInfo<MachineModuleInfoMachO>().getFnStubEntry(Sym); if (StubSym.getPointer() == 0) StubSym = MachineModuleInfoImpl:: - StubValueTy(getSymbol(GV), !GV->hasInternalLinkage()); + StubValueTy(P.getSymbol(GV), !GV->hasInternalLinkage()); } // If the name begins with a dollar-sign, enclose it in parens. We do this @@ -132,36 +128,7 @@ void X86AsmPrinter::printSymbolOperand(const MachineOperand &MO, O << *GVSym; else O << '(' << *GVSym << ')'; - printOffset(MO.getOffset(), O); - break; - } - case MachineOperand::MO_ExternalSymbol: { - const MCSymbol *SymToPrint; - if (MO.getTargetFlags() == X86II::MO_DARWIN_STUB) { - SmallString<128> TempNameStr; - TempNameStr += StringRef(MO.getSymbolName()); - TempNameStr += StringRef("$stub"); - - MCSymbol *Sym = GetExternalSymbolSymbol(TempNameStr.str()); - MachineModuleInfoImpl::StubValueTy &StubSym = - MMI->getObjFileInfo<MachineModuleInfoMachO>().getFnStubEntry(Sym); - if (StubSym.getPointer() == 0) { - TempNameStr.erase(TempNameStr.end()-5, TempNameStr.end()); - StubSym = MachineModuleInfoImpl:: - StubValueTy(OutContext.GetOrCreateSymbol(TempNameStr.str()), - true); - } - SymToPrint = StubSym.getPointer(); - } else { - SymToPrint = GetExternalSymbolSymbol(MO.getSymbolName()); - } - - // If the name begins with a dollar-sign, enclose it in parens. We do this - // to avoid having it look like an integer immediate to the assembler. - if (SymToPrint->getName()[0] != '$') - O << *SymToPrint; - else - O << '(' << *SymToPrint << '('; + P.printOffset(MO.getOffset(), O); break; } } @@ -177,12 +144,12 @@ void X86AsmPrinter::printSymbolOperand(const MachineOperand &MO, // These affect the name of the symbol, not any suffix. break; case X86II::MO_GOT_ABSOLUTE_ADDRESS: - O << " + [.-" << *MF->getPICBaseSymbol() << ']'; + O << " + [.-" << *P.MF->getPICBaseSymbol() << ']'; break; case X86II::MO_PIC_BASE_OFFSET: case X86II::MO_DARWIN_NONLAZY_PIC_BASE: case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: - O << '-' << *MF->getPICBaseSymbol(); + O << '-' << *P.MF->getPICBaseSymbol(); break; case X86II::MO_TLSGD: O << "@TLSGD"; break; case X86II::MO_TLSLD: O << "@TLSLD"; break; @@ -199,41 +166,40 @@ void X86AsmPrinter::printSymbolOperand(const MachineOperand &MO, case X86II::MO_PLT: O << "@PLT"; break; case X86II::MO_TLVP: O << "@TLVP"; break; case X86II::MO_TLVP_PIC_BASE: - O << "@TLVP" << '-' << *MF->getPICBaseSymbol(); + O << "@TLVP" << '-' << *P.MF->getPICBaseSymbol(); break; case X86II::MO_SECREL: O << "@SECREL32"; break; } } +static void printOperand(X86AsmPrinter &P, const MachineInstr *MI, + unsigned OpNo, raw_ostream &O, + const char *Modifier = 0, unsigned AsmVariant = 0); + /// printPCRelImm - This is used to print an immediate value that ends up /// being encoded as a pc-relative value. These print slightly differently, for /// example, a $ is not emitted. -void X86AsmPrinter::printPCRelImm(const MachineInstr *MI, unsigned OpNo, - raw_ostream &O) { +static void printPCRelImm(X86AsmPrinter &P, const MachineInstr *MI, + unsigned OpNo, raw_ostream &O) { const MachineOperand &MO = MI->getOperand(OpNo); switch (MO.getType()) { default: llvm_unreachable("Unknown pcrel immediate operand"); case MachineOperand::MO_Register: // pc-relativeness was handled when computing the value in the reg. - printOperand(MI, OpNo, O); + printOperand(P, MI, OpNo, O); return; case MachineOperand::MO_Immediate: O << MO.getImm(); return; - case MachineOperand::MO_MachineBasicBlock: - O << *MO.getMBB()->getSymbol(); - return; case MachineOperand::MO_GlobalAddress: - case MachineOperand::MO_ExternalSymbol: - printSymbolOperand(MO, O); + printSymbolOperand(P, MO, O); return; } } - -void X86AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo, - raw_ostream &O, const char *Modifier, - unsigned AsmVariant) { +static void printOperand(X86AsmPrinter &P, const MachineInstr *MI, + unsigned OpNo, raw_ostream &O, const char *Modifier, + unsigned AsmVariant) { const MachineOperand &MO = MI->getOperand(OpNo); switch (MO.getType()) { default: llvm_unreachable("unknown operand type!"); @@ -256,22 +222,20 @@ void X86AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo, O << MO.getImm(); return; - case MachineOperand::MO_JumpTableIndex: - case MachineOperand::MO_ConstantPoolIndex: - case MachineOperand::MO_GlobalAddress: - case MachineOperand::MO_ExternalSymbol: { + case MachineOperand::MO_GlobalAddress: { if (AsmVariant == 0) O << '$'; - printSymbolOperand(MO, O); + printSymbolOperand(P, MO, O); break; } } } -void X86AsmPrinter::printLeaMemReference(const MachineInstr *MI, unsigned Op, - raw_ostream &O, const char *Modifier) { - const MachineOperand &BaseReg = MI->getOperand(Op); - const MachineOperand &IndexReg = MI->getOperand(Op+2); - const MachineOperand &DispSpec = MI->getOperand(Op+3); +static void printLeaMemReference(X86AsmPrinter &P, const MachineInstr *MI, + unsigned Op, raw_ostream &O, + const char *Modifier = NULL) { + const MachineOperand &BaseReg = MI->getOperand(Op+X86::AddrBaseReg); + const MachineOperand &IndexReg = MI->getOperand(Op+X86::AddrIndexReg); + const MachineOperand &DispSpec = MI->getOperand(Op+X86::AddrDisp); // If we really don't want to print out (rip), don't. bool HasBaseReg = BaseReg.getReg() != 0; @@ -282,14 +246,18 @@ void X86AsmPrinter::printLeaMemReference(const MachineInstr *MI, unsigned Op, // HasParenPart - True if we will print out the () part of the mem ref. bool HasParenPart = IndexReg.getReg() || HasBaseReg; - if (DispSpec.isImm()) { + switch (DispSpec.getType()) { + default: + llvm_unreachable("unknown operand type!"); + case MachineOperand::MO_Immediate: { int DispVal = DispSpec.getImm(); if (DispVal || !HasParenPart) O << DispVal; - } else { - assert(DispSpec.isGlobal() || DispSpec.isCPI() || - DispSpec.isJTI() || DispSpec.isSymbol()); - printSymbolOperand(MI->getOperand(Op+3), O); + break; + } + case MachineOperand::MO_GlobalAddress: + case MachineOperand::MO_ConstantPoolIndex: + printSymbolOperand(P, DispSpec, O); } if (Modifier && strcmp(Modifier, "H") == 0) @@ -301,12 +269,12 @@ void X86AsmPrinter::printLeaMemReference(const MachineInstr *MI, unsigned Op, O << '('; if (HasBaseReg) - printOperand(MI, Op, O, Modifier); + printOperand(P, MI, Op+X86::AddrBaseReg, O, Modifier); if (IndexReg.getReg()) { O << ','; - printOperand(MI, Op+2, O, Modifier); - unsigned ScaleVal = MI->getOperand(Op+1).getImm(); + printOperand(P, MI, Op+X86::AddrIndexReg, O, Modifier); + unsigned ScaleVal = MI->getOperand(Op+X86::AddrScaleAmt).getImm(); if (ScaleVal != 1) O << ',' << ScaleVal; } @@ -314,29 +282,31 @@ void X86AsmPrinter::printLeaMemReference(const MachineInstr *MI, unsigned Op, } } -void X86AsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op, - raw_ostream &O, const char *Modifier) { +static void printMemReference(X86AsmPrinter &P, const MachineInstr *MI, + unsigned Op, raw_ostream &O, + const char *Modifier = NULL) { assert(isMem(MI, Op) && "Invalid memory reference!"); - const MachineOperand &Segment = MI->getOperand(Op+4); + const MachineOperand &Segment = MI->getOperand(Op+X86::AddrSegmentReg); if (Segment.getReg()) { - printOperand(MI, Op+4, O, Modifier); + printOperand(P, MI, Op+X86::AddrSegmentReg, O, Modifier); O << ':'; } - printLeaMemReference(MI, Op, O, Modifier); + printLeaMemReference(P, MI, Op, O, Modifier); } -void X86AsmPrinter::printIntelMemReference(const MachineInstr *MI, unsigned Op, - raw_ostream &O, const char *Modifier, - unsigned AsmVariant){ - const MachineOperand &BaseReg = MI->getOperand(Op); - unsigned ScaleVal = MI->getOperand(Op+1).getImm(); - const MachineOperand &IndexReg = MI->getOperand(Op+2); - const MachineOperand &DispSpec = MI->getOperand(Op+3); - const MachineOperand &SegReg = MI->getOperand(Op+4); +static void printIntelMemReference(X86AsmPrinter &P, const MachineInstr *MI, + unsigned Op, raw_ostream &O, + const char *Modifier = NULL, + unsigned AsmVariant = 1) { + const MachineOperand &BaseReg = MI->getOperand(Op+X86::AddrBaseReg); + unsigned ScaleVal = MI->getOperand(Op+X86::AddrScaleAmt).getImm(); + const MachineOperand &IndexReg = MI->getOperand(Op+X86::AddrIndexReg); + const MachineOperand &DispSpec = MI->getOperand(Op+X86::AddrDisp); + const MachineOperand &SegReg = MI->getOperand(Op+X86::AddrSegmentReg); // If this has a segment register, print it. if (SegReg.getReg()) { - printOperand(MI, Op+4, O, Modifier, AsmVariant); + printOperand(P, MI, Op+X86::AddrSegmentReg, O, Modifier, AsmVariant); O << ':'; } @@ -344,7 +314,7 @@ void X86AsmPrinter::printIntelMemReference(const MachineInstr *MI, unsigned Op, bool NeedPlus = false; if (BaseReg.getReg()) { - printOperand(MI, Op, O, Modifier, AsmVariant); + printOperand(P, MI, Op+X86::AddrBaseReg, O, Modifier, AsmVariant); NeedPlus = true; } @@ -352,13 +322,13 @@ void X86AsmPrinter::printIntelMemReference(const MachineInstr *MI, unsigned Op, if (NeedPlus) O << " + "; if (ScaleVal != 1) O << ScaleVal << '*'; - printOperand(MI, Op+2, O, Modifier, AsmVariant); + printOperand(P, MI, Op+X86::AddrIndexReg, O, Modifier, AsmVariant); NeedPlus = true; } if (!DispSpec.isImm()) { if (NeedPlus) O << " + "; - printOperand(MI, Op+3, O, Modifier, AsmVariant); + printOperand(P, MI, Op+X86::AddrDisp, O, Modifier, AsmVariant); } else { int64_t DispVal = DispSpec.getImm(); if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) { @@ -376,8 +346,8 @@ void X86AsmPrinter::printIntelMemReference(const MachineInstr *MI, unsigned Op, O << ']'; } -bool X86AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, - raw_ostream &O) { +static bool printAsmMRegister(X86AsmPrinter &P, const MachineOperand &MO, + char Mode, raw_ostream &O) { unsigned Reg = MO.getReg(); switch (Mode) { default: return true; // Unknown mode. @@ -393,9 +363,11 @@ bool X86AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, case 'k': // Print SImode register Reg = getX86SubSuperRegister(Reg, MVT::i32); break; - case 'q': // Print DImode register - // FIXME: gcc will actually print e instead of r for 32-bit. - Reg = getX86SubSuperRegister(Reg, MVT::i64); + case 'q': + // Print 64-bit register names if 64-bit integer registers are available. + // Otherwise, print 32-bit register names. + MVT::SimpleValueType Ty = P.getSubtarget().is64Bit() ? MVT::i64 : MVT::i32; + Reg = getX86SubSuperRegister(Reg, Ty); break; } @@ -419,37 +391,50 @@ bool X86AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, // See if this is a generic print operand return AsmPrinter::PrintAsmOperand(MI, OpNo, AsmVariant, ExtraCode, O); case 'a': // This is an address. Currently only 'i' and 'r' are expected. - if (MO.isImm()) { + switch (MO.getType()) { + default: + return true; + case MachineOperand::MO_Immediate: O << MO.getImm(); return false; - } - if (MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isSymbol()) { - printSymbolOperand(MO, O); + case MachineOperand::MO_ConstantPoolIndex: + case MachineOperand::MO_JumpTableIndex: + case MachineOperand::MO_ExternalSymbol: + llvm_unreachable("unexpected operand type!"); + case MachineOperand::MO_GlobalAddress: + printSymbolOperand(*this, MO, O); if (Subtarget->isPICStyleRIPRel()) O << "(%rip)"; return false; - } - if (MO.isReg()) { + case MachineOperand::MO_Register: O << '('; - printOperand(MI, OpNo, O); + printOperand(*this, MI, OpNo, O); O << ')'; return false; } - return true; case 'c': // Don't print "$" before a global var name or constant. - if (MO.isImm()) + switch (MO.getType()) { + default: + printOperand(*this, MI, OpNo, O); + break; + case MachineOperand::MO_Immediate: O << MO.getImm(); - else if (MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isSymbol()) - printSymbolOperand(MO, O); - else - printOperand(MI, OpNo, O); + break; + case MachineOperand::MO_ConstantPoolIndex: + case MachineOperand::MO_JumpTableIndex: + case MachineOperand::MO_ExternalSymbol: + llvm_unreachable("unexpected operand type!"); + case MachineOperand::MO_GlobalAddress: + printSymbolOperand(*this, MO, O); + break; + } return false; case 'A': // Print '*' before a register (it must be a register) if (MO.isReg()) { O << '*'; - printOperand(MI, OpNo, O); + printOperand(*this, MI, OpNo, O); return false; } return true; @@ -460,12 +445,12 @@ bool X86AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, case 'k': // Print SImode register case 'q': // Print DImode register if (MO.isReg()) - return printAsmMRegister(MO, ExtraCode[0], O); - printOperand(MI, OpNo, O); + return printAsmMRegister(*this, MO, ExtraCode[0], O); + printOperand(*this, MI, OpNo, O); return false; case 'P': // This is the operand of a call, treat specially. - printPCRelImm(MI, OpNo, O); + printPCRelImm(*this, MI, OpNo, O); return false; case 'n': // Negate the immediate or print a '-' before the operand. @@ -479,7 +464,7 @@ bool X86AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, } } - printOperand(MI, OpNo, O, /*Modifier*/ 0, AsmVariant); + printOperand(*this, MI, OpNo, O, /*Modifier*/ 0, AsmVariant); return false; } @@ -488,7 +473,7 @@ bool X86AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, const char *ExtraCode, raw_ostream &O) { if (AsmVariant) { - printIntelMemReference(MI, OpNo, O); + printIntelMemReference(*this, MI, OpNo, O); return false; } @@ -505,19 +490,19 @@ bool X86AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, // These only apply to registers, ignore on mem. break; case 'H': - printMemReference(MI, OpNo, O, "H"); + printMemReference(*this, MI, OpNo, O, "H"); return false; case 'P': // Don't print @PLT, but do print as memory. - printMemReference(MI, OpNo, O, "no-rip"); + printMemReference(*this, MI, OpNo, O, "no-rip"); return false; } } - printMemReference(MI, OpNo, O); + printMemReference(*this, MI, OpNo, O); return false; } void X86AsmPrinter::EmitStartOfAsmFile(Module &M) { - if (Subtarget->isTargetEnvMacho()) + if (Subtarget->isTargetMacho()) OutStreamer.SwitchSection(getObjFileLowering().getTextSection()); if (Subtarget->isTargetCOFF()) { @@ -544,7 +529,7 @@ void X86AsmPrinter::EmitStartOfAsmFile(Module &M) { void X86AsmPrinter::EmitEndOfAsmFile(Module &M) { - if (Subtarget->isTargetEnvMacho()) { + if (Subtarget->isTargetMacho()) { // All darwin targets use mach-o. MachineModuleInfoMachO &MMIMacho = MMI->getObjFileInfo<MachineModuleInfoMachO>(); @@ -556,9 +541,9 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) { if (!Stubs.empty()) { const MCSection *TheSection = OutContext.getMachOSection("__IMPORT", "__jump_table", - MCSectionMachO::S_SYMBOL_STUBS | - MCSectionMachO::S_ATTR_SELF_MODIFYING_CODE | - MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS, + MachO::S_SYMBOL_STUBS | + MachO::S_ATTR_SELF_MODIFYING_CODE | + MachO::S_ATTR_PURE_INSTRUCTIONS, 5, SectionKind::getMetadata()); OutStreamer.SwitchSection(TheSection); @@ -582,7 +567,7 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) { if (!Stubs.empty()) { const MCSection *TheSection = OutContext.getMachOSection("__IMPORT", "__pointers", - MCSectionMachO::S_NON_LAZY_SYMBOL_POINTERS, + MachO::S_NON_LAZY_SYMBOL_POINTERS, SectionKind::getMetadata()); OutStreamer.SwitchSection(TheSection); @@ -638,14 +623,13 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) { OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols); } - if (Subtarget->isTargetWindows() && !Subtarget->isTargetCygMing() && - MMI->usesVAFloatArgument()) { + if (Subtarget->isTargetKnownWindowsMSVC() && MMI->usesVAFloatArgument()) { StringRef SymbolName = Subtarget->is64Bit() ? "_fltused" : "__fltused"; MCSymbol *S = MMI->getContext().GetOrCreateSymbol(SymbolName); OutStreamer.EmitSymbolAttribute(S, MCSA_Global); } - if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho()) { + if (Subtarget->isTargetCOFF()) { X86COFFMachineModuleInfo &COFFMMI = MMI->getObjFileInfo<X86COFFMachineModuleInfo>(); @@ -664,29 +648,44 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) { // Necessary for dllexport support std::vector<const MCSymbol*> DLLExportedFns, DLLExportedGlobals; - const TargetLoweringObjectFileCOFF &TLOFCOFF = - static_cast<const TargetLoweringObjectFileCOFF&>(getObjFileLowering()); - for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) - if (I->hasDLLExportLinkage()) + if (I->hasDLLExportStorageClass()) DLLExportedFns.push_back(getSymbol(I)); for (Module::const_global_iterator I = M.global_begin(), E = M.global_end(); I != E; ++I) - if (I->hasDLLExportLinkage()) + if (I->hasDLLExportStorageClass()) DLLExportedGlobals.push_back(getSymbol(I)); + for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end(); + I != E; ++I) { + const GlobalValue *GV = I; + if (!GV->hasDLLExportStorageClass()) + continue; + + while (const GlobalAlias *A = dyn_cast<GlobalAlias>(GV)) + GV = A->getAliasedGlobal(); + + if (isa<Function>(GV)) + DLLExportedFns.push_back(getSymbol(I)); + else if (isa<GlobalVariable>(GV)) + DLLExportedGlobals.push_back(getSymbol(I)); + } + // Output linker support code for dllexported globals on windows. if (!DLLExportedGlobals.empty() || !DLLExportedFns.empty()) { + const TargetLoweringObjectFileCOFF &TLOFCOFF = + static_cast<const TargetLoweringObjectFileCOFF&>(getObjFileLowering()); + OutStreamer.SwitchSection(TLOFCOFF.getDrectveSection()); SmallString<128> name; for (unsigned i = 0, e = DLLExportedGlobals.size(); i != e; ++i) { - if (Subtarget->isTargetWindows()) + if (Subtarget->isTargetKnownWindowsMSVC()) name = " /EXPORT:"; else name = " -export:"; name += DLLExportedGlobals[i]->getName(); - if (Subtarget->isTargetWindows()) + if (Subtarget->isTargetKnownWindowsMSVC()) name += ",DATA"; else name += ",data"; @@ -694,7 +693,7 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) { } for (unsigned i = 0, e = DLLExportedFns.size(); i != e; ++i) { - if (Subtarget->isTargetWindows()) + if (Subtarget->isTargetKnownWindowsMSVC()) name = " /EXPORT:"; else name = " -export:"; diff --git a/lib/Target/X86/X86AsmPrinter.h b/lib/Target/X86/X86AsmPrinter.h index 24a768b..3308cc2 100644 --- a/lib/Target/X86/X86AsmPrinter.h +++ b/lib/Target/X86/X86AsmPrinter.h @@ -10,14 +10,10 @@ #ifndef X86ASMPRINTER_H #define X86ASMPRINTER_H -#include "X86.h" -#include "X86MachineFunctionInfo.h" -#include "X86TargetMachine.h" +#include "X86Subtarget.h" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/CodeGen/ValueTypes.h" #include "llvm/CodeGen/StackMaps.h" -#include "llvm/Support/Compiler.h" +#include "llvm/Target/TargetMachine.h" namespace llvm { @@ -27,59 +23,32 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { const X86Subtarget *Subtarget; StackMaps SM; - // Parses operands of PATCHPOINT and STACKMAP to produce stack map Location - // structures. Returns a result location and an iterator to the operand - // immediately following the operands consumed. - // - // This method is implemented in X86MCInstLower.cpp. - static std::pair<StackMaps::Location, MachineInstr::const_mop_iterator> - stackmapOperandParser(MachineInstr::const_mop_iterator MOI, - MachineInstr::const_mop_iterator MOE, - const TargetMachine &TM); - public: explicit X86AsmPrinter(TargetMachine &TM, MCStreamer &Streamer) - : AsmPrinter(TM, Streamer), SM(*this, stackmapOperandParser) { + : AsmPrinter(TM, Streamer), SM(*this) { Subtarget = &TM.getSubtarget<X86Subtarget>(); } - virtual const char *getPassName() const LLVM_OVERRIDE { + const char *getPassName() const override { return "X86 Assembly / Object Emitter"; } const X86Subtarget &getSubtarget() const { return *Subtarget; } - virtual void EmitStartOfAsmFile(Module &M) LLVM_OVERRIDE; - - virtual void EmitEndOfAsmFile(Module &M) LLVM_OVERRIDE; - - virtual void EmitInstruction(const MachineInstr *MI) LLVM_OVERRIDE; - - void printSymbolOperand(const MachineOperand &MO, raw_ostream &O); - - // These methods are used by the tablegen'erated instruction printer. - void printOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O, - const char *Modifier = 0, unsigned AsmVariant = 0); - void printPCRelImm(const MachineInstr *MI, unsigned OpNo, raw_ostream &O); + void EmitStartOfAsmFile(Module &M) override; - bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O); - virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &OS) LLVM_OVERRIDE; - virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &OS) LLVM_OVERRIDE; + void EmitEndOfAsmFile(Module &M) override; - void printMemReference(const MachineInstr *MI, unsigned Op, raw_ostream &O, - const char *Modifier=NULL); - void printLeaMemReference(const MachineInstr *MI, unsigned Op, raw_ostream &O, - const char *Modifier=NULL); + void EmitInstruction(const MachineInstr *MI) override; - void printIntelMemReference(const MachineInstr *MI, unsigned Op, - raw_ostream &O, const char *Modifier=NULL, - unsigned AsmVariant = 1); + bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, + unsigned AsmVariant, const char *ExtraCode, + raw_ostream &OS) override; + bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, + unsigned AsmVariant, const char *ExtraCode, + raw_ostream &OS) override; - virtual bool runOnMachineFunction(MachineFunction &F) LLVM_OVERRIDE; + bool runOnMachineFunction(MachineFunction &F) override; }; } // end namespace llvm diff --git a/lib/Target/X86/X86CallingConv.h b/lib/Target/X86/X86CallingConv.h index e76f9fd..040da35 100644 --- a/lib/Target/X86/X86CallingConv.h +++ b/lib/Target/X86/X86CallingConv.h @@ -29,6 +29,33 @@ inline bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &, return false; } +inline bool CC_X86_CDeclMethod_SRet(unsigned &ValNo, MVT &ValVT, MVT &LocVT, + CCValAssign::LocInfo &LocInfo, + ISD::ArgFlagsTy &ArgFlags, CCState &State) { + // Swap the order of the first two parameters if the first parameter is sret. + if (ArgFlags.isSRet()) { + assert(ValNo == 0); + assert(ValVT == MVT::i32); + State.AllocateStack(8, 4); + State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT, 4, LocVT, LocInfo)); + + // Indicate that we need to swap the order of the first and second + // parameters by "allocating" register zero. There are no register + // parameters with cdecl methods, so we can use this to communicate to the + // next call. + State.AllocateReg(1); + return true; + } else if (ValNo == 1 && State.isAllocated(1)) { + assert(ValVT == MVT::i32 && "non-i32-sized this param unsupported"); + // Stack was already allocated while processing sret. + State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT, 0, LocVT, LocInfo)); + return true; + } + + // All other args use the C calling convention. + return false; +} + } // End llvm namespace #endif diff --git a/lib/Target/X86/X86CallingConv.td b/lib/Target/X86/X86CallingConv.td index a78b5c0..1cfd827 100644 --- a/lib/Target/X86/X86CallingConv.td +++ b/lib/Target/X86/X86CallingConv.td @@ -357,9 +357,16 @@ def CC_X86_64_WebKit_JS : CallingConv<[ // Promote i8/i16 arguments to i32. CCIfType<[i8, i16], CCPromoteToType<i32>>, - // Integer/FP values are always stored in stack slots that are 8 bytes in size - // and 8-byte aligned. - CCIfType<[i32, i64, f32, f64], CCAssignToStack<8, 8>> + // Only the first integer argument is passed in register. + CCIfType<[i32], CCAssignToReg<[EAX]>>, + CCIfType<[i64], CCAssignToReg<[RAX]>>, + + // The remaining integer arguments are passed on the stack. 32bit integer and + // floating-point arguments are aligned to 4 byte and stored in 4 byte slots. + // 64bit integer and floating-point arguments are aligned to 8 byte and stored + // in 8 byte stack slots. + CCIfType<[i32, f32], CCAssignToStack<4, 4>>, + CCIfType<[i64, f64], CCAssignToStack<8, 8>> ]>; // No explicit register is specified for the AnyReg calling convention. The @@ -453,20 +460,45 @@ def CC_X86_32_FastCall : CallingConv<[ CCDelegateTo<CC_X86_32_Common> ]>; -def CC_X86_32_ThisCall : CallingConv<[ +def CC_X86_32_ThisCall_Common : CallingConv<[ + // The first integer argument is passed in ECX + CCIfType<[i32], CCAssignToReg<[ECX]>>, + + // Otherwise, same as everything else. + CCDelegateTo<CC_X86_32_Common> +]>; + +def CC_X86_32_ThisCall_Mingw : CallingConv<[ + // Promote i8/i16 arguments to i32. + CCIfType<[i8, i16], CCPromoteToType<i32>>, + + CCDelegateTo<CC_X86_32_ThisCall_Common> +]>; + +def CC_X86_32_ThisCall_Win : CallingConv<[ // Promote i8/i16 arguments to i32. CCIfType<[i8, i16], CCPromoteToType<i32>>, // Pass sret arguments indirectly through stack. CCIfSRet<CCAssignToStack<4, 4>>, - // The first integer argument is passed in ECX - CCIfType<[i32], CCAssignToReg<[ECX]>>, + CCDelegateTo<CC_X86_32_ThisCall_Common> +]>; + +def CC_X86_CDeclMethod : CallingConv<[ + // Promote i8/i16 arguments to i32. + CCIfType<[i8, i16], CCPromoteToType<i32>>, + + CCCustom<"CC_X86_CDeclMethod_SRet">, - // Otherwise, same as everything else. CCDelegateTo<CC_X86_32_Common> ]>; +def CC_X86_32_ThisCall : CallingConv<[ + CCIfSubtarget<"isTargetCygMing()", CCDelegateTo<CC_X86_32_ThisCall_Mingw>>, + CCDelegateTo<CC_X86_32_ThisCall_Win> +]>; + def CC_X86_32_FastCC : CallingConv<[ // Handles byval parameters. Note that we can't rely on the delegation // to CC_X86_32_Common for this because that happens after code that @@ -551,6 +583,7 @@ def CC_Intel_OCL_BI : CallingConv<[ def CC_X86_32 : CallingConv<[ CCIfCC<"CallingConv::X86_FastCall", CCDelegateTo<CC_X86_32_FastCall>>, CCIfCC<"CallingConv::X86_ThisCall", CCDelegateTo<CC_X86_32_ThisCall>>, + CCIfCC<"CallingConv::X86_CDeclMethod", CCDelegateTo<CC_X86_CDeclMethod>>, CCIfCC<"CallingConv::Fast", CCDelegateTo<CC_X86_32_FastCC>>, CCIfCC<"CallingConv::GHC", CCDelegateTo<CC_X86_32_GHC>>, CCIfCC<"CallingConv::HiPE", CCDelegateTo<CC_X86_32_HiPE>>, @@ -597,10 +630,26 @@ def CSR_64EHRet : CalleeSavedRegs<(add RAX, RDX, CSR_64)>; def CSR_Win64 : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R15, (sequence "XMM%u", 6, 15))>; -def CSR_MostRegs_64 : CalleeSavedRegs<(add RBX, RCX, RDX, RSI, RDI, R8, R9, R10, +// All GPRs - except r11 +def CSR_64_RT_MostRegs : CalleeSavedRegs<(add CSR_64, RAX, RCX, RDX, RSI, RDI, + R8, R9, R10, RSP)>; + +// All registers - except r11 +def CSR_64_RT_AllRegs : CalleeSavedRegs<(add CSR_64_RT_MostRegs, + (sequence "XMM%u", 0, 15))>; +def CSR_64_RT_AllRegs_AVX : CalleeSavedRegs<(add CSR_64_RT_MostRegs, + (sequence "YMM%u", 0, 15))>; + +def CSR_64_MostRegs : CalleeSavedRegs<(add RBX, RCX, RDX, RSI, RDI, R8, R9, R10, R11, R12, R13, R14, R15, RBP, (sequence "XMM%u", 0, 15))>; +def CSR_64_AllRegs : CalleeSavedRegs<(add CSR_64_MostRegs, RAX, RSP, + (sequence "XMM%u", 16, 31))>; +def CSR_64_AllRegs_AVX : CalleeSavedRegs<(sub (add CSR_64_MostRegs, RAX, RSP, + (sequence "YMM%u", 0, 31)), + (sequence "XMM%u", 0, 15))>; + // Standard C + YMM6-15 def CSR_Win64_Intel_OCL_BI_AVX : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R15, @@ -618,6 +667,6 @@ def CSR_64_Intel_OCL_BI : CalleeSavedRegs<(add CSR_64, def CSR_64_Intel_OCL_BI_AVX : CalleeSavedRegs<(add CSR_64, (sequence "YMM%u", 8, 15))>; -def CSR_64_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add CSR_64, +def CSR_64_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add RBX, RDI, RSI, R14, R15, (sequence "ZMM%u", 16, 31), K4, K5, K6, K7)>; diff --git a/lib/Target/X86/X86CodeEmitter.cpp b/lib/Target/X86/X86CodeEmitter.cpp index 14385ed..f6c4c2e 100644 --- a/lib/Target/X86/X86CodeEmitter.cpp +++ b/lib/Target/X86/X86CodeEmitter.cpp @@ -56,9 +56,9 @@ namespace { MCE(mce), PICBaseOffset(0), Is64BitMode(false), IsPIC(TM.getRelocationModel() == Reloc::PIC_) {} - bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; - virtual const char *getPassName() const { + const char *getPassName() const override { return "X86 Machine Code Emitter"; } @@ -76,7 +76,7 @@ namespace { void emitInstruction(MachineInstr &MI, const MCInstrDesc *Desc); - void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); AU.addRequired<MachineModuleInfo>(); MachineFunctionPass::getAnalysisUsage(AU); @@ -186,10 +186,6 @@ static unsigned determineREX(const MachineInstr &MI) { } switch (Desc.TSFlags & X86II::FormMask) { - case X86II::MRMInitReg: - if (X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0))) - REX |= (1 << 0) | (1 << 2); - break; case X86II::MRMSrcReg: { if (X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0))) REX |= 1 << 2; @@ -216,6 +212,7 @@ static unsigned determineREX(const MachineInstr &MI) { } break; } + case X86II::MRMXm: case X86II::MRM0m: case X86II::MRM1m: case X86II::MRM2m: case X86II::MRM3m: case X86II::MRM4m: case X86II::MRM5m: @@ -658,67 +655,20 @@ void Emitter<CodeEmitter>::emitOpcodePrefix(uint64_t TSFlags, int MemOperand, const MachineInstr &MI, const MCInstrDesc *Desc) const { - // Emit the lock opcode prefix as needed. - if (Desc->TSFlags & X86II::LOCK) - MCE.emitByte(0xF0); - - // Emit segment override opcode prefix as needed. - emitSegmentOverridePrefix(TSFlags, MemOperand, MI); - - // Emit the repeat opcode prefix as needed. - if ((Desc->TSFlags & X86II::Op0Mask) == X86II::REP) - MCE.emitByte(0xF3); - - // Emit the address size opcode prefix as needed. - bool need_address_override; - if (TSFlags & X86II::AdSize) { - need_address_override = true; - } else if (MemOperand == -1) { - need_address_override = false; - } else if (Is64BitMode) { - assert(!Is16BitMemOperand(MI, MemOperand)); - need_address_override = Is32BitMemOperand(MI, MemOperand); - } else { - assert(!Is64BitMemOperand(MI, MemOperand)); - need_address_override = Is16BitMemOperand(MI, MemOperand); - } - - if (need_address_override) - MCE.emitByte(0x67); - // Emit the operand size opcode prefix as needed. - if (TSFlags & X86II::OpSize) + if (((TSFlags & X86II::OpSizeMask) >> X86II::OpSizeShift) == X86II::OpSize16) MCE.emitByte(0x66); - bool Need0FPrefix = false; - switch (Desc->TSFlags & X86II::Op0Mask) { - case X86II::TB: // Two-byte opcode prefix - case X86II::T8: // 0F 38 - case X86II::TA: // 0F 3A - case X86II::A6: // 0F A6 - case X86II::A7: // 0F A7 - Need0FPrefix = true; - break; - case X86II::REP: break; // already handled. - case X86II::T8XS: // F3 0F 38 - case X86II::XS: // F3 0F - MCE.emitByte(0xF3); - Need0FPrefix = true; - break; - case X86II::T8XD: // F2 0F 38 - case X86II::TAXD: // F2 0F 3A - case X86II::XD: // F2 0F - MCE.emitByte(0xF2); - Need0FPrefix = true; - break; - case X86II::D8: case X86II::D9: case X86II::DA: case X86II::DB: - case X86II::DC: case X86II::DD: case X86II::DE: case X86II::DF: - MCE.emitByte(0xD8+ - (((Desc->TSFlags & X86II::Op0Mask)-X86II::D8) - >> X86II::Op0Shift)); - break; // Two-byte opcode prefix - default: llvm_unreachable("Invalid prefix!"); - case 0: break; // No prefix! + switch (Desc->TSFlags & X86II::OpPrefixMask) { + case X86II::PD: // 66 + MCE.emitByte(0x66); + break; + case X86II::XS: // F3 + MCE.emitByte(0xF3); + break; + case X86II::XD: // F2 + MCE.emitByte(0xF2); + break; } // Handle REX prefix. @@ -728,25 +678,21 @@ void Emitter<CodeEmitter>::emitOpcodePrefix(uint64_t TSFlags, } // 0x0F escape code must be emitted just before the opcode. - if (Need0FPrefix) + switch (Desc->TSFlags & X86II::OpMapMask) { + case X86II::TB: // Two-byte opcode map + case X86II::T8: // 0F 38 + case X86II::TA: // 0F 3A MCE.emitByte(0x0F); + break; + } - switch (Desc->TSFlags & X86II::Op0Mask) { - case X86II::T8XD: // F2 0F 38 - case X86II::T8XS: // F3 0F 38 - case X86II::T8: // 0F 38 - MCE.emitByte(0x38); - break; - case X86II::TAXD: // F2 0F 38 - case X86II::TA: // 0F 3A - MCE.emitByte(0x3A); - break; - case X86II::A6: // 0F A6 - MCE.emitByte(0xA6); - break; - case X86II::A7: // 0F A7 - MCE.emitByte(0xA7); - break; + switch (Desc->TSFlags & X86II::OpMapMask) { + case X86II::T8: // 0F 38 + MCE.emitByte(0x38); + break; + case X86II::TA: // 0F 3A + MCE.emitByte(0x3A); + break; } } @@ -778,29 +724,19 @@ template<class CodeEmitter> void Emitter<CodeEmitter>::emitSegmentOverridePrefix(uint64_t TSFlags, int MemOperand, const MachineInstr &MI) const { - switch (TSFlags & X86II::SegOvrMask) { - default: llvm_unreachable("Invalid segment!"); - case 0: - // No segment override, check for explicit one on memory operand. - if (MemOperand != -1) { // If the instruction has a memory operand. - switch (MI.getOperand(MemOperand+X86::AddrSegmentReg).getReg()) { - default: llvm_unreachable("Unknown segment register!"); - case 0: break; - case X86::CS: MCE.emitByte(0x2E); break; - case X86::SS: MCE.emitByte(0x36); break; - case X86::DS: MCE.emitByte(0x3E); break; - case X86::ES: MCE.emitByte(0x26); break; - case X86::FS: MCE.emitByte(0x64); break; - case X86::GS: MCE.emitByte(0x65); break; - } - } - break; - case X86II::FS: - MCE.emitByte(0x64); - break; - case X86II::GS: - MCE.emitByte(0x65); - break; + if (MemOperand < 0) + return; // No memory operand + + // Check for explicit segment override on memory operand. + switch (MI.getOperand(MemOperand+X86::AddrSegmentReg).getReg()) { + default: llvm_unreachable("Unknown segment register!"); + case 0: break; + case X86::CS: MCE.emitByte(0x2E); break; + case X86::SS: MCE.emitByte(0x36); break; + case X86::DS: MCE.emitByte(0x3E); break; + case X86::ES: MCE.emitByte(0x26); break; + case X86::FS: MCE.emitByte(0x64); break; + case X86::GS: MCE.emitByte(0x65); break; } } @@ -809,6 +745,8 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, int MemOperand, const MachineInstr &MI, const MCInstrDesc *Desc) const { + unsigned char Encoding = (TSFlags & X86II::EncodingMask) >> + X86II::EncodingShift; bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V; bool HasVEX_4VOp3 = (TSFlags >> X86II::VEXShift) & X86II::VEX_4VOp3; bool HasMemOp4 = (TSFlags >> X86II::VEXShift) & X86II::MemOp4; @@ -839,9 +777,6 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, // opcode extension, or ignored, depending on the opcode byte) unsigned char VEX_W = 0; - // XOP: Use XOP prefix byte 0x8f instead of VEX. - bool XOP = false; - // VEX_5M (VEX m-mmmmm field): // // 0b00000: Reserved for future use @@ -852,7 +787,7 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, // 0b01000: XOP map select - 08h instructions with imm byte // 0b01001: XOP map select - 09h instructions with no imm byte // 0b01010: XOP map select - 0Ah instructions with imm dword - unsigned char VEX_5M = 0x1; + unsigned char VEX_5M = 0; // VEX_4V (VEX vvvv field): a register specifier // (in 1's complement form) or 1111 if unused. @@ -875,58 +810,28 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, // unsigned char VEX_PP = 0; - // Encode the operand size opcode prefix as needed. - if (TSFlags & X86II::OpSize) - VEX_PP = 0x01; - if ((TSFlags >> X86II::VEXShift) & X86II::VEX_W) VEX_W = 1; - if ((TSFlags >> X86II::VEXShift) & X86II::XOP) - XOP = true; - if ((TSFlags >> X86II::VEXShift) & X86II::VEX_L) VEX_L = 1; - switch (TSFlags & X86II::Op0Mask) { - default: llvm_unreachable("Invalid prefix!"); - case X86II::T8: // 0F 38 - VEX_5M = 0x2; - break; - case X86II::TA: // 0F 3A - VEX_5M = 0x3; - break; - case X86II::T8XS: // F3 0F 38 - VEX_PP = 0x2; - VEX_5M = 0x2; - break; - case X86II::T8XD: // F2 0F 38 - VEX_PP = 0x3; - VEX_5M = 0x2; - break; - case X86II::TAXD: // F2 0F 3A - VEX_PP = 0x3; - VEX_5M = 0x3; - break; - case X86II::XS: // F3 0F - VEX_PP = 0x2; - break; - case X86II::XD: // F2 0F - VEX_PP = 0x3; - break; - case X86II::XOP8: - VEX_5M = 0x8; - break; - case X86II::XOP9: - VEX_5M = 0x9; - break; - case X86II::XOPA: - VEX_5M = 0xA; - break; - case X86II::TB: // VEX_5M/VEX_PP already correct - break; + switch (TSFlags & X86II::OpPrefixMask) { + default: break; // VEX_PP already correct + case X86II::PD: VEX_PP = 0x1; break; // 66 + case X86II::XS: VEX_PP = 0x2; break; // F3 + case X86II::XD: VEX_PP = 0x3; break; // F2 } + switch (TSFlags & X86II::OpMapMask) { + default: llvm_unreachable("Invalid prefix!"); + case X86II::TB: VEX_5M = 0x1; break; // 0F + case X86II::T8: VEX_5M = 0x2; break; // 0F 38 + case X86II::TA: VEX_5M = 0x3; break; // 0F 3A + case X86II::XOP8: VEX_5M = 0x8; break; + case X86II::XOP9: VEX_5M = 0x9; break; + case X86II::XOPA: VEX_5M = 0xA; break; + } // Classify VEX_B, VEX_4V, VEX_R, VEX_X unsigned NumOps = Desc->getNumOperands(); @@ -941,17 +846,8 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, } switch (TSFlags & X86II::FormMask) { - case X86II::MRMInitReg: - // Duplicate register. - if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) - VEX_R = 0x0; - - if (HasVEX_4V) - VEX_4V = getVEXRegisterEncoding(MI, CurOp); - if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) - VEX_B = 0x0; - if (HasVEX_4VOp3) - VEX_4V = getVEXRegisterEncoding(MI, CurOp); + default: llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!"); + case X86II::RawFrm: break; case X86II::MRMDestMem: { // MRMDestMem instructions forms: @@ -1069,8 +965,6 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg())) VEX_B = 0x0; break; - default: // RawFrm - break; } // Emit segment override opcode prefix as needed. @@ -1087,16 +981,21 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags, // | C5h | | R | vvvv | L | pp | // +-----+ +-------------------+ // + // XOP uses a similar prefix: + // +-----+ +--------------+ +-------------------+ + // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp | + // +-----+ +--------------+ +-------------------+ unsigned char LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3); - if (VEX_B && VEX_X && !VEX_W && !XOP && (VEX_5M == 1)) { // 2 byte VEX prefix + // Can this use the 2 byte VEX prefix? + if (Encoding == X86II::VEX && VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) { MCE.emitByte(0xC5); MCE.emitByte(LastByte | (VEX_R << 7)); return; } // 3 byte VEX prefix - MCE.emitByte(XOP ? 0x8F : 0xC4); + MCE.emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4); MCE.emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M); MCE.emitByte(LastByte | (VEX_W << 7)); } @@ -1146,8 +1045,10 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, uint64_t TSFlags = Desc->TSFlags; - // Is this instruction encoded using the AVX VEX prefix? - bool HasVEXPrefix = (TSFlags >> X86II::VEXShift) & X86II::VEX; + // Encoding type for this instruction. + unsigned char Encoding = (TSFlags & X86II::EncodingMask) >> + X86II::EncodingShift; + // It uses the VEX.VVVV field? bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V; bool HasVEX_4VOp3 = (TSFlags >> X86II::VEXShift) & X86II::VEX_4VOp3; @@ -1158,7 +1059,35 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, int MemoryOperand = X86II::getMemoryOperandNo(TSFlags, Opcode); if (MemoryOperand != -1) MemoryOperand += CurOp; - if (!HasVEXPrefix) + // Emit the lock opcode prefix as needed. + if (Desc->TSFlags & X86II::LOCK) + MCE.emitByte(0xF0); + + // Emit segment override opcode prefix as needed. + emitSegmentOverridePrefix(TSFlags, MemoryOperand, MI); + + // Emit the repeat opcode prefix as needed. + if (Desc->TSFlags & X86II::REP) + MCE.emitByte(0xF3); + + // Emit the address size opcode prefix as needed. + bool need_address_override; + if (TSFlags & X86II::AdSize) { + need_address_override = true; + } else if (MemoryOperand < 0) { + need_address_override = false; + } else if (Is64BitMode) { + assert(!Is16BitMemOperand(MI, MemoryOperand)); + need_address_override = Is32BitMemOperand(MI, MemoryOperand); + } else { + assert(!Is64BitMemOperand(MI, MemoryOperand)); + need_address_override = Is16BitMemOperand(MI, MemoryOperand); + } + + if (need_address_override) + MCE.emitByte(0x67); + + if (Encoding == 0) emitOpcodePrefix(TSFlags, MemoryOperand, MI, Desc); else emitVEXOpcodePrefix(TSFlags, MemoryOperand, MI, Desc); @@ -1186,7 +1115,8 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, if (MI.getOperand(0).getSymbolName()[0]) report_fatal_error("JIT does not support inline asm!"); break; - case TargetOpcode::PROLOG_LABEL: + case TargetOpcode::CFI_INSTRUCTION: + break; case TargetOpcode::GC_LABEL: case TargetOpcode::EH_LABEL: MCE.emitLabel(MI.getOperand(0).getMCSymbol()); @@ -1353,6 +1283,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, break; } + case X86II::MRMXr: case X86II::MRM0r: case X86II::MRM1r: case X86II::MRM2r: case X86II::MRM3r: case X86II::MRM4r: case X86II::MRM5r: @@ -1360,8 +1291,9 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). ++CurOp; MCE.emitByte(BaseOpcode); + uint64_t Form = (Desc->TSFlags & X86II::FormMask); emitRegModRMByte(MI.getOperand(CurOp++).getReg(), - (Desc->TSFlags & X86II::FormMask)-X86II::MRM0r); + (Form == X86II::MRMXr) ? 0 : Form-X86II::MRM0r); if (CurOp == NumOps) break; @@ -1390,6 +1322,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, break; } + case X86II::MRMXm: case X86II::MRM0m: case X86II::MRM1m: case X86II::MRM2m: case X86II::MRM3m: case X86II::MRM4m: case X86II::MRM5m: @@ -1401,7 +1334,8 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, X86II::getSizeOfImm(Desc->TSFlags) : 4) : 0; MCE.emitByte(BaseOpcode); - emitMemModRMByte(MI, CurOp, (Desc->TSFlags & X86II::FormMask)-X86II::MRM0m, + uint64_t Form = (Desc->TSFlags & X86II::FormMask); + emitMemModRMByte(MI, CurOp, (Form==X86II::MRMXm) ? 0 : Form - X86II::MRM0m, PCAdj); CurOp += X86::AddrNumOperands; @@ -1432,41 +1366,81 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI, break; } - case X86II::MRMInitReg: + case X86II::MRM_C0: case X86II::MRM_C1: case X86II::MRM_C2: + case X86II::MRM_C3: case X86II::MRM_C4: case X86II::MRM_C8: + case X86II::MRM_C9: case X86II::MRM_CA: case X86II::MRM_CB: + case X86II::MRM_D0: case X86II::MRM_D1: case X86II::MRM_D4: + case X86II::MRM_D5: case X86II::MRM_D6: case X86II::MRM_D8: + case X86II::MRM_D9: case X86II::MRM_DA: case X86II::MRM_DB: + case X86II::MRM_DC: case X86II::MRM_DD: case X86II::MRM_DE: + case X86II::MRM_DF: case X86II::MRM_E0: case X86II::MRM_E1: + case X86II::MRM_E2: case X86II::MRM_E3: case X86II::MRM_E4: + case X86II::MRM_E5: case X86II::MRM_E8: case X86II::MRM_E9: + case X86II::MRM_EA: case X86II::MRM_EB: case X86II::MRM_EC: + case X86II::MRM_ED: case X86II::MRM_EE: case X86II::MRM_F0: + case X86II::MRM_F1: case X86II::MRM_F2: case X86II::MRM_F3: + case X86II::MRM_F4: case X86II::MRM_F5: case X86II::MRM_F6: + case X86II::MRM_F7: case X86II::MRM_F8: case X86II::MRM_F9: + case X86II::MRM_FA: case X86II::MRM_FB: case X86II::MRM_FC: + case X86II::MRM_FD: case X86II::MRM_FE: case X86II::MRM_FF: MCE.emitByte(BaseOpcode); - // Duplicate register, used by things like MOV8r0 (aka xor reg,reg). - emitRegModRMByte(MI.getOperand(CurOp).getReg(), - getX86RegNum(MI.getOperand(CurOp).getReg())); - ++CurOp; - break; - case X86II::MRM_C1: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xC1); - break; - case X86II::MRM_C8: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xC8); - break; - case X86II::MRM_C9: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xC9); - break; - case X86II::MRM_CA: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xCA); - break; - case X86II::MRM_CB: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xCB); - break; - case X86II::MRM_E8: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xE8); - break; - case X86II::MRM_F0: - MCE.emitByte(BaseOpcode); - MCE.emitByte(0xF0); + unsigned char MRM; + switch (TSFlags & X86II::FormMask) { + default: llvm_unreachable("Invalid Form"); + case X86II::MRM_C0: MRM = 0xC0; break; + case X86II::MRM_C1: MRM = 0xC1; break; + case X86II::MRM_C2: MRM = 0xC2; break; + case X86II::MRM_C3: MRM = 0xC3; break; + case X86II::MRM_C4: MRM = 0xC4; break; + case X86II::MRM_C8: MRM = 0xC8; break; + case X86II::MRM_C9: MRM = 0xC9; break; + case X86II::MRM_CA: MRM = 0xCA; break; + case X86II::MRM_CB: MRM = 0xCB; break; + case X86II::MRM_D0: MRM = 0xD0; break; + case X86II::MRM_D1: MRM = 0xD1; break; + case X86II::MRM_D4: MRM = 0xD4; break; + case X86II::MRM_D5: MRM = 0xD5; break; + case X86II::MRM_D6: MRM = 0xD6; break; + case X86II::MRM_D8: MRM = 0xD8; break; + case X86II::MRM_D9: MRM = 0xD9; break; + case X86II::MRM_DA: MRM = 0xDA; break; + case X86II::MRM_DB: MRM = 0xDB; break; + case X86II::MRM_DC: MRM = 0xDC; break; + case X86II::MRM_DD: MRM = 0xDD; break; + case X86II::MRM_DE: MRM = 0xDE; break; + case X86II::MRM_DF: MRM = 0xDF; break; + case X86II::MRM_E0: MRM = 0xE0; break; + case X86II::MRM_E1: MRM = 0xE1; break; + case X86II::MRM_E2: MRM = 0xE2; break; + case X86II::MRM_E3: MRM = 0xE3; break; + case X86II::MRM_E4: MRM = 0xE4; break; + case X86II::MRM_E5: MRM = 0xE5; break; + case X86II::MRM_E8: MRM = 0xE8; break; + case X86II::MRM_E9: MRM = 0xE9; break; + case X86II::MRM_EA: MRM = 0xEA; break; + case X86II::MRM_EB: MRM = 0xEB; break; + case X86II::MRM_EC: MRM = 0xEC; break; + case X86II::MRM_ED: MRM = 0xED; break; + case X86II::MRM_EE: MRM = 0xEE; break; + case X86II::MRM_F0: MRM = 0xF0; break; + case X86II::MRM_F1: MRM = 0xF1; break; + case X86II::MRM_F2: MRM = 0xF2; break; + case X86II::MRM_F3: MRM = 0xF3; break; + case X86II::MRM_F4: MRM = 0xF4; break; + case X86II::MRM_F5: MRM = 0xF5; break; + case X86II::MRM_F6: MRM = 0xF6; break; + case X86II::MRM_F7: MRM = 0xF7; break; + case X86II::MRM_F8: MRM = 0xF8; break; + case X86II::MRM_F9: MRM = 0xF9; break; + case X86II::MRM_FA: MRM = 0xFA; break; + case X86II::MRM_FB: MRM = 0xFB; break; + case X86II::MRM_FC: MRM = 0xFC; break; + case X86II::MRM_FD: MRM = 0xFD; break; + case X86II::MRM_FE: MRM = 0xFE; break; + case X86II::MRM_FF: MRM = 0xFF; break; + } + MCE.emitByte(MRM); break; } diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp index 97f96ab..1aab1ea 100644 --- a/lib/Target/X86/X86FastISel.cpp +++ b/lib/Target/X86/X86FastISel.cpp @@ -15,8 +15,8 @@ #include "X86.h" #include "X86CallingConv.h" -#include "X86ISelLowering.h" #include "X86InstrBuilder.h" +#include "X86MachineFunctionInfo.h" #include "X86RegisterInfo.h" #include "X86Subtarget.h" #include "X86TargetMachine.h" @@ -26,22 +26,22 @@ #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Operator.h" -#include "llvm/Support/CallSite.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/GetElementPtrTypeIterator.h" #include "llvm/Target/TargetOptions.h" using namespace llvm; namespace { -class X86FastISel : public FastISel { +class X86FastISel final : public FastISel { /// Subtarget - Keep a pointer to the X86Subtarget around so that we can /// make the right decision when generating code for different targets. const X86Subtarget *Subtarget; @@ -62,16 +62,16 @@ public: X86ScalarSSEf32 = Subtarget->hasSSE1(); } - virtual bool TargetSelectInstruction(const Instruction *I); + bool TargetSelectInstruction(const Instruction *I) override; /// \brief The specified machine instr operand is a vreg, and that /// vreg is being provided by the specified load instruction. If possible, /// try to fold the load as an operand to the instruction, returning true if /// possible. - virtual bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo, - const LoadInst *LI); + bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo, + const LoadInst *LI) override; - virtual bool FastLowerArguments(); + bool FastLowerArguments() override; #include "X86GenFastISel.inc" @@ -128,11 +128,11 @@ private: bool handleConstantAddresses(const Value *V, X86AddressMode &AM); - unsigned TargetMaterializeConstant(const Constant *C); + unsigned TargetMaterializeConstant(const Constant *C) override; - unsigned TargetMaterializeAlloca(const AllocaInst *C); + unsigned TargetMaterializeAlloca(const AllocaInst *C) override; - unsigned TargetMaterializeFloatZero(const ConstantFP *CF); + unsigned TargetMaterializeFloatZero(const ConstantFP *CF) override; /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is /// computed in an SSE register, not on the X87 floating point stack. @@ -229,7 +229,7 @@ bool X86FastISel::X86FastEmitLoad(EVT VT, const X86AddressMode &AM, ResultReg = createResultReg(RC); addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, - DL, TII.get(Opc), ResultReg), AM); + DbgLoc, TII.get(Opc), ResultReg), AM); return true; } @@ -248,7 +248,7 @@ X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, case MVT::i1: { // Mask out all but lowest bit. unsigned AndResult = createResultReg(&X86::GR8RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::AND8ri), AndResult).addReg(ValReg).addImm(1); ValReg = AndResult; } @@ -289,7 +289,7 @@ X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, } addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, - DL, TII.get(Opc)), AM).addReg(ValReg); + DbgLoc, TII.get(Opc)), AM).addReg(ValReg); return true; } @@ -297,7 +297,7 @@ bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val, const X86AddressMode &AM, bool Aligned) { // Handle 'null' like i32/i64 0. if (isa<ConstantPointerNull>(Val)) - Val = Constant::getNullValue(TD.getIntPtrType(Val->getContext())); + Val = Constant::getNullValue(DL.getIntPtrType(Val->getContext())); // If this is a store of a simple constant, fold the constant into the store. if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) { @@ -318,7 +318,7 @@ bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val, if (Opc) { addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, - DL, TII.get(Opc)), AM) + DbgLoc, TII.get(Opc)), AM) .addImm(Signed ? (uint64_t) CI->getSExtValue() : CI->getZExtValue()); return true; @@ -363,7 +363,7 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) { // it works...). if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) if (const GlobalVariable *GVar = - dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false))) + dyn_cast_or_null<GlobalVariable>(GA->getAliasedGlobal())) if (GVar->isThreadLocal()) return false; @@ -428,7 +428,7 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) { LoadReg = createResultReg(RC); MachineInstrBuilder LoadMI = - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc), LoadReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), LoadReg); addFullAddress(LoadMI, StubAM); // Ok, back to normal mode. @@ -547,14 +547,14 @@ redo_gep: i != e; ++i, ++GTI) { const Value *Op = *i; if (StructType *STy = dyn_cast<StructType>(*GTI)) { - const StructLayout *SL = TD.getStructLayout(STy); + const StructLayout *SL = DL.getStructLayout(STy); Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue()); continue; } // A array/variable index is always of the form i*S where S is the // constant scale size. See if we can push the scale into immediates. - uint64_t S = TD.getTypeAllocSize(GTI.getIndexedType()); + uint64_t S = DL.getTypeAllocSize(GTI.getIndexedType()); for (;;) { if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) { // Constant-offset addressing. @@ -696,8 +696,8 @@ bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) { (AM.Base.Reg != 0 || AM.IndexReg != 0)) return false; - // Can't handle DLLImport. - if (GV->hasDLLImportLinkage()) + // Can't handle DbgLocLImport. + if (GV->hasDLLImportStorageClass()) return false; // Can't handle TLS. @@ -750,7 +750,7 @@ bool X86FastISel::X86SelectStore(const Instruction *I) { return false; unsigned SABIAlignment = - TD.getABITypeAlignment(S->getValueOperand()->getType()); + DL.getABITypeAlignment(S->getValueOperand()->getType()); bool Aligned = S->getAlignment() == 0 || S->getAlignment() >= SABIAlignment; MVT VT; @@ -864,7 +864,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) { // Avoid a cross-class copy. This is very unlikely. if (!SrcRC->contains(DstReg)) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), DstReg).addReg(SrcReg); // Add register to return instruction. @@ -876,19 +876,19 @@ bool X86FastISel::X86SelectRet(const Instruction *I) { // a virtual register in the entry block, so now we copy the value out // and into %rax. We also do the same with %eax for Win32. if (F.hasStructRetAttr() && - (Subtarget->is64Bit() || Subtarget->isTargetWindows())) { + (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC())) { unsigned Reg = X86MFInfo->getSRetReturnReg(); assert(Reg && "SRetReturnReg should have been set in LowerFormalArguments()!"); unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), RetReg).addReg(Reg); RetRegs.push_back(RetReg); } // Now emit the RET. MachineInstrBuilder MIB = - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::RET)); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Subtarget->is64Bit() ? X86::RETQ : X86::RETL)); for (unsigned i = 0, e = RetRegs.size(); i != e; ++i) MIB.addReg(RetRegs[i], RegState::Implicit); return true; @@ -961,14 +961,14 @@ bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, // Handle 'null' like i32/i64 0. if (isa<ConstantPointerNull>(Op1)) - Op1 = Constant::getNullValue(TD.getIntPtrType(Op0->getContext())); + Op1 = Constant::getNullValue(DL.getIntPtrType(Op0->getContext())); // We have two options: compare with register or immediate. If the RHS of // the compare is an immediate that we can fold into this compare, use // CMPri, otherwise use CMPrr. if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) { if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, Op1C)) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(CompareImmOpc)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CompareImmOpc)) .addReg(Op0Reg) .addImm(Op1C->getSExtValue()); return true; @@ -980,7 +980,7 @@ bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, unsigned Op1Reg = getRegForValue(Op1); if (Op1Reg == 0) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(CompareOpc)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CompareOpc)) .addReg(Op0Reg) .addReg(Op1Reg); @@ -1004,10 +1004,10 @@ bool X86FastISel::X86SelectCmp(const Instruction *I) { unsigned EReg = createResultReg(&X86::GR8RegClass); unsigned NPReg = createResultReg(&X86::GR8RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::SETEr), EReg); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SETEr), EReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SETNPr), NPReg); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::AND8rr), ResultReg).addReg(NPReg).addReg(EReg); UpdateValueMap(I, ResultReg); return true; @@ -1018,9 +1018,9 @@ bool X86FastISel::X86SelectCmp(const Instruction *I) { unsigned NEReg = createResultReg(&X86::GR8RegClass); unsigned PReg = createResultReg(&X86::GR8RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::SETNEr), NEReg); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::SETPr), PReg); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::OR8rr),ResultReg) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SETNEr), NEReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SETPr), PReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::OR8rr),ResultReg) .addReg(PReg).addReg(NEReg); UpdateValueMap(I, ResultReg); return true; @@ -1060,7 +1060,7 @@ bool X86FastISel::X86SelectCmp(const Instruction *I) { if (!X86FastEmitCompare(Op0, Op1, VT)) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(SetCCOpc), ResultReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SetCCOpc), ResultReg); UpdateValueMap(I, ResultReg); return true; } @@ -1097,11 +1097,11 @@ bool X86FastISel::X86SelectZExt(const Instruction *I) { } unsigned Result32 = createResultReg(&X86::GR32RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(MovInst), Result32) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(MovInst), Result32) .addReg(ResultReg); ResultReg = createResultReg(&X86::GR64RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::SUBREG_TO_REG), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg) .addImm(0).addReg(Result32).addImm(X86::sub_32bit); } else if (DstVT != MVT::i8) { @@ -1181,17 +1181,17 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) { if (!X86FastEmitCompare(Op0, Op1, VT)) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(BranchOpc)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc)) .addMBB(TrueMBB); if (Predicate == CmpInst::FCMP_UNE) { // X86 requires a second branch to handle UNE (and OEQ, // which is mapped to UNE above). - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::JP_4)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JP_4)) .addMBB(TrueMBB); } - FastEmitBranch(FalseMBB, DL); + FastEmitBranch(FalseMBB, DbgLoc); FuncInfo.MBB->addSuccessor(TrueMBB); return true; } @@ -1212,7 +1212,7 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) { if (TestOpc) { unsigned OpReg = getRegForValue(TI->getOperand(0)); if (OpReg == 0) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TestOpc)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TestOpc)) .addReg(OpReg).addImm(1); unsigned JmpOpc = X86::JNE_4; @@ -1221,9 +1221,9 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) { JmpOpc = X86::JE_4; } - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(JmpOpc)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(JmpOpc)) .addMBB(TrueMBB); - FastEmitBranch(FalseMBB, DL); + FastEmitBranch(FalseMBB, DbgLoc); FuncInfo.MBB->addSuccessor(TrueMBB); return true; } @@ -1236,11 +1236,11 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) { unsigned OpReg = getRegForValue(BI->getCondition()); if (OpReg == 0) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::TEST8ri)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri)) .addReg(OpReg).addImm(1); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::JNE_4)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JNE_4)) .addMBB(TrueMBB); - FastEmitBranch(FalseMBB, DL); + FastEmitBranch(FalseMBB, DbgLoc); FuncInfo.MBB->addSuccessor(TrueMBB); return true; } @@ -1297,18 +1297,18 @@ bool X86FastISel::X86SelectShift(const Instruction *I) { unsigned Op1Reg = getRegForValue(I->getOperand(1)); if (Op1Reg == 0) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), CReg).addReg(Op1Reg); // The shift instruction uses X86::CL. If we defined a super-register // of X86::CL, emit a subreg KILL to precisely describe what we're doing here. if (CReg != X86::CL) - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::KILL), X86::CL) .addReg(CReg, RegState::Kill); unsigned ResultReg = createResultReg(RC); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(OpReg), ResultReg) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpReg), ResultReg) .addReg(Op0Reg); UpdateValueMap(I, ResultReg); return true; @@ -1409,38 +1409,38 @@ bool X86FastISel::X86SelectDivRem(const Instruction *I) { return false; // Move op0 into low-order input register. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg); // Zero-extend or sign-extend into high-order input register. if (OpEntry.OpSignExtend) { if (OpEntry.IsOpSigned) - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpEntry.OpSignExtend)); else { unsigned Zero32 = createResultReg(&X86::GR32RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV32r0), Zero32); // Copy the zero into the appropriate sub/super/identical physical // register. Unfortunately the operations needed are not uniform enough to // fit neatly into the table above. if (VT.SimpleTy == MVT::i16) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), TypeEntry.HighInReg) .addReg(Zero32, 0, X86::sub_16bit); } else if (VT.SimpleTy == MVT::i32) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), TypeEntry.HighInReg) .addReg(Zero32); } else if (VT.SimpleTy == MVT::i64) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg) .addImm(0).addReg(Zero32).addImm(X86::sub_32bit); } } } // Generate the DIV/IDIV instruction. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpEntry.OpDivRem)).addReg(Op1Reg); // For i8 remainder, we can't reference AH directly, as we'll end // up with bogus copies like %R9B = COPY %AH. Reference AX @@ -1456,11 +1456,11 @@ bool X86FastISel::X86SelectDivRem(const Instruction *I) { OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) { unsigned SourceSuperReg = createResultReg(&X86::GR16RegClass); unsigned ResultSuperReg = createResultReg(&X86::GR16RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), SourceSuperReg).addReg(X86::AX); // Shift AX right by 8 bits instead of using AH. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::SHR16ri), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SHR16ri), ResultSuperReg).addReg(SourceSuperReg).addImm(8); // Now reference the 8-bit subreg of the result. @@ -1470,7 +1470,7 @@ bool X86FastISel::X86SelectDivRem(const Instruction *I) { // Copy the result out of the physreg if we haven't already. if (!ResultReg) { ResultReg = createResultReg(TypeEntry.RC); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Copy), ResultReg) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), ResultReg) .addReg(OpEntry.DivRemResultReg); } UpdateValueMap(I, ResultReg); @@ -1508,10 +1508,15 @@ bool X86FastISel::X86SelectSelect(const Instruction *I) { unsigned Op2Reg = getRegForValue(I->getOperand(2)); if (Op2Reg == 0) return false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::TEST8rr)) - .addReg(Op0Reg).addReg(Op0Reg); + // Selects operate on i1, however, Op0Reg is 8 bits width and may contain + // garbage. Indeed, only the less significant bit is supposed to be accurate. + // If we read more than the lsb, we may see non-zero values whereas lsb + // is zero. Therefore, we have to truncate Op0Reg to i1 for the select. + // This is achieved by performing TEST against 1. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri)) + .addReg(Op0Reg).addImm(1); unsigned ResultReg = createResultReg(RC); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc), ResultReg) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg) .addReg(Op1Reg).addReg(Op2Reg); UpdateValueMap(I, ResultReg); return true; @@ -1526,7 +1531,7 @@ bool X86FastISel::X86SelectFPExt(const Instruction *I) { unsigned OpReg = getRegForValue(V); if (OpReg == 0) return false; unsigned ResultReg = createResultReg(&X86::FR64RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::CVTSS2SDrr), ResultReg) .addReg(OpReg); UpdateValueMap(I, ResultReg); @@ -1545,7 +1550,7 @@ bool X86FastISel::X86SelectFPTrunc(const Instruction *I) { unsigned OpReg = getRegForValue(V); if (OpReg == 0) return false; unsigned ResultReg = createResultReg(&X86::FR32RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::CVTSD2SSrr), ResultReg) .addReg(OpReg); UpdateValueMap(I, ResultReg); @@ -1585,7 +1590,7 @@ bool X86FastISel::X86SelectTrunc(const Instruction *I) { (const TargetRegisterClass*)&X86::GR16_ABCDRegClass : (const TargetRegisterClass*)&X86::GR32_ABCDRegClass; unsigned CopyReg = createResultReg(CopyRC); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), CopyReg).addReg(InputReg); InputReg = CopyReg; } @@ -1696,6 +1701,8 @@ bool X86FastISel::X86VisitIntrinsicCall(const IntrinsicInst &I) { const Value *Op1 = I.getArgOperand(0); // The guard's value. const AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); + MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]); + // Grab the frame index. X86AddressMode AM; if (!X86SelectAddress(Slot, AM)) return false; @@ -1711,12 +1718,12 @@ bool X86FastISel::X86VisitIntrinsicCall(const IntrinsicInst &I) { const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); // FIXME may need to add RegState::Debug to any registers produced, // although ESP/EBP should be the only ones at the moment. - addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II), AM). + addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II), AM). addImm(0).addMetadata(DI->getVariable()); return true; } case Intrinsic::trap: { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::TRAP)); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TRAP)); return true; } case Intrinsic::sadd_with_overflow: @@ -1753,13 +1760,14 @@ bool X86FastISel::X86VisitIntrinsicCall(const IntrinsicInst &I) { // The call to CreateRegs builds two sequential registers, to store the // both the returned values. unsigned ResultReg = FuncInfo.CreateRegs(I.getType()); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(OpC), ResultReg) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpC), ResultReg) .addReg(Reg1).addReg(Reg2); unsigned Opc = X86::SETBr; if (I.getIntrinsicID() == Intrinsic::sadd_with_overflow) Opc = X86::SETOr; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc), ResultReg+1); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), + ResultReg + 1); UpdateValueMap(&I, ResultReg, 2); return true; @@ -1833,7 +1841,8 @@ bool X86FastISel::FastLowerArguments() { // Without this, EmitLiveInCopies may eliminate the livein if its only // use is a bitcast (which isn't turned into an instruction). unsigned ResultReg = createResultReg(RC); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, + TII.get(TargetOpcode::COPY), ResultReg).addReg(DstReg, getKillRegState(true)); UpdateValueMap(I, ResultReg); } @@ -1863,7 +1872,7 @@ static unsigned computeBytesPoppedByCallee(const X86Subtarget &Subtarget, const ImmutableCallSite &CS) { if (Subtarget.is64Bit()) return 0; - if (Subtarget.isTargetWindows()) + if (Subtarget.getTargetTriple().isOSMSVCRT()) return 0; CallingConv::ID CC = CS.getCallingConv(); if (CC == CallingConv::Fast || CC == CallingConv::GHC) @@ -1903,6 +1912,10 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { if (isVarArg && isWin64) return false; + // Don't know about inalloca yet. + if (CS.hasInAllocaArgument()) + return false; + // Fast-isel doesn't know about callee-pop yet. if (X86::isCalleePop(CC, Subtarget->is64Bit(), isVarArg, TM.Options.GuaranteedTailCallOpt)) @@ -1958,7 +1971,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { if (CS.paramHasAttr(AttrInd, Attribute::ByVal)) { PointerType *Ty = cast<PointerType>(ArgVal->getType()); Type *ElementTy = Ty->getElementType(); - unsigned FrameSize = TD.getTypeAllocSize(ElementTy); + unsigned FrameSize = DL.getTypeAllocSize(ElementTy); unsigned FrameAlign = CS.getParamAlignment(AttrInd); if (!FrameAlign) FrameAlign = TLI.getByValTypeAlignment(ElementTy); @@ -2015,7 +2028,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { return false; if (ArgVT == MVT::x86mmx) return false; - unsigned OriginalAlignment = TD.getABITypeAlignment(ArgTy); + unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy); Flags.setOrigAlign(OriginalAlignment); Args.push_back(ArgReg); @@ -2040,7 +2053,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { // Issue CALLSEQ_START unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(AdjStackDown)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown)) .addImm(NumBytes); // Process argument: walk the register/memloc assignments, inserting @@ -2104,11 +2117,13 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully // support this. return false; + case CCValAssign::FPExt: + llvm_unreachable("Unexpected loc info!"); } if (VA.isRegLoc()) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), - VA.getLocReg()).addReg(Arg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, + TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(Arg); RegArgs.push_back(VA.getLocReg()); } else { unsigned LocMemOffset = VA.getLocMemOffset(); @@ -2142,8 +2157,8 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { // GOT pointer. if (Subtarget->isPICStyleGOT()) { unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), - X86::EBX).addReg(Base); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, + TII.get(TargetOpcode::COPY), X86::EBX).addReg(Base); } if (Subtarget->is64Bit() && isVarArg && !isWin64) { @@ -2153,7 +2168,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 }; unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::MOV8ri), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri), X86::AL).addImm(NumXMMRegs); } @@ -2166,7 +2181,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { CallOpc = X86::CALL64r; else CallOpc = X86::CALL32r; - MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(CallOpc)) + MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc)) .addReg(CalleeOp); } else { @@ -2200,7 +2215,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { } - MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(CallOpc)); + MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc)); if (MemIntName) MIB.addExternalSymbol(MemIntName, OpFlags); else @@ -2225,7 +2240,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { // Issue CALLSEQ_END unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); const unsigned NumBytesCallee = computeBytesPoppedByCallee(*Subtarget, CS); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(AdjStackUp)) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp)) .addImm(NumBytes).addImm(NumBytesCallee); // Build info for return calling conv lowering code. @@ -2271,10 +2286,11 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { CopyVT = MVT::f80; CopyReg = createResultReg(&X86::RFP80RegClass); } - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::FpPOP_RETVAL), - CopyReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, + TII.get(X86::FpPOP_RETVAL), CopyReg); } else { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, + TII.get(TargetOpcode::COPY), CopyReg).addReg(RVLocs[i].getLocReg()); UsedRegs.push_back(RVLocs[i].getLocReg()); } @@ -2287,11 +2303,11 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) { unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64; unsigned MemSize = ResVT.getSizeInBits()/8; int FI = MFI.CreateStackObject(MemSize, MemSize, false); - addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc)), FI) .addReg(CopyReg); Opc = ResVT == MVT::f32 ? X86::MOVSSrm : X86::MOVSDrm; - addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg + i), FI); } } @@ -2426,7 +2442,7 @@ unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) { Opc = TLI.getPointerTy() == MVT::i32 ? X86::LEA32r : X86::LEA64r; unsigned ResultReg = createResultReg(RC); - addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg), AM); return ResultReg; } @@ -2434,10 +2450,10 @@ unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) { } // MachineConstantPool wants an explicit alignment. - unsigned Align = TD.getPrefTypeAlignment(C->getType()); + unsigned Align = DL.getPrefTypeAlignment(C->getType()); if (Align == 0) { // Alignment of vector types. FIXME! - Align = TD.getTypeAllocSize(C->getType()); + Align = DL.getTypeAllocSize(C->getType()); } // x86-32 PIC requires a PIC base register for constant pools. @@ -2457,7 +2473,7 @@ unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) { // Create the load from the constant pool. unsigned MCPOffset = MCP.getConstantPoolIndex(C, Align); unsigned ResultReg = createResultReg(RC); - addConstantPoolReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + addConstantPoolReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg), MCPOffset, PICBase, OpFlag); @@ -2474,6 +2490,7 @@ unsigned X86FastISel::TargetMaterializeAlloca(const AllocaInst *C) { // X86SelectAddrss, and TargetMaterializeAlloca. if (!FuncInfo.StaticAllocaMap.count(C)) return 0; + assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?"); X86AddressMode AM; if (!X86SelectAddress(C, AM)) @@ -2481,7 +2498,7 @@ unsigned X86FastISel::TargetMaterializeAlloca(const AllocaInst *C) { unsigned Opc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r; const TargetRegisterClass* RC = TLI.getRegClassFor(TLI.getPointerTy()); unsigned ResultReg = createResultReg(RC); - addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg), AM); return ResultReg; } @@ -2520,7 +2537,7 @@ unsigned X86FastISel::TargetMaterializeFloatZero(const ConstantFP *CF) { } unsigned ResultReg = createResultReg(RC); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc), ResultReg); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg); return ResultReg; } @@ -2533,7 +2550,7 @@ bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo, const X86InstrInfo &XII = (const X86InstrInfo&)TII; - unsigned Size = TD.getTypeAllocSize(LI->getType()); + unsigned Size = DL.getTypeAllocSize(LI->getType()); unsigned Alignment = LI->getAlignment(); SmallVector<MachineOperand, 8> AddrOps; diff --git a/lib/Target/X86/X86FixupLEAs.cpp b/lib/Target/X86/X86FixupLEAs.cpp index 38a8351..c2c234b 100644 --- a/lib/Target/X86/X86FixupLEAs.cpp +++ b/lib/Target/X86/X86FixupLEAs.cpp @@ -39,7 +39,7 @@ namespace { /// where appropriate. bool processBasicBlock(MachineFunction &MF, MachineFunction::iterator MFI); - virtual const char *getPassName() const { return "X86 Atom LEA Fixup";} + const char *getPassName() const override { return "X86 Atom LEA Fixup";} /// \brief Given a machine register, look for the instruction /// which writes it in the current basic block. If found, @@ -80,7 +80,7 @@ namespace { /// \brief Loop over all of the basic blocks, /// replacing instructions by equivalent LEA instructions /// if needed and when possible. - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; private: MachineFunction *MF; diff --git a/lib/Target/X86/X86FloatingPoint.cpp b/lib/Target/X86/X86FloatingPoint.cpp index 48470da..7955ade 100644 --- a/lib/Target/X86/X86FloatingPoint.cpp +++ b/lib/Target/X86/X86FloatingPoint.cpp @@ -59,7 +59,7 @@ namespace { memset(RegMap, 0, sizeof(RegMap)); } - virtual void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); AU.addRequired<EdgeBundles>(); AU.addPreservedID(MachineLoopInfoID); @@ -67,9 +67,9 @@ namespace { MachineFunctionPass::getAnalysisUsage(AU); } - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; - virtual const char *getPassName() const { return "X86 FP Stackifier"; } + const char *getPassName() const override { return "X86 FP Stackifier"; } private: const TargetInstrInfo *TII; // Machine instruction info. @@ -432,7 +432,7 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { MachineInstr *PrevMI = 0; if (I != BB.begin()) - PrevMI = prior(I); + PrevMI = std::prev(I); ++NumFP; // Keep track of # of pseudo instrs DEBUG(dbgs() << "\nFPInst:\t" << *MI); @@ -475,10 +475,10 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { } else { MachineBasicBlock::iterator Start = I; // Rewind to first instruction newly inserted. - while (Start != BB.begin() && prior(Start) != PrevI) --Start; + while (Start != BB.begin() && std::prev(Start) != PrevI) --Start; dbgs() << "Inserted instructions:\n\t"; Start->print(dbgs(), &MF.getTarget()); - while (++Start != llvm::next(I)) {} + while (++Start != std::next(I)) {} } dumpStack(); ); @@ -905,7 +905,7 @@ void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { // Kill registers by popping. if (Kills && I != MBB->begin()) { - MachineBasicBlock::iterator I2 = llvm::prior(I); + MachineBasicBlock::iterator I2 = std::prev(I); while (StackTop) { unsigned KReg = getStackEntry(0); if (!(Kills & (1 << KReg))) @@ -1671,8 +1671,10 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) { break; } - case X86::RET: - case X86::RETI: + case X86::RETQ: + case X86::RETL: + case X86::RETIL: + case X86::RETIQ: // If RET has an FP register use operand, pass the first one in ST(0) and // the second one in ST(1). diff --git a/lib/Target/X86/X86FrameLowering.cpp b/lib/Target/X86/X86FrameLowering.cpp index a06ba9d..f0ad4d1 100644 --- a/lib/Target/X86/X86FrameLowering.cpp +++ b/lib/Target/X86/X86FrameLowering.cpp @@ -50,7 +50,7 @@ bool X86FrameLowering::hasFP(const MachineFunction &MF) const { return (MF.getTarget().Options.DisableFramePointerElim(MF) || RegInfo->needsStackRealignment(MF) || MFI->hasVarSizedObjects() || - MFI->isFrameAddressTaken() || MF.hasMSInlineAsm() || + MFI->isFrameAddressTaken() || MFI->hasInlineAsmWithSPAdjust() || MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() || MMI.callsUnwindInit() || MMI.callsEHReturn()); } @@ -107,8 +107,10 @@ static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB, unsigned Opc = MBBI->getOpcode(); switch (Opc) { default: return 0; - case X86::RET: - case X86::RETI: + case X86::RETL: + case X86::RETQ: + case X86::RETIL: + case X86::RETIQ: case X86::TCRETURNdi: case X86::TCRETURNri: case X86::TCRETURNmi: @@ -205,7 +207,7 @@ void mergeSPUpdatesUp(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI, unsigned StackPtr, uint64_t *NumBytes = NULL) { if (MBBI == MBB.begin()) return; - MachineBasicBlock::iterator PI = prior(MBBI); + MachineBasicBlock::iterator PI = std::prev(MBBI); unsigned Opc = PI->getOpcode(); if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 || Opc == X86::ADD32ri || Opc == X86::ADD32ri8 || @@ -233,7 +235,7 @@ void mergeSPUpdatesDown(MachineBasicBlock &MBB, if (MBBI == MBB.end()) return; - MachineBasicBlock::iterator NI = llvm::next(MBBI); + MachineBasicBlock::iterator NI = std::next(MBBI); if (NI == MBB.end()) return; unsigned Opc = NI->getOpcode(); @@ -266,8 +268,8 @@ static int mergeSPUpdates(MachineBasicBlock &MBB, (!doMergeWithPrevious && MBBI == MBB.end())) return 0; - MachineBasicBlock::iterator PI = doMergeWithPrevious ? prior(MBBI) : MBBI; - MachineBasicBlock::iterator NI = doMergeWithPrevious ? 0 : llvm::next(MBBI); + MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI; + MachineBasicBlock::iterator NI = doMergeWithPrevious ? 0 : std::next(MBBI); unsigned Opc = PI->getOpcode(); int Offset = 0; @@ -302,12 +304,14 @@ static bool isEAXLiveIn(MachineFunction &MF) { return false; } -void X86FrameLowering::emitCalleeSavedFrameMoves(MachineFunction &MF, - MCSymbol *Label, - unsigned FramePtr) const { +void X86FrameLowering::emitCalleeSavedFrameMoves( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL, + unsigned FramePtr) const { + MachineFunction &MF = *MBB.getParent(); MachineFrameInfo *MFI = MF.getFrameInfo(); MachineModuleInfo &MMI = MF.getMMI(); const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); + const X86InstrInfo &TII = *TM.getInstrInfo(); // Add callee saved registers to move list. const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo(); @@ -361,7 +365,9 @@ void X86FrameLowering::emitCalleeSavedFrameMoves(MachineFunction &MF, continue; unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); - MMI.addFrameInst(MCCFIInstruction::createOffset(Label, DwarfReg, Offset)); + unsigned CFIIndex = + MMI.addFrameInst(MCCFIInstruction::createOffset(0, DwarfReg, Offset)); + BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)).addCFIIndex(CFIIndex); } } @@ -373,8 +379,9 @@ void X86FrameLowering::emitCalleeSavedFrameMoves(MachineFunction &MF, static bool usesTheStack(const MachineFunction &MF) { const MachineRegisterInfo &MRI = MF.getRegInfo(); - for (MachineRegisterInfo::reg_iterator ri = MRI.reg_begin(X86::EFLAGS), - re = MRI.reg_end(); ri != re; ++ri) + for (MachineRegisterInfo::reg_instr_iterator + ri = MRI.reg_instr_begin(X86::EFLAGS), re = MRI.reg_instr_end(); + ri != re; ++ri) if (ri->isCopy()) return true; @@ -501,19 +508,19 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const { if (needsFrameMoves) { // Mark the place where EBP/RBP was saved. - MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol(); - BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)) - .addSym(FrameLabel); - // Define the current CFA rule to use the provided offset. assert(StackSize); - MMI.addFrameInst( - MCCFIInstruction::createDefCfaOffset(FrameLabel, 2 * stackGrowth)); + unsigned CFIIndex = MMI.addFrameInst( + MCCFIInstruction::createDefCfaOffset(0, 2 * stackGrowth)); + BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex); // Change the rule for the FramePtr to be an "offset" rule. unsigned DwarfFramePtr = RegInfo->getDwarfRegNum(FramePtr, true); - MMI.addFrameInst(MCCFIInstruction::createOffset(FrameLabel, DwarfFramePtr, - 2 * stackGrowth)); + CFIIndex = MMI.addFrameInst( + MCCFIInstruction::createOffset(0, DwarfFramePtr, 2 * stackGrowth)); + BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex); } // Update EBP with the new base value. @@ -524,18 +531,16 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const { if (needsFrameMoves) { // Mark effective beginning of when frame pointer becomes valid. - MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol(); - BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)) - .addSym(FrameLabel); - // Define the current CFA to use the EBP/RBP register. unsigned DwarfFramePtr = RegInfo->getDwarfRegNum(FramePtr, true); - MMI.addFrameInst( - MCCFIInstruction::createDefCfaRegister(FrameLabel, DwarfFramePtr)); + unsigned CFIIndex = MMI.addFrameInst( + MCCFIInstruction::createDefCfaRegister(0, DwarfFramePtr)); + BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex); } // Mark the FramePtr as live-in in every block except the entry. - for (MachineFunction::iterator I = llvm::next(MF.begin()), E = MF.end(); + for (MachineFunction::iterator I = std::next(MF.begin()), E = MF.end(); I != E; ++I) I->addLiveIn(FramePtr); } else { @@ -555,13 +560,12 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const { if (!HasFP && needsFrameMoves) { // Mark callee-saved push instruction. - MCSymbol *Label = MMI.getContext().CreateTempSymbol(); - BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)).addSym(Label); - // Define the current CFA rule to use the provided offset. assert(StackSize); - MMI.addFrameInst( - MCCFIInstruction::createDefCfaOffset(Label, StackOffset)); + unsigned CFIIndex = MMI.addFrameInst( + MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset)); + BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex); StackOffset += stackGrowth; } } @@ -606,16 +610,14 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const { // responsible for adjusting the stack pointer. Touching the stack at 4K // increments is necessary to ensure that the guard pages used by the OS // virtual memory manager are allocated in correct sequence. - if (NumBytes >= 4096 && STI.isOSWindows() && !STI.isTargetEnvMacho()) { + if (NumBytes >= 4096 && STI.isOSWindows() && !STI.isTargetMacho()) { const char *StackProbeSymbol; - bool isSPUpdateNeeded = false; if (Is64Bit) { - if (STI.isTargetCygMing()) - StackProbeSymbol = "___chkstk"; - else { + if (STI.isTargetCygMing()) { + StackProbeSymbol = "___chkstk_ms"; + } else { StackProbeSymbol = "__chkstk"; - isSPUpdateNeeded = true; } } else if (STI.isTargetCygMing()) StackProbeSymbol = "_alloca"; @@ -657,15 +659,15 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const { .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit) .setMIFlag(MachineInstr::FrameSetup); - // MSVC x64's __chkstk does not adjust %rsp itself. - // It also does not clobber %rax so we can reuse it when adjusting %rsp. - if (isSPUpdateNeeded) { + if (Is64Bit) { + // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp + // themself. It also does not clobber %rax so we can reuse it when + // adjusting %rsp. BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64rr), StackPtr) .addReg(StackPtr) .addReg(X86::RAX) .setMIFlag(MachineInstr::FrameSetup); } - if (isEAXAlive) { // Restore EAX MachineInstr *MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), @@ -692,20 +694,19 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const { if (( (!HasFP && NumBytes) || PushedRegs) && needsFrameMoves) { // Mark end of stack pointer adjustment. - MCSymbol *Label = MMI.getContext().CreateTempSymbol(); - BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)) - .addSym(Label); - if (!HasFP && NumBytes) { // Define the current CFA rule to use the provided offset. assert(StackSize); - MMI.addFrameInst(MCCFIInstruction::createDefCfaOffset( - Label, -StackSize + stackGrowth)); + unsigned CFIIndex = MMI.addFrameInst( + MCCFIInstruction::createDefCfaOffset(0, -StackSize + stackGrowth)); + + BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex); } // Emit DWARF info specifying the offsets of the callee-saved registers. if (PushedRegs) - emitCalleeSavedFrameMoves(MF, Label, HasFP ? FramePtr : StackPtr); + emitCalleeSavedFrameMoves(MBB, MBBI, DL, HasFP ? FramePtr : StackPtr); } } @@ -730,8 +731,10 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF, switch (RetOpcode) { default: llvm_unreachable("Can only insert epilog into returning blocks"); - case X86::RET: - case X86::RETI: + case X86::RETQ: + case X86::RETL: + case X86::RETIL: + case X86::RETIQ: case X86::TCRETURNdi: case X86::TCRETURNri: case X86::TCRETURNmi: @@ -781,7 +784,7 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF, // Skip the callee-saved pop instructions. while (MBBI != MBB.begin()) { - MachineBasicBlock::iterator PI = prior(MBBI); + MachineBasicBlock::iterator PI = std::prev(MBBI); unsigned Opc = PI->getOpcode(); if (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::DBG_VALUE && @@ -883,12 +886,13 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF, addReg(JumpTarget.getReg(), RegState::Kill); } - MachineInstr *NewMI = prior(MBBI); + MachineInstr *NewMI = std::prev(MBBI); NewMI->copyImplicitOps(MF, MBBI); // Delete the pseudo instruction TCRETURN. MBB.erase(MBBI); - } else if ((RetOpcode == X86::RET || RetOpcode == X86::RETI) && + } else if ((RetOpcode == X86::RETQ || RetOpcode == X86::RETL || + RetOpcode == X86::RETIQ || RetOpcode == X86::RETIL) && (X86FI->getTCReturnAddrDelta() < 0)) { // Add the return addr area delta back since we are not tail calling. int delta = -1*X86FI->getTCReturnAddrDelta(); @@ -1163,7 +1167,7 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { if (MF.getFunction()->isVarArg()) report_fatal_error("Segmented stacks do not support vararg functions."); if (!STI.isTargetLinux() && !STI.isTargetDarwin() && - !STI.isTargetWin32() && !STI.isTargetFreeBSD()) + !STI.isTargetWin32() && !STI.isTargetWin64() && !STI.isTargetFreeBSD()) report_fatal_error("Segmented stacks not supported on this platform."); MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock(); @@ -1207,6 +1211,9 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { } else if (STI.isTargetDarwin()) { TlsReg = X86::GS; TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90. + } else if (STI.isTargetWin64()) { + TlsReg = X86::GS; + TlsOffset = 0x28; // pvArbitrary, reserved for application use } else if (STI.isTargetFreeBSD()) { TlsReg = X86::FS; TlsOffset = 0x18; @@ -1244,7 +1251,7 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP) .addImm(1).addReg(0).addImm(-StackSize).addReg(0); - if (STI.isTargetLinux() || STI.isTargetWin32()) { + if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64()) { BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg) .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg); } else if (STI.isTargetDarwin()) { @@ -1552,7 +1559,7 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, // sure we restore the stack pointer immediately after the call, there may // be spill code inserted between the CALL and ADJCALLSTACKUP instructions. MachineBasicBlock::iterator B = MBB.begin(); - while (I != B && !llvm::prior(I)->isCall()) + while (I != B && !std::prev(I)->isCall()) --I; MBB.insert(I, New); } diff --git a/lib/Target/X86/X86FrameLowering.h b/lib/Target/X86/X86FrameLowering.h index 3d3b011..f0db8cb 100644 --- a/lib/Target/X86/X86FrameLowering.h +++ b/lib/Target/X86/X86FrameLowering.h @@ -15,7 +15,6 @@ #define X86_FRAMELOWERING_H #include "X86Subtarget.h" -#include "llvm/MC/MCDwarf.h" #include "llvm/Target/TargetFrameLowering.h" namespace llvm { @@ -34,41 +33,42 @@ public: TM(tm), STI(sti) { } - void emitCalleeSavedFrameMoves(MachineFunction &MF, MCSymbol *Label, + void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, DebugLoc DL, unsigned FramePtr) const; /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const; - void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const; + void emitPrologue(MachineFunction &MF) const override; + void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; - void adjustForSegmentedStacks(MachineFunction &MF) const; + void adjustForSegmentedStacks(MachineFunction &MF) const override; - void adjustForHiPEPrologue(MachineFunction &MF) const; + void adjustForHiPEPrologue(MachineFunction &MF) const override; void processFunctionBeforeCalleeSavedScan(MachineFunction &MF, - RegScavenger *RS = NULL) const; + RegScavenger *RS = NULL) const override; bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, - const TargetRegisterInfo *TRI) const; + const TargetRegisterInfo *TRI) const override; bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI, - const std::vector<CalleeSavedInfo> &CSI, - const TargetRegisterInfo *TRI) const; + MachineBasicBlock::iterator MI, + const std::vector<CalleeSavedInfo> &CSI, + const TargetRegisterInfo *TRI) const override; - bool hasFP(const MachineFunction &MF) const; - bool hasReservedCallFrame(const MachineFunction &MF) const; + bool hasFP(const MachineFunction &MF) const override; + bool hasReservedCallFrame(const MachineFunction &MF) const override; - int getFrameIndexOffset(const MachineFunction &MF, int FI) const; + int getFrameIndexOffset(const MachineFunction &MF, int FI) const override; int getFrameIndexReference(const MachineFunction &MF, int FI, - unsigned &FrameReg) const; + unsigned &FrameReg) const override; void eliminateCallFramePseudoInstr(MachineFunction &MF, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI) const; + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI) const override; }; } // End llvm namespace diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp index 36d1690..3e45adb 100644 --- a/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -141,7 +141,7 @@ namespace { /// ISel - X86 specific code to select X86 machine instructions for /// SelectionDAG operations. /// - class X86DAGToDAGISel : public SelectionDAGISel { + class X86DAGToDAGISel final : public SelectionDAGISel { /// Subtarget - Keep a pointer to the X86Subtarget around so that we can /// make the right decision when generating code for different targets. const X86Subtarget *Subtarget; @@ -156,15 +156,15 @@ namespace { Subtarget(&tm.getSubtarget<X86Subtarget>()), OptForSize(false) {} - virtual const char *getPassName() const { + const char *getPassName() const override { return "X86 DAG->DAG Instruction Selection"; } - virtual void EmitFunctionEntryCode(); + void EmitFunctionEntryCode() override; - virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const; + bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; - virtual void PreprocessISelDAG(); + void PreprocessISelDAG() override; inline bool immSext8(SDNode *N) const { return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); @@ -181,7 +181,7 @@ namespace { #include "X86GenDAGISel.inc" private: - SDNode *Select(SDNode *N); + SDNode *Select(SDNode *N) override; SDNode *SelectGather(SDNode *N, unsigned Opc); SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); SDNode *SelectAtomicLoadArith(SDNode *Node, MVT NVT); @@ -219,9 +219,9 @@ namespace { /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for /// inline asm expressions. - virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, - char ConstraintCode, - std::vector<SDValue> &OutOps); + bool SelectInlineAsmMemoryOperand(const SDValue &Op, + char ConstraintCode, + std::vector<SDValue> &OutOps) override; void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); @@ -344,7 +344,7 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { // addl %gs:0, %eax // if the block also has an access to a second TLS address this will save // a load. - // FIXME: This is probably also true for non TLS addresses. + // FIXME: This is probably also true for non-TLS addresses. if (Op1.getOpcode() == X86ISD::Wrapper) { SDValue Val = Op1.getOperand(0); if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 081c558..2a35061 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -15,9 +15,9 @@ #define DEBUG_TYPE "x86-isel" #include "X86ISelLowering.h" #include "Utils/X86ShuffleDecode.h" -#include "X86.h" #include "X86CallingConv.h" #include "X86InstrBuilder.h" +#include "X86MachineFunctionInfo.h" #include "X86TargetMachine.h" #include "X86TargetObjectFile.h" #include "llvm/ADT/SmallSet.h" @@ -31,6 +31,7 @@ #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" @@ -39,12 +40,10 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Support/CallSite.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -180,7 +179,7 @@ static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) { const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>(); bool is64Bit = Subtarget->is64Bit(); - if (Subtarget->isTargetEnvMacho()) { + if (Subtarget->isTargetMacho()) { if (is64Bit) return new X86_64MachoTargetObjectFile(); return new TargetLoweringObjectFileMachO(); @@ -190,7 +189,9 @@ static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) { return new X86LinuxTargetObjectFile(); if (Subtarget->isTargetELF()) return new TargetLoweringObjectFileELF(); - if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho()) + if (Subtarget->isTargetKnownWindowsMSVC()) + return new X86WindowsTargetObjectFile(); + if (Subtarget->isTargetCOFF()) return new TargetLoweringObjectFileCOFF(); llvm_unreachable("unknown subtarget type"); } @@ -249,7 +250,7 @@ void X86TargetLowering::resetOperationActions() { addBypassSlowDiv(64, 16); } - if (Subtarget->isTargetWindows() && !Subtarget->isTargetCygMing()) { + if (Subtarget->isTargetKnownWindowsMSVC()) { // Setup Windows compiler runtime calls. setLibcallName(RTLIB::SDIV_I64, "_alldiv"); setLibcallName(RTLIB::UDIV_I64, "_aulldiv"); @@ -274,7 +275,7 @@ void X86TargetLowering::resetOperationActions() { // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. setUseUnderscoreSetJmp(false); setUseUnderscoreLongJmp(false); - } else if (Subtarget->isTargetMingw()) { + } else if (Subtarget->isTargetWindowsGNU()) { // MS runtime is weird: it exports _setjmp, but longjmp! setUseUnderscoreSetJmp(true); setUseUnderscoreLongJmp(false); @@ -504,7 +505,9 @@ void X86TargetLowering::resetOperationActions() { } setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); - setOperationAction(ISD::BSWAP , MVT::i16 , Expand); + + if (!Subtarget->hasMOVBE()) + setOperationAction(ISD::BSWAP , MVT::i16 , Expand); // These should be promoted to a larger select which is supported. setOperationAction(ISD::SELECT , MVT::i1 , Promote); @@ -632,7 +635,7 @@ void X86TargetLowering::resetOperationActions() { setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); - if (Subtarget->isOSWindows() && !Subtarget->isTargetEnvMacho()) + if (Subtarget->isOSWindows() && !Subtarget->isTargetMacho()) setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ? MVT::i64 : MVT::i32, Custom); else if (TM.Options.EnableSegmentedStacks) @@ -1151,9 +1154,12 @@ void X86TargetLowering::resetOperationActions() { setOperationAction(ISD::FNEG, MVT::v4f64, Custom); setOperationAction(ISD::FABS, MVT::v4f64, Custom); - setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom); - + // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted + // even though v8i16 is a legal type. + setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote); + setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote); setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); + setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); @@ -1306,9 +1312,15 @@ void X86TargetLowering::resetOperationActions() { addRegisterClass(MVT::v8i64, &X86::VR512RegClass); addRegisterClass(MVT::v8f64, &X86::VR512RegClass); + addRegisterClass(MVT::i1, &X86::VK1RegClass); addRegisterClass(MVT::v8i1, &X86::VK8RegClass); addRegisterClass(MVT::v16i1, &X86::VK16RegClass); + setOperationAction(ISD::BR_CC, MVT::i1, Expand); + setOperationAction(ISD::SETCC, MVT::i1, Custom); + setOperationAction(ISD::XOR, MVT::i1, Legal); + setOperationAction(ISD::OR, MVT::i1, Legal); + setOperationAction(ISD::AND, MVT::i1, Legal); setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, Legal); setOperationAction(ISD::LOAD, MVT::v16f32, Legal); setOperationAction(ISD::LOAD, MVT::v8f64, Legal); @@ -1352,11 +1364,12 @@ void X86TargetLowering::resetOperationActions() { setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal); setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal); - setOperationAction(ISD::TRUNCATE, MVT::i1, Legal); + setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom); setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom); + setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); @@ -1370,12 +1383,15 @@ void X86TargetLowering::resetOperationActions() { setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); + setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Legal); setOperationAction(ISD::SETCC, MVT::v16i1, Custom); setOperationAction(ISD::SETCC, MVT::v8i1, Custom); setOperationAction(ISD::MUL, MVT::v8i64, Custom); + setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom); + setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom); setOperationAction(ISD::SELECT, MVT::v8f64, Custom); @@ -1548,14 +1564,13 @@ void X86TargetLowering::resetOperationActions() { EVT X86TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const { if (!VT.isVector()) - return MVT::i8; + return Subtarget->hasAVX512() ? MVT::i1: MVT::i8; - const TargetMachine &TM = getTargetMachine(); - if (!TM.Options.UseSoftFloat && Subtarget->hasAVX512()) + if (Subtarget->hasAVX512()) switch(VT.getVectorNumElements()) { case 8: return MVT::v8i1; case 16: return MVT::v16i1; - } + } return VT.changeVectorElementTypeToInteger(); } @@ -1661,7 +1676,9 @@ bool X86TargetLowering::isSafeMemOpType(MVT VT) const { } bool -X86TargetLowering::allowsUnalignedMemoryAccesses(EVT VT, bool *Fast) const { +X86TargetLowering::allowsUnalignedMemoryAccesses(EVT VT, + unsigned, + bool *Fast) const { if (Fast) *Fast = Subtarget->isUnalignedMemAccessFast(); return true; @@ -1832,6 +1849,9 @@ X86TargetLowering::LowerReturn(SDValue Chain, else if (VA.getLocInfo() == CCValAssign::BCvt) ValToCopy = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), ValToCopy); + assert(VA.getLocInfo() != CCValAssign::FPExt && + "Unexpected FP-extend for return value."); + // If this is x86-64, and we disabled SSE, we can't return FP values, // or SSE or MMX vectors. if ((ValVT == MVT::f32 || ValVT == MVT::f64 || @@ -1886,7 +1906,7 @@ X86TargetLowering::LowerReturn(SDValue Chain, // We saved the argument into a virtual register in the entry block, // so now we copy the value out and into %rax/%eax. if (DAG.getMachineFunction().getFunction()->hasStructRetAttr() && - (Subtarget->is64Bit() || Subtarget->isTargetWindows())) { + (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC())) { MachineFunction &MF = DAG.getMachineFunction(); X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); unsigned Reg = FuncInfo->getSRetReturnReg(); @@ -2175,7 +2195,6 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain, MachineFrameInfo *MFI = MF.getFrameInfo(); bool Is64Bit = Subtarget->is64Bit(); - bool IsWindows = Subtarget->isTargetWindows(); bool IsWin64 = Subtarget->isCallingConvWin64(CallConv); assert(!(isVarArg && IsTailCallConvention(CallConv)) && @@ -2222,6 +2241,8 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain, RC = &X86::VR128RegClass; else if (RegVT == MVT::x86mmx) RC = &X86::VR64RegClass; + else if (RegVT == MVT::i1) + RC = &X86::VK1RegClass; else if (RegVT == MVT::v8i1) RC = &X86::VK8RegClass; else if (RegVT == MVT::v16i1) @@ -2270,7 +2291,7 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain, // Save the argument into a virtual register so that we can access it // from the return points. if (MF.getFunction()->hasStructRetAttr() && - (Subtarget->is64Bit() || Subtarget->isTargetWindows())) { + (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC())) { X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); unsigned Reg = FuncInfo->getSRetReturnReg(); if (!Reg) { @@ -2420,7 +2441,8 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain, } else { FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing. // If this is an sret function, the return should pop the hidden pointer. - if (!Is64Bit && !IsTailCallConvention(CallConv) && !IsWindows && + if (!Is64Bit && !IsTailCallConvention(CallConv) && + !Subtarget->getTargetTriple().isOSMSVCRT() && argsAreStructReturn(Ins) == StackStructReturn) FuncInfo->setBytesToPopOnReturn(4); } @@ -2509,7 +2531,6 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, MachineFunction &MF = DAG.getMachineFunction(); bool Is64Bit = Subtarget->is64Bit(); bool IsWin64 = Subtarget->isCallingConvWin64(CallConv); - bool IsWindows = Subtarget->isTargetWindows(); StructReturnType SR = callIsStructReturn(Outs); bool IsSibcall = false; @@ -2570,9 +2591,21 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, X86Info->setTCReturnAddrDelta(FPDiff); } + unsigned NumBytesToPush = NumBytes; + unsigned NumBytesToPop = NumBytes; + + // If we have an inalloca argument, all stack space has already been allocated + // for us and be right at the top of the stack. We don't support multiple + // arguments passed in memory when using inalloca. + if (!Outs.empty() && Outs.back().Flags.isInAlloca()) { + NumBytesToPush = 0; + assert(ArgLocs.back().getLocMemOffset() == 0 && + "an inalloca argument must be the only memory argument"); + } + if (!IsSibcall) - Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true), - dl); + Chain = DAG.getCALLSEQ_START( + Chain, DAG.getIntPtrConstant(NumBytesToPush, true), dl); SDValue RetAddrFrIdx; // Load return address for tail calls. @@ -2589,10 +2622,14 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, const X86RegisterInfo *RegInfo = static_cast<const X86RegisterInfo*>(getTargetMachine().getRegisterInfo()); for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { + // Skip inalloca arguments, they have already been written. + ISD::ArgFlagsTy Flags = Outs[i].Flags; + if (Flags.isInAlloca()) + continue; + CCValAssign &VA = ArgLocs[i]; EVT RegVT = VA.getLocVT(); SDValue Arg = OutVals[i]; - ISD::ArgFlagsTy Flags = Outs[i].Flags; bool isByVal = Flags.isByVal(); // Promote the value if needed. @@ -2787,7 +2824,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // We should use extra load for direct calls to dllimported functions in // non-JIT mode. const GlobalValue *GV = G->getGlobal(); - if (!GV->hasDLLImportLinkage()) { + if (!GV->hasDLLImportStorageClass()) { unsigned char OpFlags = 0; bool ExtraLoad = false; unsigned WrapperKind = ISD::DELETED_NODE; @@ -2859,8 +2896,9 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVector<SDValue, 8> Ops; if (!IsSibcall && isTailCall) { - Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), - DAG.getIntPtrConstant(0, true), InFlag, dl); + Chain = DAG.getCALLSEQ_END(Chain, + DAG.getIntPtrConstant(NumBytesToPop, true), + DAG.getIntPtrConstant(0, true), InFlag, dl); InFlag = Chain.getValue(1); } @@ -2899,25 +2937,26 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, InFlag = Chain.getValue(1); // Create the CALLSEQ_END node. - unsigned NumBytesForCalleeToPush; + unsigned NumBytesForCalleeToPop; if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, getTargetMachine().Options.GuaranteedTailCallOpt)) - NumBytesForCalleeToPush = NumBytes; // Callee pops everything - else if (!Is64Bit && !IsTailCallConvention(CallConv) && !IsWindows && + NumBytesForCalleeToPop = NumBytes; // Callee pops everything + else if (!Is64Bit && !IsTailCallConvention(CallConv) && + !Subtarget->getTargetTriple().isOSMSVCRT() && SR == StackStructReturn) // If this is a call to a struct-return function, the callee // pops the hidden struct pointer, so we have to push it back. // This is common for Darwin/X86, Linux & Mingw32 targets. // For MSVC Win32 targets, the caller pops the hidden struct pointer. - NumBytesForCalleeToPush = 4; + NumBytesForCalleeToPop = 4; else - NumBytesForCalleeToPush = 0; // Callee pops nothing. + NumBytesForCalleeToPop = 0; // Callee pops nothing. // Returns a flag for retval copy to use. if (!IsSibcall) { Chain = DAG.getCALLSEQ_END(Chain, - DAG.getIntPtrConstant(NumBytes, true), - DAG.getIntPtrConstant(NumBytesForCalleeToPush, + DAG.getIntPtrConstant(NumBytesToPop, true), + DAG.getIntPtrConstant(NumBytesForCalleeToPop, true), InFlag, dl); InFlag = Chain.getValue(1); @@ -3092,9 +3131,13 @@ X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, if (isCalleeStructRet || isCallerStructRet) return false; - // An stdcall caller is expected to clean up its arguments; the callee - // isn't going to do that. - if (!CCMatch && CallerCC == CallingConv::X86_StdCall) + // An stdcall/thiscall caller is expected to clean up its arguments; the + // callee isn't going to do that. + // FIXME: this is more restrictive than needed. We could produce a tailcall + // when the stack adjustment matches. For example, with a thiscall that takes + // only one argument. + if (!CCMatch && (CallerCC == CallingConv::X86_StdCall || + CallerCC == CallingConv::X86_ThisCall)) return false; // Do not sibcall optimize vararg calls unless all arguments are passed via @@ -3415,6 +3458,24 @@ bool X86::isCalleePop(CallingConv::ID CallingConv, } } +/// \brief Return true if the condition is an unsigned comparison operation. +static bool isX86CCUnsigned(unsigned X86CC) { + switch (X86CC) { + default: llvm_unreachable("Invalid integer condition!"); + case X86::COND_E: return true; + case X86::COND_G: return false; + case X86::COND_GE: return false; + case X86::COND_L: return false; + case X86::COND_LE: return false; + case X86::COND_NE: return true; + case X86::COND_B: return true; + case X86::COND_A: return true; + case X86::COND_BE: return true; + case X86::COND_AE: return true; + } + llvm_unreachable("covered switch fell through?!"); +} + /// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 /// specific condition code, returning the condition code and the LHS/RHS of the /// comparison to make. @@ -3533,6 +3594,18 @@ bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { return false; } +/// \brief Returns true if it is beneficial to convert a load of a constant +/// to just the constant itself. +bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, + Type *Ty) const { + assert(Ty->isIntegerTy()); + + unsigned BitSize = Ty->getPrimitiveSizeInBits(); + if (BitSize == 0 || BitSize > 64) + return false; + return true; +} + /// isUndefOrInRange - Return true if Val is undef or if its value falls within /// the specified range (L, H]. static bool isUndefOrInRange(int Val, int Low, int Hi) { @@ -4208,7 +4281,7 @@ static bool isVPERMILPMask(ArrayRef<int> Mask, MVT VT) { unsigned NumLanes = VT.getSizeInBits()/128; unsigned LaneSize = NumElts/NumLanes; // 2 or 4 elements in one lane - + SmallVector<int, 4> ExpectedMaskVal(LaneSize, -1); for (unsigned l = 0; l != NumElts; l += LaneSize) { for (unsigned i = 0; i != LaneSize; ++i) { @@ -4745,6 +4818,13 @@ static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget, SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops, 16); + } else if (VT.getScalarType() == MVT::i1) { + assert(VT.getVectorNumElements() <= 16 && "Unexpected vector type"); + SDValue Cst = DAG.getTargetConstant(0, MVT::i1); + SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, + Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, + Ops, VT.getVectorNumElements()); } else llvm_unreachable("Unexpected vector type"); @@ -5727,16 +5807,20 @@ X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const { bool AllContants = true; uint64_t Immediate = 0; + int NonConstIdx = -1; + bool IsSplat = true; for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) { SDValue In = Op.getOperand(idx); if (In.getOpcode() == ISD::UNDEF) continue; if (!isa<ConstantSDNode>(In)) { AllContants = false; - break; + NonConstIdx = idx; } - if (cast<ConstantSDNode>(In)->getZExtValue()) + else if (cast<ConstantSDNode>(In)->getZExtValue()) Immediate |= (1ULL << idx); + if (In != Op.getOperand(0)) + IsSplat = false; } if (AllContants) { @@ -5746,63 +5830,19 @@ X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const { DAG.getIntPtrConstant(0)); } - // Splat vector (with undefs) - SDValue In = Op.getOperand(0); - for (unsigned i = 1, e = Op.getNumOperands(); i != e; ++i) { - if (Op.getOperand(i) != In && Op.getOperand(i).getOpcode() != ISD::UNDEF) - llvm_unreachable("Unsupported predicate operation"); - } - - SDValue EFLAGS, X86CC; - if (In.getOpcode() == ISD::SETCC) { - SDValue Op0 = In.getOperand(0); - SDValue Op1 = In.getOperand(1); - ISD::CondCode CC = cast<CondCodeSDNode>(In.getOperand(2))->get(); - bool isFP = Op1.getValueType().isFloatingPoint(); - unsigned X86CCVal = TranslateX86CC(CC, isFP, Op0, Op1, DAG); - - assert(X86CCVal != X86::COND_INVALID && "Unsupported predicate operation"); - - X86CC = DAG.getConstant(X86CCVal, MVT::i8); - EFLAGS = EmitCmp(Op0, Op1, X86CCVal, DAG); - EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG); - } else if (In.getOpcode() == X86ISD::SETCC) { - X86CC = In.getOperand(0); - EFLAGS = In.getOperand(1); - } else { - // The algorithm: - // Bit1 = In & 0x1 - // if (Bit1 != 0) - // ZF = 0 - // else - // ZF = 1 - // if (ZF == 0) - // res = allOnes ### CMOVNE -1, %res - // else - // res = allZero - MVT InVT = In.getSimpleValueType(); - SDValue Bit1 = DAG.getNode(ISD::AND, dl, InVT, In, DAG.getConstant(1, InVT)); - EFLAGS = EmitTest(Bit1, X86::COND_NE, DAG); - X86CC = DAG.getConstant(X86::COND_NE, MVT::i8); - } - - if (VT == MVT::v16i1) { - SDValue Cst1 = DAG.getConstant(-1, MVT::i16); - SDValue Cst0 = DAG.getConstant(0, MVT::i16); - SDValue CmovOp = DAG.getNode(X86ISD::CMOV, dl, MVT::i16, - Cst0, Cst1, X86CC, EFLAGS); - return DAG.getNode(ISD::BITCAST, dl, VT, CmovOp); - } - - if (VT == MVT::v8i1) { - SDValue Cst1 = DAG.getConstant(-1, MVT::i32); - SDValue Cst0 = DAG.getConstant(0, MVT::i32); - SDValue CmovOp = DAG.getNode(X86ISD::CMOV, dl, MVT::i32, - Cst0, Cst1, X86CC, EFLAGS); - CmovOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, CmovOp); - return DAG.getNode(ISD::BITCAST, dl, VT, CmovOp); - } - llvm_unreachable("Unsupported predicate operation"); + if (!IsSplat && (NonConstIdx != 0)) + llvm_unreachable("Unsupported BUILD_VECTOR operation"); + MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8; + SDValue Select; + if (IsSplat) + Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), + DAG.getConstant(-1, SelectVT), + DAG.getConstant(0, SelectVT)); + else + Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), + DAG.getConstant((Immediate | 1), SelectVT), + DAG.getConstant(Immediate, SelectVT)); + return DAG.getNode(ISD::BITCAST, dl, VT, Select); } SDValue @@ -5995,8 +6035,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { // For AVX-length vectors, build the individual 128-bit pieces and use // shuffles to put them in place. - if (VT.is256BitVector()) { - SmallVector<SDValue, 32> V; + if (VT.is256BitVector() || VT.is512BitVector()) { + SmallVector<SDValue, 64> V; for (unsigned i = 0; i != NumElems; ++i) V.push_back(Op.getOperand(i)); @@ -6008,7 +6048,9 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { NumElems/2); // Recreate the wider vector with the lower and upper part. - return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl); + if (VT.is256BitVector()) + return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl); + return Concat256BitVectors(Lower, Upper, VT, NumElems, DAG, dl); } // Let legalizer expand 2-wide build_vectors. @@ -6157,14 +6199,27 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { if(ResVT.is256BitVector()) return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl); + if (Op.getNumOperands() == 4) { + MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(), + ResVT.getVectorNumElements()/2); + SDValue V3 = Op.getOperand(2); + SDValue V4 = Op.getOperand(3); + return Concat256BitVectors(Concat128BitVectors(V1, V2, HalfVT, NumElems/2, DAG, dl), + Concat128BitVectors(V3, V4, HalfVT, NumElems/2, DAG, dl), ResVT, NumElems, DAG, dl); + } return Concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl); } static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { - assert(Op.getNumOperands() == 2); + MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType(); + assert((VT.is256BitVector() && Op.getNumOperands() == 2) || + (VT.is512BitVector() && (Op.getNumOperands() == 2 || + Op.getNumOperands() == 4))); - // AVX/AVX-512 can use the vinsertf128 instruction to create 256-bit vectors + // AVX can use the vinsertf128 instruction to create 256-bit vectors // from two other 128-bit ones. + + // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors return LowerAVXCONCAT_VECTORS(Op, DAG); } @@ -6228,6 +6283,59 @@ LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp, return DAG.getNode(ISD::BITCAST, dl, VT, Ret); } +/// In vector type \p VT, return true if the element at index \p InputIdx +/// falls on a different 128-bit lane than \p OutputIdx. +static bool ShuffleCrosses128bitLane(MVT VT, unsigned InputIdx, + unsigned OutputIdx) { + unsigned EltSize = VT.getVectorElementType().getSizeInBits(); + return InputIdx * EltSize / 128 != OutputIdx * EltSize / 128; +} + +/// Generate a PSHUFB if possible. Selects elements from \p V1 according to +/// \p MaskVals. MaskVals[OutputIdx] = InputIdx specifies that we want to +/// shuffle the element at InputIdx in V1 to OutputIdx in the result. If \p +/// MaskVals refers to elements outside of \p V1 or is undef (-1), insert a +/// zero. +static SDValue getPSHUFB(ArrayRef<int> MaskVals, SDValue V1, SDLoc &dl, + SelectionDAG &DAG) { + MVT VT = V1.getSimpleValueType(); + assert(VT.is128BitVector() || VT.is256BitVector()); + + MVT EltVT = VT.getVectorElementType(); + unsigned EltSizeInBytes = EltVT.getSizeInBits() / 8; + unsigned NumElts = VT.getVectorNumElements(); + + SmallVector<SDValue, 32> PshufbMask; + for (unsigned OutputIdx = 0; OutputIdx < NumElts; ++OutputIdx) { + int InputIdx = MaskVals[OutputIdx]; + unsigned InputByteIdx; + + if (InputIdx < 0 || NumElts <= (unsigned)InputIdx) + InputByteIdx = 0x80; + else { + // Cross lane is not allowed. + if (ShuffleCrosses128bitLane(VT, InputIdx, OutputIdx)) + return SDValue(); + InputByteIdx = InputIdx * EltSizeInBytes; + // Index is an byte offset within the 128-bit lane. + InputByteIdx &= 0xf; + } + + for (unsigned j = 0; j < EltSizeInBytes; ++j) { + PshufbMask.push_back(DAG.getConstant(InputByteIdx, MVT::i8)); + if (InputByteIdx != 0x80) + ++InputByteIdx; + } + } + + MVT ShufVT = MVT::getVectorVT(MVT::i8, PshufbMask.size()); + if (ShufVT != VT) + V1 = DAG.getNode(ISD::BITCAST, dl, ShufVT, V1); + return DAG.getNode(X86ISD::PSHUFB, dl, ShufVT, V1, + DAG.getNode(ISD::BUILD_VECTOR, dl, ShufVT, + PshufbMask.data(), PshufbMask.size())); +} + // v8i16 shuffles - Prefer shuffles in the following order: // 1. [all] pshuflw, pshufhw, optional move // 2. [ssse3] 1 x pshufb @@ -6245,8 +6353,12 @@ LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget, // Determine if more than 1 of the words in each of the low and high quadwords // of the result come from the same quadword of one of the two inputs. Undef // mask values count as coming from any quadword, for better codegen. + // + // Lo/HiQuad[i] = j indicates how many words from the ith quad of the input + // feeds this quad. For i, 0 and 1 refer to V1, 2 and 3 refer to V2. unsigned LoQuad[] = { 0, 0, 0, 0 }; unsigned HiQuad[] = { 0, 0, 0, 0 }; + // Indices of quads used. std::bitset<4> InputQuads; for (unsigned i = 0; i < 8; ++i) { unsigned *Quad = i < 4 ? LoQuad : HiQuad; @@ -6283,7 +6395,7 @@ LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget, // For SSSE3, If all 8 words of the result come from only 1 quadword of each // of the two input vectors, shuffle them into one input vector so only a - // single pshufb instruction is necessary. If There are more than 2 input + // single pshufb instruction is necessary. If there are more than 2 input // quads, disable the next transformation since it does not help SSSE3. bool V1Used = InputQuads[0] || InputQuads[1]; bool V2Used = InputQuads[2] || InputQuads[3]; @@ -6375,34 +6487,14 @@ LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget, // mask, and elements that come from V1 in the V2 mask, so that the two // results can be OR'd together. bool TwoInputs = V1Used && V2Used; - for (unsigned i = 0; i != 8; ++i) { - int EltIdx = MaskVals[i] * 2; - int Idx0 = (TwoInputs && (EltIdx >= 16)) ? 0x80 : EltIdx; - int Idx1 = (TwoInputs && (EltIdx >= 16)) ? 0x80 : EltIdx+1; - pshufbMask.push_back(DAG.getConstant(Idx0, MVT::i8)); - pshufbMask.push_back(DAG.getConstant(Idx1, MVT::i8)); - } - V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V1); - V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, - DAG.getNode(ISD::BUILD_VECTOR, dl, - MVT::v16i8, &pshufbMask[0], 16)); + V1 = getPSHUFB(MaskVals, V1, dl, DAG); if (!TwoInputs) return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); // Calculate the shuffle mask for the second input, shuffle it, and // OR it with the first shuffled input. - pshufbMask.clear(); - for (unsigned i = 0; i != 8; ++i) { - int EltIdx = MaskVals[i] * 2; - int Idx0 = (EltIdx < 16) ? 0x80 : EltIdx - 16; - int Idx1 = (EltIdx < 16) ? 0x80 : EltIdx - 15; - pshufbMask.push_back(DAG.getConstant(Idx0, MVT::i8)); - pshufbMask.push_back(DAG.getConstant(Idx1, MVT::i8)); - } - V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V2); - V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, - DAG.getNode(ISD::BUILD_VECTOR, dl, - MVT::v16i8, &pshufbMask[0], 16)); + CommuteVectorShuffleMask(MaskVals, 8); + V2 = getPSHUFB(MaskVals, V2, dl, DAG); V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); } @@ -6484,6 +6576,25 @@ LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget, return NewV; } +/// \brief v16i16 shuffles +/// +/// FIXME: We only support generation of a single pshufb currently. We can +/// generalize the other applicable cases from LowerVECTOR_SHUFFLEv8i16 as +/// well (e.g 2 x pshufb + 1 x por). +static SDValue +LowerVECTOR_SHUFFLEv16i16(SDValue Op, SelectionDAG &DAG) { + ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); + SDValue V1 = SVOp->getOperand(0); + SDValue V2 = SVOp->getOperand(1); + SDLoc dl(SVOp); + + if (V2.getOpcode() != ISD::UNDEF) + return SDValue(); + + SmallVector<int, 16> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end()); + return getPSHUFB(MaskVals, V1, dl, DAG); +} + // v16i8 shuffles - Prefer shuffles in the following order: // 1. [ssse3] 1 x pshufb // 2. [ssse3] 2 x pshufb + 1 x por @@ -6642,22 +6753,7 @@ SDValue LowerVECTOR_SHUFFLEv32i8(ShuffleVectorSDNode *SVOp, CommuteVectorShuffleMask(MaskVals, 32); V1 = V2; } - SmallVector<SDValue, 32> pshufbMask; - for (unsigned i = 0; i != 32; i++) { - int EltIdx = MaskVals[i]; - if (EltIdx < 0 || EltIdx >= 32) - EltIdx = 0x80; - else { - if ((EltIdx >= 16 && i < 16) || (EltIdx < 16 && i >= 16)) - // Cross lane is not allowed. - return SDValue(); - EltIdx &= 0xf; - } - pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); - } - return DAG.getNode(X86ISD::PSHUFB, dl, MVT::v32i8, V1, - DAG.getNode(ISD::BUILD_VECTOR, dl, - MVT::v32i8, &pshufbMask[0], 32)); + return getPSHUFB(MaskVals, V1, dl, DAG); } /// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide @@ -7281,7 +7377,11 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { if (V1IsUndef && V2IsUndef) return DAG.getUNDEF(VT); - assert(!V1IsUndef && "Op 1 of shuffle should not be undef"); + // When we create a shuffle node we put the UNDEF node to second operand, + // but in some cases the first operand may be transformed to UNDEF. + // In this case we should just commute the node. + if (V1IsUndef) + return CommuteVectorShuffle(SVOp, DAG); // Vector shuffle lowering takes 3 steps: // @@ -7450,7 +7550,6 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { CommuteVectorShuffleMask(M, NumElems); std::swap(V1, V2); std::swap(V1IsSplat, V2IsSplat); - Commuted = false; if (isUNPCKLMask(M, VT, HasInt256)) return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); @@ -7538,8 +7637,8 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { // Bitcast is for VPERMPS since mask is v8i32 but node takes v8f32 return DAG.getNode(X86ISD::VPERMV, dl, VT, DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1); - return DAG.getNode(X86ISD::VPERMV3, dl, VT, - DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1, V2); + return DAG.getNode(X86ISD::VPERMV3, dl, VT, V1, + DAG.getNode(ISD::BITCAST, dl, VT, Mask), V2); } //===--------------------------------------------------------------------===// @@ -7555,6 +7654,12 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { return NewOp; } + if (VT == MVT::v16i16 && Subtarget->hasInt256()) { + SDValue NewOp = LowerVECTOR_SHUFFLEv16i16(Op, DAG); + if (NewOp.getNode()) + return NewOp; + } + if (VT == MVT::v16i8) { SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, Subtarget, DAG); if (NewOp.getNode()) @@ -7641,6 +7746,39 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) { return SDValue(); } +/// Extract one bit from mask vector, like v16i1 or v8i1. +/// AVX-512 feature. +SDValue +X86TargetLowering::ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG) const { + SDValue Vec = Op.getOperand(0); + SDLoc dl(Vec); + MVT VecVT = Vec.getSimpleValueType(); + SDValue Idx = Op.getOperand(1); + MVT EltVT = Op.getSimpleValueType(); + + assert((EltVT == MVT::i1) && "Unexpected operands in ExtractBitFromMaskVector"); + + // variable index can't be handled in mask registers, + // extend vector to VR512 + if (!isa<ConstantSDNode>(Idx)) { + MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); + SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Vec); + SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + ExtVT.getVectorElementType(), Ext, Idx); + return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt); + } + + unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + const TargetRegisterClass* rc = getRegClassFor(VecVT); + unsigned MaxSift = rc->getSize()*8 - 1; + Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec, + DAG.getConstant(MaxSift - IdxVal, MVT::i8)); + Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec, + DAG.getConstant(MaxSift, MVT::i8)); + return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec, + DAG.getIntPtrConstant(0)); +} + SDValue X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { @@ -7648,6 +7786,10 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SDValue Vec = Op.getOperand(0); MVT VecVT = Vec.getSimpleValueType(); SDValue Idx = Op.getOperand(1); + + if (Op.getSimpleValueType() == MVT::i1) + return ExtractBitFromMaskVector(Op, DAG); + if (!isa<ConstantSDNode>(Idx)) { if (VecVT.is512BitVector() || (VecVT.is256BitVector() && Subtarget->hasInt256() && @@ -7657,7 +7799,7 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, MVT::getIntegerVT(VecVT.getVectorElementType().getSizeInBits()); MVT MaskVT = MVT::getVectorVT(MaskEltVT, VecVT.getSizeInBits() / MaskEltVT.getSizeInBits()); - + Idx = DAG.getZExtOrTrunc(Idx, dl, MaskEltVT); SDValue Mask = DAG.getNode(X86ISD::VINSERT, dl, MaskVT, getZeroVector(MaskVT, Subtarget, DAG, dl), @@ -8352,7 +8494,8 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { Chain.getValue(1)); } - if (Subtarget->isTargetWindows() || Subtarget->isTargetMingw()) { + if (Subtarget->isTargetKnownWindowsMSVC() || + Subtarget->isTargetWindowsGNU()) { // Just use the implicit TLS architecture // Need to generate someting similar to: // mov rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage @@ -8367,7 +8510,7 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { // If GV is an alias then use the aliasee for determining // thread-localness. if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) - GV = GA->resolveAliasedGlobal(false); + GV = GA->getAliasedGlobal(); SDLoc dl(GA); SDValue Chain = DAG.getEntryNode(); @@ -8380,13 +8523,16 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { : Type::getInt32PtrTy(*DAG.getContext(), 257)); - SDValue TlsArray = Subtarget->is64Bit() ? DAG.getIntPtrConstant(0x58) : - (Subtarget->isTargetMingw() ? DAG.getIntPtrConstant(0x2C) : - DAG.getExternalSymbol("_tls_array", getPointerTy())); + SDValue TlsArray = + Subtarget->is64Bit() + ? DAG.getIntPtrConstant(0x58) + : (Subtarget->isTargetWindowsGNU() + ? DAG.getIntPtrConstant(0x2C) + : DAG.getExternalSymbol("_tls_array", getPointerTy())); - SDValue ThreadPointer = DAG.getLoad(getPointerTy(), dl, Chain, TlsArray, - MachinePointerInfo(Ptr), - false, false, false, 0); + SDValue ThreadPointer = + DAG.getLoad(getPointerTy(), dl, Chain, TlsArray, + MachinePointerInfo(Ptr), false, false, false, 0); // Load the _tls_index variable SDValue IDX = DAG.getExternalSymbol("_tls_index", getPointerTy()); @@ -8422,9 +8568,9 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { /// LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values /// and take a 2 x i32 value to shift plus a shift amount. -SDValue X86TargetLowering::LowerShiftParts(SDValue Op, SelectionDAG &DAG) const{ +static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) { assert(Op.getNumOperands() == 3 && "Not a double-shift!"); - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); unsigned VTBits = VT.getSizeInBits(); SDLoc dl(Op); bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; @@ -8476,12 +8622,12 @@ SDValue X86TargetLowering::LowerShiftParts(SDValue Op, SelectionDAG &DAG) const{ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { - EVT SrcVT = Op.getOperand(0).getValueType(); + MVT SrcVT = Op.getOperand(0).getSimpleValueType(); if (SrcVT.isVector()) return SDValue(); - assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && + assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && "Unknown SINT_TO_FP to lower!"); // These are really Legal; return the operand so the caller accepts it as @@ -8685,15 +8831,14 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op, SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG) const { SDValue N0 = Op.getOperand(0); - EVT SVT = N0.getValueType(); + MVT SVT = N0.getSimpleValueType(); SDLoc dl(Op); assert((SVT == MVT::v4i8 || SVT == MVT::v4i16 || SVT == MVT::v8i8 || SVT == MVT::v8i16) && "Custom UINT_TO_FP is not supported!"); - EVT NVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, - SVT.getVectorNumElements()); + MVT NVT = MVT::getVectorVT(MVT::i32, SVT.getVectorNumElements()); return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N0)); } @@ -8712,8 +8857,8 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, if (DAG.SignBitIsZero(N0)) return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0); - EVT SrcVT = N0.getValueType(); - EVT DstVT = Op.getValueType(); + MVT SrcVT = N0.getSimpleValueType(); + MVT DstVT = Op.getSimpleValueType(); if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) return LowerUINT_TO_FP_i64(Op, DAG); if (SrcVT == MVT::i32 && X86ScalarSSEf64) @@ -8908,7 +9053,7 @@ static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG, return SDValue(); if (Subtarget->hasInt256()) - return DAG.getNode(X86ISD::VZEXT_MOVL, dl, VT, In); + return DAG.getNode(X86ISD::VZEXT, dl, VT, In); SDValue ZeroVec = getZeroVector(InVT, Subtarget, DAG, dl); SDValue Undef = DAG.getUNDEF(InVT); @@ -8927,9 +9072,9 @@ static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG, static SDValue LowerZERO_EXTEND_AVX512(SDValue Op, SelectionDAG &DAG) { - MVT VT = Op->getValueType(0).getSimpleVT(); + MVT VT = Op->getSimpleValueType(0); SDValue In = Op->getOperand(0); - MVT InVT = In.getValueType().getSimpleVT(); + MVT InVT = In.getSimpleValueType(); SDLoc DL(Op); unsigned int NumElts = VT.getVectorNumElements(); if (NumElts != 8 && NumElts != 16) @@ -8990,9 +9135,21 @@ static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget *Subtarget, SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); - MVT VT = Op.getSimpleValueType(); + MVT VT = Op.getSimpleValueType(); SDValue In = Op.getOperand(0); MVT InVT = In.getSimpleValueType(); + + if (VT == MVT::i1) { + assert((InVT.isInteger() && (InVT.getSizeInBits() <= 64)) && + "Invalid scalar TRUNCATE operation"); + if (InVT == MVT::i32) + return SDValue(); + if (InVT.getSizeInBits() == 64) + In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::i32, In); + else if (InVT.getSizeInBits() < 32) + In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In); + return DAG.getNode(ISD::TRUNCATE, DL, VT, In); + } assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && "Invalid TRUNCATE operation"); @@ -9008,6 +9165,7 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In); InVT = ExtVT; } + SDValue Cst = DAG.getTargetConstant(1, InVT.getVectorElementType()); const Constant *C = (dyn_cast<ConstantSDNode>(Cst))->getConstantIntValue(); SDValue CP = DAG.getConstantPool(C, getPointerTy()); @@ -9031,24 +9189,14 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { DAG.getIntPtrConstant(0)); } - // On AVX, v4i64 -> v4i32 becomes a sequence that uses PSHUFD and MOVLHPS. SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, DAG.getIntPtrConstant(0)); SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, DAG.getIntPtrConstant(2)); - OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); - - // The PSHUFD mask: - static const int ShufMask1[] = {0, 2, 0, 0}; - SDValue Undef = DAG.getUNDEF(VT); - OpLo = DAG.getVectorShuffle(VT, DL, OpLo, Undef, ShufMask1); - OpHi = DAG.getVectorShuffle(VT, DL, OpHi, Undef, ShufMask1); - - // The MOVLHPS mask: - static const int ShufMask2[] = {0, 1, 4, 5}; - return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask2); + static const int ShufMask[] = {0, 2, 4, 6}; + return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask); } if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { @@ -9115,8 +9263,7 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { assert(Subtarget->hasFp256() && "256-bit vector without AVX!"); unsigned NumElems = VT.getVectorNumElements(); - EVT NVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), - NumElems * 2); + MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2); SmallVector<int, 16> MaskVec(NumElems * 2, -1); // Prepare truncation shuffle mask @@ -9131,14 +9278,7 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const { - MVT VT = Op.getSimpleValueType(); - if (VT.isVector()) { - if (VT == MVT::v8i16) - return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT, - DAG.getNode(ISD::FP_TO_SINT, SDLoc(Op), - MVT::v8i32, Op.getOperand(0))); - return SDValue(); - } + assert(!Op.getSimpleValueType().isVector()); std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, /*IsSigned=*/ true, /*IsReplace=*/ false); @@ -9186,7 +9326,7 @@ static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) { In, DAG.getUNDEF(SVT))); } -SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) const { +static SDValue LowerFABS(SDValue Op, SelectionDAG &DAG) { LLVMContext *Context = DAG.getContext(); SDLoc dl(Op); MVT VT = Op.getSimpleValueType(); @@ -9204,7 +9344,8 @@ SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) const { C = ConstantFP::get(*Context, APFloat(APFloat::IEEEsingle, APInt(32, ~(1U << 31)))); C = ConstantVector::getSplat(NumElts, C); - SDValue CPIdx = DAG.getConstantPool(C, getPointerTy()); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy()); unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, MachinePointerInfo::getConstantPool(), @@ -9220,7 +9361,7 @@ SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) const { return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask); } -SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const { +static SDValue LowerFNEG(SDValue Op, SelectionDAG &DAG) { LLVMContext *Context = DAG.getContext(); SDLoc dl(Op); MVT VT = Op.getSimpleValueType(); @@ -9238,7 +9379,8 @@ SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const { C = ConstantFP::get(*Context, APFloat(APFloat::IEEEsingle, APInt(32, 1U << 31))); C = ConstantVector::getSplat(NumElts, C); - SDValue CPIdx = DAG.getConstantPool(C, getPointerTy()); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy()); unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, MachinePointerInfo::getConstantPool(), @@ -9255,7 +9397,8 @@ SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const { return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask); } -SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { +static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); LLVMContext *Context = DAG.getContext(); SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); @@ -9291,7 +9434,7 @@ SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0)))); } Constant *C = ConstantVector::get(CV); - SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); + SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16); SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx, MachinePointerInfo::getConstantPool(), false, false, false, 16); @@ -9324,7 +9467,7 @@ SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0)))); } C = ConstantVector::get(CV); - CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); + CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16); SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, MachinePointerInfo::getConstantPool(), false, false, false, 16); @@ -9362,6 +9505,7 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget *Subtarget, SmallVector<SDValue, 8> Opnds; DenseMap<SDValue, unsigned> VecInMap; + SmallVector<SDValue, 8> VecIns; EVT VT = MVT::Other; // Recognize a special case where a vector is casted into wide integer to @@ -9401,6 +9545,7 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget *Subtarget, VT != VecInMap.begin()->first.getValueType()) return SDValue(); M = VecInMap.insert(std::make_pair(ExtractedFromVec, 0)).first; + VecIns.push_back(ExtractedFromVec); } M->second |= 1U << cast<ConstantSDNode>(Idx)->getZExtValue(); } @@ -9409,14 +9554,12 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget *Subtarget, "Not extracted from 128-/256-bit vector."); unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U; - SmallVector<SDValue, 8> VecIns; for (DenseMap<SDValue, unsigned>::const_iterator I = VecInMap.begin(), E = VecInMap.end(); I != E; ++I) { // Quit if not all elements are used. if (I->second != FullMask) return SDValue(); - VecIns.push_back(I->first); } EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; @@ -9444,6 +9587,11 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SelectionDAG &DAG) const { SDLoc dl(Op); + if (Op.getValueType() == MVT::i1) + // KORTEST instruction should be selected + return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, + DAG.getConstant(0, Op.getValueType())); + // CF and OF aren't always set the way we want. Determine which // of these we need. bool NeedCF = false; @@ -9460,15 +9608,17 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, NeedOF = true; break; } - // See if we can use the EFLAGS value from the operand instead of // doing a separate TEST. TEST always sets OF and CF to 0, so unless // we prove that the arithmetic won't overflow, we can't use OF or CF. - if (Op.getResNo() != 0 || NeedOF || NeedCF) + if (Op.getResNo() != 0 || NeedOF || NeedCF) { // Emit a CMP with 0, which is the TEST pattern. + //if (Op.getValueType() == MVT::i1) + // return DAG.getNode(X86ISD::CMP, dl, MVT::i1, Op, + // DAG.getConstant(0, MVT::i1)); return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, DAG.getConstant(0, Op.getValueType())); - + } unsigned Opcode = 0; unsigned NumOperands = 0; @@ -9653,13 +9803,30 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, /// equivalent. SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, SelectionDAG &DAG) const { - if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) + SDLoc dl(Op0); + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) { if (C->getAPIntValue() == 0) return EmitTest(Op0, X86CC, DAG); - SDLoc dl(Op0); + if (Op0.getValueType() == MVT::i1) + llvm_unreachable("Unexpected comparison operation for MVT::i1 operands"); + } + if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 || Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) { + // Do the comparison at i32 if it's smaller, besides the Atom case. + // This avoids subregister aliasing issues. Keep the smaller reference + // if we're optimizing for size, however, as that'll allow better folding + // of memory operations. + if (Op0.getValueType() != MVT::i32 && Op0.getValueType() != MVT::i64 && + !DAG.getMachineFunction().getFunction()->getAttributes().hasAttribute( + AttributeSet::FunctionIndex, Attribute::MinSize) && + !Subtarget->isAtom()) { + unsigned ExtendOp = + isX86CCUnsigned(X86CC) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; + Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0); + Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1); + } // Use SUB instead of CMP to enable CSE between SUB and CMP. SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32); SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, @@ -9844,38 +10011,75 @@ static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) { DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC)); } -static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) { +static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG, + const X86Subtarget *Subtarget) { SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); SDValue CC = Op.getOperand(2); MVT VT = Op.getSimpleValueType(); + SDLoc dl(Op); assert(Op0.getValueType().getVectorElementType().getSizeInBits() >= 32 && Op.getValueType().getScalarType() == MVT::i1 && "Cannot set masked compare for this operation"); ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); - SDLoc dl(Op); - + unsigned Opc = 0; bool Unsigned = false; + bool Swap = false; unsigned SSECC; switch (SetCCOpcode) { default: llvm_unreachable("Unexpected SETCC condition"); case ISD::SETNE: SSECC = 4; break; - case ISD::SETEQ: SSECC = 0; break; - case ISD::SETUGT: Unsigned = true; - case ISD::SETGT: SSECC = 6; break; // NLE - case ISD::SETULT: Unsigned = true; - case ISD::SETLT: SSECC = 1; break; - case ISD::SETUGE: Unsigned = true; - case ISD::SETGE: SSECC = 5; break; // NLT - case ISD::SETULE: Unsigned = true; + case ISD::SETEQ: Opc = X86ISD::PCMPEQM; break; + case ISD::SETUGT: SSECC = 6; Unsigned = true; break; + case ISD::SETLT: Swap = true; //fall-through + case ISD::SETGT: Opc = X86ISD::PCMPGTM; break; + case ISD::SETULT: SSECC = 1; Unsigned = true; break; + case ISD::SETUGE: SSECC = 5; Unsigned = true; break; //NLT + case ISD::SETGE: Swap = true; SSECC = 2; break; // LE + swap + case ISD::SETULE: Unsigned = true; //fall-through case ISD::SETLE: SSECC = 2; break; } - unsigned Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM; + + if (Swap) + std::swap(Op0, Op1); + if (Opc) + return DAG.getNode(Opc, dl, VT, Op0, Op1); + Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM; return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); +} + +/// \brief Try to turn a VSETULT into a VSETULE by modifying its second +/// operand \p Op1. If non-trivial (for example because it's not constant) +/// return an empty value. +static SDValue ChangeVSETULTtoVSETULE(SDValue Op1, SelectionDAG &DAG) +{ + BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op1.getNode()); + if (!BV) + return SDValue(); + + MVT VT = Op1.getSimpleValueType(); + MVT EVT = VT.getVectorElementType(); + unsigned n = VT.getVectorNumElements(); + SmallVector<SDValue, 8> ULTOp1; + + for (unsigned i = 0; i < n; ++i) { + ConstantSDNode *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i)); + if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT) + return SDValue(); + + // Avoid underflow. + APInt Val = Elt->getAPIntValue(); + if (Val == 0) + return SDValue(); + ULTOp1.push_back(DAG.getConstant(Val - 1, EVT)); + } + + return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op1), VT, ULTOp1.data(), + ULTOp1.size()); } static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, @@ -9931,7 +10135,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, if (Subtarget->hasAVX512()) { if (Op1.getValueType().is512BitVector() || (MaskResult && OpVT.getVectorElementType().getSizeInBits() >= 32)) - return LowerIntVSETCC_AVX512(Op, DAG); + return LowerIntVSETCC_AVX512(Op, DAG, Subtarget); // In AVX-512 architecture setcc returns mask with i1 elements, // But there is no compare instruction for i8 and i16 elements. @@ -9949,40 +10153,75 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, // operations may be required for some comparisons. unsigned Opc; bool Swap = false, Invert = false, FlipSigns = false, MinMax = false; - + bool Subus = false; + switch (SetCCOpcode) { default: llvm_unreachable("Unexpected SETCC condition"); case ISD::SETNE: Invert = true; - case ISD::SETEQ: Opc = MaskResult? X86ISD::PCMPEQM: X86ISD::PCMPEQ; break; + case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break; case ISD::SETLT: Swap = true; - case ISD::SETGT: Opc = MaskResult? X86ISD::PCMPGTM: X86ISD::PCMPGT; break; + case ISD::SETGT: Opc = X86ISD::PCMPGT; break; case ISD::SETGE: Swap = true; - case ISD::SETLE: Opc = MaskResult? X86ISD::PCMPGTM: X86ISD::PCMPGT; + case ISD::SETLE: Opc = X86ISD::PCMPGT; Invert = true; break; case ISD::SETULT: Swap = true; - case ISD::SETUGT: Opc = MaskResult? X86ISD::PCMPGTM: X86ISD::PCMPGT; + case ISD::SETUGT: Opc = X86ISD::PCMPGT; FlipSigns = true; break; case ISD::SETUGE: Swap = true; - case ISD::SETULE: Opc = MaskResult? X86ISD::PCMPGTM: X86ISD::PCMPGT; + case ISD::SETULE: Opc = X86ISD::PCMPGT; FlipSigns = true; Invert = true; break; } - + // Special case: Use min/max operations for SETULE/SETUGE MVT VET = VT.getVectorElementType(); bool hasMinMax = (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32)) || (Subtarget->hasSSE2() && (VET == MVT::i8)); - + if (hasMinMax) { switch (SetCCOpcode) { default: break; case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break; case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break; } - + if (MinMax) { Swap = false; Invert = false; FlipSigns = false; } } - + + bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); + if (!MinMax && hasSubus) { + // As another special case, use PSUBUS[BW] when it's profitable. E.g. for + // Op0 u<= Op1: + // t = psubus Op0, Op1 + // pcmpeq t, <0..0> + switch (SetCCOpcode) { + default: break; + case ISD::SETULT: { + // If the comparison is against a constant we can turn this into a + // setule. With psubus, setule does not require a swap. This is + // beneficial because the constant in the register is no longer + // destructed as the destination so it can be hoisted out of a loop. + // Only do this pre-AVX since vpcmp* is no longer destructive. + if (Subtarget->hasAVX()) + break; + SDValue ULEOp1 = ChangeVSETULTtoVSETULE(Op1, DAG); + if (ULEOp1.getNode()) { + Op1 = ULEOp1; + Subus = true; Invert = false; Swap = false; + } + break; + } + // Psubus is better than flip-sign because it requires no inversion. + case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break; + case ISD::SETULE: Subus = true; Invert = false; Swap = false; break; + } + + if (Subus) { + Opc = X86ISD::SUBUS; + FlipSigns = false; + } + } + if (Swap) std::swap(Op0, Op1); @@ -10069,10 +10308,14 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, // If the logical-not of the result is required, perform that now. if (Invert) Result = DAG.getNOT(dl, Result, VT); - + if (MinMax) Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result); + if (Subus) + Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result, + getZeroVector(VT, Subtarget, DAG, dl)); + return Result; } @@ -10082,7 +10325,8 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG); - assert(VT == MVT::i8 && "SetCC type must be 8-bit integer"); + assert(((!Subtarget->hasAVX512() && VT == MVT::i8) || (VT == MVT::i1)) + && "SetCC type must be 8-bit or 1-bit integer"); SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); SDLoc dl(Op); @@ -10114,13 +10358,25 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0); bool Invert = (CC == ISD::SETNE) ^ cast<ConstantSDNode>(Op1)->isNullValue(); - if (!Invert) return Op0; + if (!Invert) + return Op0; CCode = X86::GetOppositeBranchCondition(CCode); - return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, - DAG.getConstant(CCode, MVT::i8), Op0.getOperand(1)); + SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, + DAG.getConstant(CCode, MVT::i8), + Op0.getOperand(1)); + if (VT == MVT::i1) + return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); + return SetCC; } } + if ((Op0.getValueType() == MVT::i1) && (Op1.getOpcode() == ISD::Constant) && + (cast<ConstantSDNode>(Op1)->getZExtValue() == 1) && + (CC == ISD::SETEQ || CC == ISD::SETNE)) { + + ISD::CondCode NewCC = ISD::getSetCCInverse(CC, true); + return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, MVT::i1), NewCC); + } bool isFP = Op1.getSimpleValueType().isFloatingPoint(); unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG); @@ -10129,8 +10385,11 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, DAG); EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG); - return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, - DAG.getConstant(X86CC, MVT::i8), EFLAGS); + SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, + DAG.getConstant(X86CC, MVT::i8), EFLAGS); + if (VT == MVT::i1) + return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); + return SetCC; } // isX86LogicalCmp - Return true if opcode is a X86 logical comparison. @@ -10195,8 +10454,12 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { cast<CondCodeSDNode>(Cond.getOperand(2))->get(), CondOp0, CondOp1); if (SSECC != 8) { - unsigned Opcode = VT == MVT::f32 ? X86ISD::FSETCCss : X86ISD::FSETCCsd; - SDValue Cmp = DAG.getNode(Opcode, DL, VT, CondOp0, CondOp1, + if (Subtarget->hasAVX512()) { + SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1, + DAG.getConstant(SSECC, MVT::i8)); + return DAG.getNode(X86ISD::SELECT, DL, VT, Cmp, Op1, Op2); + } + SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1, DAG.getConstant(SSECC, MVT::i8)); SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2); SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1); @@ -10433,7 +10696,7 @@ static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget *Subtarget, return SDValue(); if (Subtarget->hasInt256()) - return DAG.getNode(X86ISD::VSEXT_MOVL, dl, VT, In); + return DAG.getNode(X86ISD::VSEXT, dl, VT, In); // Optimize vectors in AVX mode // Sign extend v8i16 to v8i32 and @@ -10462,8 +10725,8 @@ static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget *Subtarget, MVT HalfVT = MVT::getVectorVT(VT.getScalarType(), VT.getVectorNumElements()/2); - OpLo = DAG.getNode(X86ISD::VSEXT_MOVL, dl, HalfVT, OpLo); - OpHi = DAG.getNode(X86ISD::VSEXT_MOVL, dl, HalfVT, OpHi); + OpLo = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpLo); + OpHi = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpHi); return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); } @@ -10576,11 +10839,26 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { unsigned X86Opcode; unsigned X86Cond; SDVTList VTs; + // Keep this in sync with LowerXALUO, otherwise we might create redundant + // instructions that can't be removed afterwards (i.e. X86ISD::ADD and + // X86ISD::INC). switch (CondOpcode) { case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break; - case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break; + case ISD::SADDO: + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) + if (C->isOne()) { + X86Opcode = X86ISD::INC; X86Cond = X86::COND_O; + break; + } + X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break; case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break; - case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break; + case ISD::SSUBO: + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) + if (C->isOne()) { + X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O; + break; + } + X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break; case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break; case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break; default: llvm_unreachable("unexpected overflowing operator"); @@ -10764,11 +11042,11 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { SDValue X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { - assert((Subtarget->isTargetCygMing() || Subtarget->isTargetWindows() || + assert((Subtarget->isOSWindows() || getTargetMachine().Options.EnableSegmentedStacks) && "This should be used only on Windows targets or when segmented stacks " "are being used"); - assert(!Subtarget->isTargetEnvMacho() && "Not implemented"); + assert(!Subtarget->isTargetMacho() && "Not implemented"); SDLoc dl(Op); // Get the inputs. @@ -10979,14 +11257,15 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget, // getTargetVShiftByConstNode - Handle vector element shifts where the shift // amount is a constant. Takes immediate version of shift as input. -static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, EVT VT, +static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG) { + MVT ElementType = VT.getVectorElementType(); // Check for ShiftAmt >= element width - if (ShiftAmt >= VT.getVectorElementType().getSizeInBits()) { + if (ShiftAmt >= ElementType.getSizeInBits()) { if (Opc == X86ISD::VSRAI) - ShiftAmt = VT.getVectorElementType().getSizeInBits() - 1; + ShiftAmt = ElementType.getSizeInBits() - 1; else return DAG.getConstant(0, VT); } @@ -10994,12 +11273,63 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, EVT VT, assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI) && "Unknown target vector shift-by-constant node"); + // Fold this packed vector shift into a build vector if SrcOp is a + // vector of Constants or UNDEFs, and SrcOp valuetype is the same as VT. + if (VT == SrcOp.getSimpleValueType() && + ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) { + SmallVector<SDValue, 8> Elts; + unsigned NumElts = SrcOp->getNumOperands(); + ConstantSDNode *ND; + + switch(Opc) { + default: llvm_unreachable(0); + case X86ISD::VSHLI: + for (unsigned i=0; i!=NumElts; ++i) { + SDValue CurrentOp = SrcOp->getOperand(i); + if (CurrentOp->getOpcode() == ISD::UNDEF) { + Elts.push_back(CurrentOp); + continue; + } + ND = cast<ConstantSDNode>(CurrentOp); + const APInt &C = ND->getAPIntValue(); + Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), ElementType)); + } + break; + case X86ISD::VSRLI: + for (unsigned i=0; i!=NumElts; ++i) { + SDValue CurrentOp = SrcOp->getOperand(i); + if (CurrentOp->getOpcode() == ISD::UNDEF) { + Elts.push_back(CurrentOp); + continue; + } + ND = cast<ConstantSDNode>(CurrentOp); + const APInt &C = ND->getAPIntValue(); + Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), ElementType)); + } + break; + case X86ISD::VSRAI: + for (unsigned i=0; i!=NumElts; ++i) { + SDValue CurrentOp = SrcOp->getOperand(i); + if (CurrentOp->getOpcode() == ISD::UNDEF) { + Elts.push_back(CurrentOp); + continue; + } + ND = cast<ConstantSDNode>(CurrentOp); + const APInt &C = ND->getAPIntValue(); + Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), ElementType)); + } + break; + } + + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Elts[0], NumElts); + } + return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8)); } // getTargetVShiftNode - Handle vector element shifts where the shift amount // may or may not be a constant. Takes immediate version of shift as input. -static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, EVT VT, +static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, SDValue ShAmt, SelectionDAG &DAG) { assert(ShAmt.getValueType() == MVT::i32 && "ShAmt is not i32"); @@ -11027,7 +11357,7 @@ static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, EVT VT, // The return type has to be a 128-bit type with the same element // type as the input type. - MVT EltVT = VT.getVectorElementType().getSimpleVT(); + MVT EltVT = VT.getVectorElementType(); EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits()); ShAmt = DAG.getNode(ISD::BITCAST, dl, ShVT, ShAmt); @@ -11210,32 +11540,24 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_avx2_pmaxu_b: case Intrinsic::x86_avx2_pmaxu_w: case Intrinsic::x86_avx2_pmaxu_d: - case Intrinsic::x86_avx512_pmaxu_d: - case Intrinsic::x86_avx512_pmaxu_q: case Intrinsic::x86_sse2_pminu_b: case Intrinsic::x86_sse41_pminuw: case Intrinsic::x86_sse41_pminud: case Intrinsic::x86_avx2_pminu_b: case Intrinsic::x86_avx2_pminu_w: case Intrinsic::x86_avx2_pminu_d: - case Intrinsic::x86_avx512_pminu_d: - case Intrinsic::x86_avx512_pminu_q: case Intrinsic::x86_sse41_pmaxsb: case Intrinsic::x86_sse2_pmaxs_w: case Intrinsic::x86_sse41_pmaxsd: case Intrinsic::x86_avx2_pmaxs_b: case Intrinsic::x86_avx2_pmaxs_w: case Intrinsic::x86_avx2_pmaxs_d: - case Intrinsic::x86_avx512_pmaxs_d: - case Intrinsic::x86_avx512_pmaxs_q: case Intrinsic::x86_sse41_pminsb: case Intrinsic::x86_sse2_pmins_w: case Intrinsic::x86_sse41_pminsd: case Intrinsic::x86_avx2_pmins_b: case Intrinsic::x86_avx2_pmins_w: - case Intrinsic::x86_avx2_pmins_d: - case Intrinsic::x86_avx512_pmins_d: - case Intrinsic::x86_avx512_pmins_q: { + case Intrinsic::x86_avx2_pmins_d: { unsigned Opcode; switch (IntNo) { default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. @@ -11245,8 +11567,6 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_avx2_pmaxu_b: case Intrinsic::x86_avx2_pmaxu_w: case Intrinsic::x86_avx2_pmaxu_d: - case Intrinsic::x86_avx512_pmaxu_d: - case Intrinsic::x86_avx512_pmaxu_q: Opcode = X86ISD::UMAX; break; case Intrinsic::x86_sse2_pminu_b: @@ -11255,8 +11575,6 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_avx2_pminu_b: case Intrinsic::x86_avx2_pminu_w: case Intrinsic::x86_avx2_pminu_d: - case Intrinsic::x86_avx512_pminu_d: - case Intrinsic::x86_avx512_pminu_q: Opcode = X86ISD::UMIN; break; case Intrinsic::x86_sse41_pmaxsb: @@ -11265,8 +11583,6 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_avx2_pmaxs_b: case Intrinsic::x86_avx2_pmaxs_w: case Intrinsic::x86_avx2_pmaxs_d: - case Intrinsic::x86_avx512_pmaxs_d: - case Intrinsic::x86_avx512_pmaxs_q: Opcode = X86ISD::SMAX; break; case Intrinsic::x86_sse41_pminsb: @@ -11275,8 +11591,6 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_avx2_pmins_b: case Intrinsic::x86_avx2_pmins_w: case Intrinsic::x86_avx2_pmins_d: - case Intrinsic::x86_avx512_pmins_d: - case Intrinsic::x86_avx512_pmins_q: Opcode = X86ISD::SMIN; break; } @@ -11289,14 +11603,10 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_sse2_max_pd: case Intrinsic::x86_avx_max_ps_256: case Intrinsic::x86_avx_max_pd_256: - case Intrinsic::x86_avx512_max_ps_512: - case Intrinsic::x86_avx512_max_pd_512: case Intrinsic::x86_sse_min_ps: case Intrinsic::x86_sse2_min_pd: case Intrinsic::x86_avx_min_ps_256: - case Intrinsic::x86_avx_min_pd_256: - case Intrinsic::x86_avx512_min_ps_512: - case Intrinsic::x86_avx512_min_pd_512: { + case Intrinsic::x86_avx_min_pd_256: { unsigned Opcode; switch (IntNo) { default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. @@ -11304,16 +11614,12 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_sse2_max_pd: case Intrinsic::x86_avx_max_ps_256: case Intrinsic::x86_avx_max_pd_256: - case Intrinsic::x86_avx512_max_ps_512: - case Intrinsic::x86_avx512_max_pd_512: Opcode = X86ISD::FMAX; break; case Intrinsic::x86_sse_min_ps: case Intrinsic::x86_sse2_min_pd: case Intrinsic::x86_avx_min_ps_256: case Intrinsic::x86_avx_min_pd_256: - case Intrinsic::x86_avx512_min_ps_512: - case Intrinsic::x86_avx512_min_pd_512: Opcode = X86ISD::FMIN; break; } @@ -11459,14 +11765,14 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); } - case Intrinsic::x86_avx512_kortestz: - case Intrinsic::x86_avx512_kortestc: { - unsigned X86CC = (IntNo == Intrinsic::x86_avx512_kortestz)? X86::COND_E: X86::COND_B; + case Intrinsic::x86_avx512_kortestz_w: + case Intrinsic::x86_avx512_kortestc_w: { + unsigned X86CC = (IntNo == Intrinsic::x86_avx512_kortestz_w)? X86::COND_E: X86::COND_B; SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1)); SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2)); SDValue CC = DAG.getConstant(X86CC, MVT::i8); SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); - SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); + SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test); return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); } @@ -11560,7 +11866,7 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { Opcode = X86ISD::VSRAI; break; } - return getTargetVShiftNode(Opcode, dl, Op.getValueType(), + return getTargetVShiftNode(Opcode, dl, Op.getSimpleValueType(), Op.getOperand(1), Op.getOperand(2), DAG); } @@ -11675,7 +11981,7 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { case Intrinsic::x86_fma_vfmaddsub_ps_512: case Intrinsic::x86_fma_vfmaddsub_pd_512: case Intrinsic::x86_fma_vfmsubadd_ps_512: - case Intrinsic::x86_fma_vfmsubadd_pd_512: { + case Intrinsic::x86_fma_vfmsubadd_pd_512: { unsigned Opc; switch (IntNo) { default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. @@ -11743,9 +12049,9 @@ static SDValue getGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp); assert(C && "Invalid scale type"); SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); - SDValue Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl); - EVT MaskVT = MVT::getVectorVT(MVT::i1, - Index.getValueType().getVectorNumElements()); + SDValue Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl); + EVT MaskVT = MVT::getVectorVT(MVT::i1, + Index.getSimpleValueType().getVectorNumElements()); SDValue MaskInReg = DAG.getConstant(~0, MaskVT); SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); SDValue Disp = DAG.getTargetConstant(0, MVT::i32); @@ -11765,13 +12071,13 @@ static SDValue getMGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, assert(C && "Invalid scale type"); SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); EVT MaskVT = MVT::getVectorVT(MVT::i1, - Index.getValueType().getVectorNumElements()); + Index.getSimpleValueType().getVectorNumElements()); SDValue MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask); SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); SDValue Disp = DAG.getTargetConstant(0, MVT::i32); SDValue Segment = DAG.getRegister(0, MVT::i32); if (Src.getOpcode() == ISD::UNDEF) - Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl); + Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl); SDValue Ops[] = {Src, MaskInReg, Base, Scale, Index, Disp, Segment, Chain}; SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops); SDValue RetOps[] = { SDValue(Res, 0), SDValue(Res, 2) }; @@ -11788,7 +12094,7 @@ static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Disp = DAG.getTargetConstant(0, MVT::i32); SDValue Segment = DAG.getRegister(0, MVT::i32); EVT MaskVT = MVT::getVectorVT(MVT::i1, - Index.getValueType().getVectorNumElements()); + Index.getSimpleValueType().getVectorNumElements()); SDValue MaskInReg = DAG.getConstant(~0, MaskVT); SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); SDValue Ops[] = {Base, Scale, Index, Disp, Segment, MaskInReg, Src, Chain}; @@ -11806,7 +12112,7 @@ static SDValue getMScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Disp = DAG.getTargetConstant(0, MVT::i32); SDValue Segment = DAG.getRegister(0, MVT::i32); EVT MaskVT = MVT::getVectorVT(MVT::i1, - Index.getValueType().getVectorNumElements()); + Index.getSimpleValueType().getVectorNumElements()); SDValue MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask); SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); SDValue Ops[] = {Base, Scale, Index, Disp, Segment, MaskInReg, Src, Chain}; @@ -11861,15 +12167,15 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget, case Intrinsic::x86_avx512_gather_dpi_512: { unsigned Opc; switch (IntNo) { - default: llvm_unreachable("Unexpected intrinsic!"); - case Intrinsic::x86_avx512_gather_qps_512: Opc = X86::VGATHERQPSZrm; break; - case Intrinsic::x86_avx512_gather_qpd_512: Opc = X86::VGATHERQPDZrm; break; - case Intrinsic::x86_avx512_gather_dpd_512: Opc = X86::VGATHERDPDZrm; break; - case Intrinsic::x86_avx512_gather_dps_512: Opc = X86::VGATHERDPSZrm; break; - case Intrinsic::x86_avx512_gather_qpi_512: Opc = X86::VPGATHERQDZrm; break; - case Intrinsic::x86_avx512_gather_qpq_512: Opc = X86::VPGATHERQQZrm; break; - case Intrinsic::x86_avx512_gather_dpi_512: Opc = X86::VPGATHERDDZrm; break; - case Intrinsic::x86_avx512_gather_dpq_512: Opc = X86::VPGATHERDQZrm; break; + default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::x86_avx512_gather_qps_512: Opc = X86::VGATHERQPSZrm; break; + case Intrinsic::x86_avx512_gather_qpd_512: Opc = X86::VGATHERQPDZrm; break; + case Intrinsic::x86_avx512_gather_dpd_512: Opc = X86::VGATHERDPDZrm; break; + case Intrinsic::x86_avx512_gather_dps_512: Opc = X86::VGATHERDPSZrm; break; + case Intrinsic::x86_avx512_gather_qpi_512: Opc = X86::VPGATHERQDZrm; break; + case Intrinsic::x86_avx512_gather_qpq_512: Opc = X86::VPGATHERQQZrm; break; + case Intrinsic::x86_avx512_gather_dpi_512: Opc = X86::VPGATHERDDZrm; break; + case Intrinsic::x86_avx512_gather_dpq_512: Opc = X86::VPGATHERDQZrm; break; } SDValue Chain = Op.getOperand(0); SDValue Index = Op.getOperand(2); @@ -11888,23 +12194,23 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget, case Intrinsic::x86_avx512_gather_dpq_mask_512: { unsigned Opc; switch (IntNo) { - default: llvm_unreachable("Unexpected intrinsic!"); - case Intrinsic::x86_avx512_gather_qps_mask_512: - Opc = X86::VGATHERQPSZrm; break; - case Intrinsic::x86_avx512_gather_qpd_mask_512: - Opc = X86::VGATHERQPDZrm; break; - case Intrinsic::x86_avx512_gather_dpd_mask_512: - Opc = X86::VGATHERDPDZrm; break; - case Intrinsic::x86_avx512_gather_dps_mask_512: - Opc = X86::VGATHERDPSZrm; break; - case Intrinsic::x86_avx512_gather_qpi_mask_512: - Opc = X86::VPGATHERQDZrm; break; - case Intrinsic::x86_avx512_gather_qpq_mask_512: - Opc = X86::VPGATHERQQZrm; break; - case Intrinsic::x86_avx512_gather_dpi_mask_512: - Opc = X86::VPGATHERDDZrm; break; - case Intrinsic::x86_avx512_gather_dpq_mask_512: - Opc = X86::VPGATHERDQZrm; break; + default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::x86_avx512_gather_qps_mask_512: + Opc = X86::VGATHERQPSZrm; break; + case Intrinsic::x86_avx512_gather_qpd_mask_512: + Opc = X86::VGATHERQPDZrm; break; + case Intrinsic::x86_avx512_gather_dpd_mask_512: + Opc = X86::VGATHERDPDZrm; break; + case Intrinsic::x86_avx512_gather_dps_mask_512: + Opc = X86::VGATHERDPSZrm; break; + case Intrinsic::x86_avx512_gather_qpi_mask_512: + Opc = X86::VPGATHERQDZrm; break; + case Intrinsic::x86_avx512_gather_qpq_mask_512: + Opc = X86::VPGATHERQQZrm; break; + case Intrinsic::x86_avx512_gather_dpi_mask_512: + Opc = X86::VPGATHERDDZrm; break; + case Intrinsic::x86_avx512_gather_dpq_mask_512: + Opc = X86::VPGATHERDQZrm; break; } SDValue Chain = Op.getOperand(0); SDValue Src = Op.getOperand(2); @@ -11926,23 +12232,23 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget, case Intrinsic::x86_avx512_scatter_dpi_512: { unsigned Opc; switch (IntNo) { - default: llvm_unreachable("Unexpected intrinsic!"); - case Intrinsic::x86_avx512_scatter_qpd_512: - Opc = X86::VSCATTERQPDZmr; break; - case Intrinsic::x86_avx512_scatter_qps_512: - Opc = X86::VSCATTERQPSZmr; break; - case Intrinsic::x86_avx512_scatter_dpd_512: - Opc = X86::VSCATTERDPDZmr; break; - case Intrinsic::x86_avx512_scatter_dps_512: - Opc = X86::VSCATTERDPSZmr; break; - case Intrinsic::x86_avx512_scatter_qpi_512: - Opc = X86::VPSCATTERQDZmr; break; - case Intrinsic::x86_avx512_scatter_qpq_512: - Opc = X86::VPSCATTERQQZmr; break; - case Intrinsic::x86_avx512_scatter_dpq_512: - Opc = X86::VPSCATTERDQZmr; break; - case Intrinsic::x86_avx512_scatter_dpi_512: - Opc = X86::VPSCATTERDDZmr; break; + default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::x86_avx512_scatter_qpd_512: + Opc = X86::VSCATTERQPDZmr; break; + case Intrinsic::x86_avx512_scatter_qps_512: + Opc = X86::VSCATTERQPSZmr; break; + case Intrinsic::x86_avx512_scatter_dpd_512: + Opc = X86::VSCATTERDPDZmr; break; + case Intrinsic::x86_avx512_scatter_dps_512: + Opc = X86::VSCATTERDPSZmr; break; + case Intrinsic::x86_avx512_scatter_qpi_512: + Opc = X86::VPSCATTERQDZmr; break; + case Intrinsic::x86_avx512_scatter_qpq_512: + Opc = X86::VPSCATTERQQZmr; break; + case Intrinsic::x86_avx512_scatter_dpq_512: + Opc = X86::VPSCATTERDQZmr; break; + case Intrinsic::x86_avx512_scatter_dpi_512: + Opc = X86::VPSCATTERDDZmr; break; } SDValue Chain = Op.getOperand(0); SDValue Base = Op.getOperand(2); @@ -11962,23 +12268,23 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget, case Intrinsic::x86_avx512_scatter_dpq_mask_512: { unsigned Opc; switch (IntNo) { - default: llvm_unreachable("Unexpected intrinsic!"); - case Intrinsic::x86_avx512_scatter_qpd_mask_512: - Opc = X86::VSCATTERQPDZmr; break; - case Intrinsic::x86_avx512_scatter_qps_mask_512: - Opc = X86::VSCATTERQPSZmr; break; - case Intrinsic::x86_avx512_scatter_dpd_mask_512: - Opc = X86::VSCATTERDPDZmr; break; - case Intrinsic::x86_avx512_scatter_dps_mask_512: - Opc = X86::VSCATTERDPSZmr; break; - case Intrinsic::x86_avx512_scatter_qpi_mask_512: - Opc = X86::VPSCATTERQDZmr; break; - case Intrinsic::x86_avx512_scatter_qpq_mask_512: - Opc = X86::VPSCATTERQQZmr; break; - case Intrinsic::x86_avx512_scatter_dpq_mask_512: - Opc = X86::VPSCATTERDQZmr; break; - case Intrinsic::x86_avx512_scatter_dpi_mask_512: - Opc = X86::VPSCATTERDDZmr; break; + default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::x86_avx512_scatter_qpd_mask_512: + Opc = X86::VSCATTERQPDZmr; break; + case Intrinsic::x86_avx512_scatter_qps_mask_512: + Opc = X86::VSCATTERQPSZmr; break; + case Intrinsic::x86_avx512_scatter_dpd_mask_512: + Opc = X86::VSCATTERDPDZmr; break; + case Intrinsic::x86_avx512_scatter_dps_mask_512: + Opc = X86::VSCATTERDPSZmr; break; + case Intrinsic::x86_avx512_scatter_qpi_mask_512: + Opc = X86::VPSCATTERQDZmr; break; + case Intrinsic::x86_avx512_scatter_qpq_mask_512: + Opc = X86::VPSCATTERQQZmr; break; + case Intrinsic::x86_avx512_scatter_dpq_mask_512: + Opc = X86::VPSCATTERDQZmr; break; + case Intrinsic::x86_avx512_scatter_dpi_mask_512: + Opc = X86::VPSCATTERDDZmr; break; } SDValue Chain = Op.getOperand(0); SDValue Base = Op.getOperand(2); @@ -12007,6 +12313,9 @@ SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); MFI->setReturnAddressIsTaken(true); + if (verifyReturnAddressArgumentIsConstant(Op, DAG)) + return SDValue(); + unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); SDLoc dl(Op); EVT PtrVT = getPointerTy(); @@ -12277,7 +12586,7 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, const TargetMachine &TM = MF.getTarget(); const TargetFrameLowering &TFI = *TM.getFrameLowering(); unsigned StackAlignment = TFI.getStackAlignment(); - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); SDLoc DL(Op); // Save FP Control Word to stack slot @@ -12322,7 +12631,7 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, } static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); EVT OpVT = VT; unsigned NumBits = VT.getSizeInBits(); SDLoc dl(Op); @@ -12356,7 +12665,7 @@ static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) { } static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); EVT OpVT = VT; unsigned NumBits = VT.getSizeInBits(); SDLoc dl(Op); @@ -12381,7 +12690,7 @@ static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) { } static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); unsigned NumBits = VT.getSizeInBits(); SDLoc dl(Op); Op = Op.getOperand(0); @@ -12403,7 +12712,7 @@ static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) { // Lower256IntArith - Break a 256-bit integer operation into two new 128-bit // ones, and then concatenate the result back. static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); assert(VT.is256BitVector() && VT.isInteger() && "Unsupported value type for operation"); @@ -12421,8 +12730,8 @@ static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) { SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl); SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl); - MVT EltVT = VT.getVectorElementType().getSimpleVT(); - EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); + MVT EltVT = VT.getVectorElementType(); + MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1), @@ -12430,15 +12739,15 @@ static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) { } static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) { - assert(Op.getValueType().is256BitVector() && - Op.getValueType().isInteger() && + assert(Op.getSimpleValueType().is256BitVector() && + Op.getSimpleValueType().isInteger() && "Only handle AVX 256-bit vector integer operation"); return Lower256IntArith(Op, DAG); } static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) { - assert(Op.getValueType().is256BitVector() && - Op.getValueType().isInteger() && + assert(Op.getSimpleValueType().is256BitVector() && + Op.getSimpleValueType().isInteger() && "Only handle AVX 256-bit vector integer operation"); return Lower256IntArith(Op, DAG); } @@ -12446,7 +12755,7 @@ static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) { static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) { SDLoc dl(Op); - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); // Decompose 256-bit ops into smaller 128-bit ops. if (VT.is256BitVector() && !Subtarget->hasInt256()) @@ -12516,8 +12825,8 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget, } static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) { - EVT VT = Op.getValueType(); - EVT EltTy = VT.getVectorElementType(); + MVT VT = Op.getSimpleValueType(); + MVT EltTy = VT.getVectorElementType(); unsigned NumElts = VT.getVectorNumElements(); SDValue N0 = Op.getOperand(0); SDLoc dl(Op); @@ -12572,7 +12881,7 @@ static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) { static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); SDLoc dl(Op); SDValue R = Op.getOperand(0); SDValue Amt = Op.getOperand(1); @@ -12604,7 +12913,7 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG, // Make a large shift. SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, R, ShiftAmt, - DAG); + DAG); SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL); // Zero out the rightmost bits. SmallVector<SDValue, 16> V(16, @@ -12699,7 +13008,7 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG, Amt.getOpcode() == ISD::BITCAST && Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { Amt = Amt.getOperand(0); - unsigned Ratio = Amt.getValueType().getVectorNumElements() / + unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() / VT.getVectorNumElements(); unsigned RatioInLog2 = Log2_32_Ceil(Ratio); uint64_t ShiftAmt = 0; @@ -12744,7 +13053,7 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG, static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, const X86Subtarget* Subtarget) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); SDLoc dl(Op); SDValue R = Op.getOperand(0); SDValue Amt = Op.getOperand(1); @@ -12814,7 +13123,7 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, default: llvm_unreachable("Unknown shift opcode!"); case ISD::SHL: - switch (VT.getSimpleVT().SimpleTy) { + switch (VT.SimpleTy) { default: return SDValue(); case MVT::v2i64: case MVT::v4i32: @@ -12827,7 +13136,7 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, return getTargetVShiftNode(X86ISD::VSHLI, dl, VT, R, BaseShAmt, DAG); } case ISD::SRA: - switch (VT.getSimpleVT().SimpleTy) { + switch (VT.SimpleTy) { default: return SDValue(); case MVT::v4i32: case MVT::v8i16: @@ -12838,7 +13147,7 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, R, BaseShAmt, DAG); } case ISD::SRL: - switch (VT.getSimpleVT().SimpleTy) { + switch (VT.SimpleTy) { default: return SDValue(); case MVT::v2i64: case MVT::v4i32: @@ -12861,7 +13170,7 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, Amt.getOpcode() == ISD::BITCAST && Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { Amt = Amt.getOperand(0); - unsigned Ratio = Amt.getValueType().getVectorNumElements() / + unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() / VT.getVectorNumElements(); std::vector<SDValue> Vals(Ratio); for (unsigned i = 0; i != Ratio; ++i) @@ -12889,7 +13198,7 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget, SelectionDAG &DAG) { - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); SDLoc dl(Op); SDValue R = Op.getOperand(0); SDValue Amt = Op.getOperand(1); @@ -12922,6 +13231,39 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget, return Op; } + // If possible, lower this packed shift into a vector multiply instead of + // expanding it into a sequence of scalar shifts. + // Do this only if the vector shift count is a constant build_vector. + if (Op.getOpcode() == ISD::SHL && + (VT == MVT::v8i16 || VT == MVT::v4i32 || + (Subtarget->hasInt256() && VT == MVT::v16i16)) && + ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) { + SmallVector<SDValue, 8> Elts; + EVT SVT = VT.getScalarType(); + unsigned SVTBits = SVT.getSizeInBits(); + const APInt &One = APInt(SVTBits, 1); + unsigned NumElems = VT.getVectorNumElements(); + + for (unsigned i=0; i !=NumElems; ++i) { + SDValue Op = Amt->getOperand(i); + if (Op->getOpcode() == ISD::UNDEF) { + Elts.push_back(Op); + continue; + } + + ConstantSDNode *ND = cast<ConstantSDNode>(Op); + const APInt &C = APInt(SVTBits, ND->getAPIntValue().getZExtValue()); + uint64_t ShAmt = C.getZExtValue(); + if (ShAmt >= SVTBits) { + Elts.push_back(DAG.getUNDEF(SVT)); + continue; + } + Elts.push_back(DAG.getConstant(One.shl(ShAmt), SVT)); + } + SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Elts[0], NumElems); + return DAG.getNode(ISD::MUL, dl, VT, R, BV); + } + // Lower SHL with variable shift amount. if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT)); @@ -12931,6 +13273,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget, Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); return DAG.getNode(ISD::MUL, dl, VT, Op, R); } + if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) { assert(Subtarget->hasSSE2() && "Need SSE2 for pslli/pcmpeq."); @@ -12974,10 +13317,23 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget, return R; } + // It's worth extending once and using the v8i32 shifts for 16-bit types, but + // the extra overheads to get from v16i8 to v8i32 make the existing SSE + // solution better. + if (Subtarget->hasInt256() && VT == MVT::v8i16) { + MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16; + unsigned ExtOpc = + Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + R = DAG.getNode(ExtOpc, dl, NewVT, R); + Amt = DAG.getNode(ISD::ANY_EXTEND, dl, NewVT, Amt); + return DAG.getNode(ISD::TRUNCATE, dl, VT, + DAG.getNode(Op.getOpcode(), dl, NewVT, R, Amt)); + } + // Decompose 256-bit shifts into smaller 128-bit shifts. if (VT.is256BitVector()) { unsigned NumElems = VT.getVectorNumElements(); - MVT EltVT = VT.getVectorElementType().getSimpleVT(); + MVT EltVT = VT.getVectorElementType(); EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); // Extract the two vectors @@ -13095,7 +13451,7 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); if (!Subtarget->hasSSE2() || !VT.isVector()) return SDValue(); @@ -13103,7 +13459,7 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, unsigned BitsDiff = VT.getScalarType().getSizeInBits() - ExtraVT.getScalarType().getSizeInBits(); - switch (VT.getSimpleVT().SimpleTy) { + switch (VT.SimpleTy) { default: return SDValue(); case MVT::v8i32: case MVT::v16i16: @@ -13118,7 +13474,7 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl); SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl); - MVT EltVT = VT.getVectorElementType().getSimpleVT(); + MVT EltVT = VT.getVectorElementType(); EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); EVT ExtraEltVT = ExtraVT.getVectorElementType(); @@ -13205,11 +13561,11 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget *Subtarget, static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) { - EVT T = Op.getValueType(); + MVT T = Op.getSimpleValueType(); SDLoc DL(Op); unsigned Reg = 0; unsigned size = 0; - switch(T.getSimpleVT().SimpleTy) { + switch(T.SimpleTy) { default: llvm_unreachable("Invalid value type!"); case MVT::i8: Reg = X86::AL; size = 1; break; case MVT::i16: Reg = X86::AX; size = 2; break; @@ -13317,7 +13673,7 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) { } static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) { - EVT VT = Op.getNode()->getValueType(0); + EVT VT = Op.getNode()->getSimpleValueType(0); // Let legalize expand this if it isn't a legal type yet. if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) @@ -13497,6 +13853,7 @@ static void ReplaceATOMIC_LOAD(SDNode *Node, Node->getOperand(1), Zero, Zero, cast<AtomicSDNode>(Node)->getMemOperand(), cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getOrdering(), cast<AtomicSDNode>(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -13735,8 +14092,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::CMPMU: return "X86ISD::CMPMU"; case X86ISD::SETCC: return "X86ISD::SETCC"; case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY"; - case X86ISD::FSETCCsd: return "X86ISD::FSETCCsd"; - case X86ISD::FSETCCss: return "X86ISD::FSETCCss"; + case X86ISD::FSETCC: return "X86ISD::FSETCC"; case X86ISD::CMOV: return "X86ISD::CMOV"; case X86ISD::BRCOND: return "X86ISD::BRCOND"; case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; @@ -13788,7 +14144,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG"; case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG"; case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; - case X86ISD::VSEXT_MOVL: return "X86ISD::VSEXT_MOVL"; case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; case X86ISD::VZEXT: return "X86ISD::VZEXT"; case X86ISD::VSEXT: return "X86ISD::VSEXT"; @@ -13821,17 +14176,14 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::OR: return "X86ISD::OR"; case X86ISD::XOR: return "X86ISD::XOR"; case X86ISD::AND: return "X86ISD::AND"; - case X86ISD::BLSI: return "X86ISD::BLSI"; - case X86ISD::BLSMSK: return "X86ISD::BLSMSK"; - case X86ISD::BLSR: return "X86ISD::BLSR"; case X86ISD::BZHI: return "X86ISD::BZHI"; case X86ISD::BEXTR: return "X86ISD::BEXTR"; case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM"; case X86ISD::PTEST: return "X86ISD::PTEST"; case X86ISD::TESTP: return "X86ISD::TESTP"; case X86ISD::TESTM: return "X86ISD::TESTM"; + case X86ISD::TESTNM: return "X86ISD::TESTNM"; case X86ISD::KORTEST: return "X86ISD::KORTEST"; - case X86ISD::KTEST: return "X86ISD::KTEST"; case X86ISD::PALIGNR: return "X86ISD::PALIGNR"; case X86ISD::PSHUFD: return "X86ISD::PSHUFD"; case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW"; @@ -13855,6 +14207,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128"; case X86ISD::VPERMV: return "X86ISD::VPERMV"; case X86ISD::VPERMV3: return "X86ISD::VPERMV3"; + case X86ISD::VPERMIV3: return "X86ISD::VPERMIV3"; case X86ISD::VPERMI: return "X86ISD::VPERMI"; case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ"; case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS"; @@ -13932,6 +14285,24 @@ bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, return true; } +bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const { + unsigned Bits = Ty->getScalarSizeInBits(); + + // 8-bit shifts are always expensive, but versions with a scalar amount aren't + // particularly cheaper than those without. + if (Bits == 8) + return false; + + // On AVX2 there are new vpsllv[dq] instructions (and other shifts), that make + // variable shifts just as cheap as scalar ones. + if (Subtarget->hasInt256() && (Bits == 32 || Bits == 64)) + return false; + + // Otherwise, it's significantly cheaper to shift by a scalar amount than by a + // fully general vector. + return true; +} + bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) return false; @@ -14115,7 +14486,7 @@ static MachineBasicBlock *EmitXBegin(MachineInstr *MI, MachineBasicBlock *MBB, // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, - llvm::next(MachineBasicBlock::iterator(MI)), MBB->end()); + std::next(MachineBasicBlock::iterator(MI)), MBB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // thisMBB: @@ -14349,7 +14720,7 @@ X86TargetLowering::EmitAtomicLoadArith(MachineInstr *MI, // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, - llvm::next(MachineBasicBlock::iterator(MI)), MBB->end()); + std::next(MachineBasicBlock::iterator(MI)), MBB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // thisMBB: @@ -14635,7 +15006,7 @@ X86TargetLowering::EmitAtomicLoadArith6432(MachineInstr *MI, // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, - llvm::next(MachineBasicBlock::iterator(MI)), MBB->end()); + std::next(MachineBasicBlock::iterator(MI)), MBB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // thisMBB: @@ -15031,8 +15402,7 @@ X86TargetLowering::EmitVAARG64WithCustomInserter( // Transfer the remainder of MBB and its successor edges to endMBB. endMBB->splice(endMBB->begin(), thisMBB, - llvm::next(MachineBasicBlock::iterator(MI)), - thisMBB->end()); + std::next(MachineBasicBlock::iterator(MI)), thisMBB->end()); endMBB->transferSuccessorsAndUpdatePHIs(thisMBB); // Make offsetMBB and overflowMBB successors of thisMBB @@ -15202,8 +15572,7 @@ X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( // Transfer the remainder of MBB and its successor edges to EndMBB. EndMBB->splice(EndMBB->begin(), MBB, - llvm::next(MachineBasicBlock::iterator(MI)), - MBB->end()); + std::next(MachineBasicBlock::iterator(MI)), MBB->end()); EndMBB->transferSuccessorsAndUpdatePHIs(MBB); // The original block will now fall through to the XMM save block. @@ -15226,9 +15595,15 @@ X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( MBB->addSuccessor(EndMBB); } + // Make sure the last operand is EFLAGS, which gets clobbered by the branch + // that was just emitted, but clearly shouldn't be "saved". + assert((MI->getNumOperands() <= 3 || + !MI->getOperand(MI->getNumOperands() - 1).isReg() || + MI->getOperand(MI->getNumOperands() - 1).getReg() == X86::EFLAGS) + && "Expected last argument to be EFLAGS"); unsigned MOVOpc = Subtarget->hasFp256() ? X86::VMOVAPSmr : X86::MOVAPSmr; // In the XMM save block, save all the XMM argument registers. - for (int i = 3, e = MI->getNumOperands(); i != e; ++i) { + for (int i = 3, e = MI->getNumOperands() - 1; i != e; ++i) { int64_t Offset = (i - 3) * 16 + VarArgsFPOffset; MachineMemOperand *MMO = F->getMachineMemOperand( @@ -15259,7 +15634,7 @@ static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock* BB, const TargetRegisterInfo* TRI) { // Scan forward through BB for a use/def of EFLAGS. - MachineBasicBlock::iterator miI(llvm::next(SelectItr)); + MachineBasicBlock::iterator miI(std::next(SelectItr)); for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) { const MachineInstr& mi = *miI; if (mi.readsRegister(X86::EFLAGS)) @@ -15324,8 +15699,7 @@ X86TargetLowering::EmitLoweredSelect(MachineInstr *MI, // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, - llvm::next(MachineBasicBlock::iterator(MI)), - BB->end()); + std::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); // Add the true and fallthrough blocks as its successors. @@ -15405,8 +15779,8 @@ X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI, MachineBasicBlock *BB, MF->insert(MBBIter, mallocMBB); MF->insert(MBBIter, continueMBB); - continueMBB->splice(continueMBB->begin(), BB, llvm::next - (MachineBasicBlock::iterator(MI)), BB->end()); + continueMBB->splice(continueMBB->begin(), BB, + std::next(MachineBasicBlock::iterator(MI)), BB->end()); continueMBB->transferSuccessorsAndUpdatePHIs(BB); // Add code to the main basic block to check if the stack limit has been hit, @@ -15481,7 +15855,7 @@ X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI, const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); DebugLoc DL = MI->getDebugLoc(); - assert(!Subtarget->isTargetEnvMacho()); + assert(!Subtarget->isTargetMacho()); // The lowering is pretty easy: we're just emitting the call to _alloca. The // non-trivial part is impdef of ESP. @@ -15512,7 +15886,7 @@ X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI, } } else { const char *StackProbeSymbol = - Subtarget->isTargetWindows() ? "_chkstk" : "_alloca"; + Subtarget->isTargetKnownWindowsMSVC() ? "_chkstk" : "_alloca"; BuildMI(*BB, MI, DL, TII->get(X86::CALLpcrel32)) .addExternalSymbol(StackProbeSymbol) @@ -15647,7 +16021,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr *MI, // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, - llvm::next(MachineBasicBlock::iterator(MI)), MBB->end()); + std::next(MachineBasicBlock::iterator(MI)), MBB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // thisMBB: @@ -15788,6 +16162,89 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr *MI, return MBB; } +// Replace 213-type (isel default) FMA3 instructions with 231-type for +// accumulator loops. Writing back to the accumulator allows the coalescer +// to remove extra copies in the loop. +MachineBasicBlock * +X86TargetLowering::emitFMA3Instr(MachineInstr *MI, + MachineBasicBlock *MBB) const { + MachineOperand &AddendOp = MI->getOperand(3); + + // Bail out early if the addend isn't a register - we can't switch these. + if (!AddendOp.isReg()) + return MBB; + + MachineFunction &MF = *MBB->getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + + // Check whether the addend is defined by a PHI: + assert(MRI.hasOneDef(AddendOp.getReg()) && "Multiple defs in SSA?"); + MachineInstr &AddendDef = *MRI.def_instr_begin(AddendOp.getReg()); + if (!AddendDef.isPHI()) + return MBB; + + // Look for the following pattern: + // loop: + // %addend = phi [%entry, 0], [%loop, %result] + // ... + // %result<tied1> = FMA213 %m2<tied0>, %m1, %addend + + // Replace with: + // loop: + // %addend = phi [%entry, 0], [%loop, %result] + // ... + // %result<tied1> = FMA231 %addend<tied0>, %m1, %m2 + + for (unsigned i = 1, e = AddendDef.getNumOperands(); i < e; i += 2) { + assert(AddendDef.getOperand(i).isReg()); + MachineOperand PHISrcOp = AddendDef.getOperand(i); + MachineInstr &PHISrcInst = *MRI.def_instr_begin(PHISrcOp.getReg()); + if (&PHISrcInst == MI) { + // Found a matching instruction. + unsigned NewFMAOpc = 0; + switch (MI->getOpcode()) { + case X86::VFMADDPDr213r: NewFMAOpc = X86::VFMADDPDr231r; break; + case X86::VFMADDPSr213r: NewFMAOpc = X86::VFMADDPSr231r; break; + case X86::VFMADDSDr213r: NewFMAOpc = X86::VFMADDSDr231r; break; + case X86::VFMADDSSr213r: NewFMAOpc = X86::VFMADDSSr231r; break; + case X86::VFMSUBPDr213r: NewFMAOpc = X86::VFMSUBPDr231r; break; + case X86::VFMSUBPSr213r: NewFMAOpc = X86::VFMSUBPSr231r; break; + case X86::VFMSUBSDr213r: NewFMAOpc = X86::VFMSUBSDr231r; break; + case X86::VFMSUBSSr213r: NewFMAOpc = X86::VFMSUBSSr231r; break; + case X86::VFNMADDPDr213r: NewFMAOpc = X86::VFNMADDPDr231r; break; + case X86::VFNMADDPSr213r: NewFMAOpc = X86::VFNMADDPSr231r; break; + case X86::VFNMADDSDr213r: NewFMAOpc = X86::VFNMADDSDr231r; break; + case X86::VFNMADDSSr213r: NewFMAOpc = X86::VFNMADDSSr231r; break; + case X86::VFNMSUBPDr213r: NewFMAOpc = X86::VFNMSUBPDr231r; break; + case X86::VFNMSUBPSr213r: NewFMAOpc = X86::VFNMSUBPSr231r; break; + case X86::VFNMSUBSDr213r: NewFMAOpc = X86::VFNMSUBSDr231r; break; + case X86::VFNMSUBSSr213r: NewFMAOpc = X86::VFNMSUBSSr231r; break; + case X86::VFMADDPDr213rY: NewFMAOpc = X86::VFMADDPDr231rY; break; + case X86::VFMADDPSr213rY: NewFMAOpc = X86::VFMADDPSr231rY; break; + case X86::VFMSUBPDr213rY: NewFMAOpc = X86::VFMSUBPDr231rY; break; + case X86::VFMSUBPSr213rY: NewFMAOpc = X86::VFMSUBPSr231rY; break; + case X86::VFNMADDPDr213rY: NewFMAOpc = X86::VFNMADDPDr231rY; break; + case X86::VFNMADDPSr213rY: NewFMAOpc = X86::VFNMADDPSr231rY; break; + case X86::VFNMSUBPDr213rY: NewFMAOpc = X86::VFNMSUBPDr231rY; break; + case X86::VFNMSUBPSr213rY: NewFMAOpc = X86::VFNMSUBPSr231rY; break; + default: llvm_unreachable("Unrecognized FMA variant."); + } + + const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo(); + MachineInstrBuilder MIB = + BuildMI(MF, MI->getDebugLoc(), TII.get(NewFMAOpc)) + .addOperand(MI->getOperand(0)) + .addOperand(MI->getOperand(3)) + .addOperand(MI->getOperand(2)) + .addOperand(MI->getOperand(1)); + MBB->insert(MachineBasicBlock::iterator(MI), MIB); + MI->eraseFromParent(); + } + } + + return MBB; +} + MachineBasicBlock * X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { @@ -16015,6 +16472,36 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, case X86::EH_SjLj_LongJmp32: case X86::EH_SjLj_LongJmp64: return emitEHSjLjLongJmp(MI, BB); + + case TargetOpcode::STACKMAP: + case TargetOpcode::PATCHPOINT: + return emitPatchPoint(MI, BB); + + case X86::VFMADDPDr213r: + case X86::VFMADDPSr213r: + case X86::VFMADDSDr213r: + case X86::VFMADDSSr213r: + case X86::VFMSUBPDr213r: + case X86::VFMSUBPSr213r: + case X86::VFMSUBSDr213r: + case X86::VFMSUBSSr213r: + case X86::VFNMADDPDr213r: + case X86::VFNMADDPSr213r: + case X86::VFNMADDSDr213r: + case X86::VFNMADDSSr213r: + case X86::VFNMSUBPDr213r: + case X86::VFNMSUBPSr213r: + case X86::VFNMSUBSDr213r: + case X86::VFNMSUBSSr213r: + case X86::VFMADDPDr213rY: + case X86::VFMADDPSr213rY: + case X86::VFMSUBPDr213rY: + case X86::VFMSUBPSr213rY: + case X86::VFNMADDPDr213rY: + case X86::VFNMADDPSr213rY: + case X86::VFNMSUBPDr213rY: + case X86::VFNMSUBPSr213rY: + return emitFMA3Instr(MI, BB); } } @@ -16377,44 +16864,6 @@ static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG, EltNo); } -/// Extract one bit from mask vector, like v16i1 or v8i1. -/// AVX-512 feature. -static SDValue ExtractBitFromMaskVector(SDNode *N, SelectionDAG &DAG) { - SDValue Vec = N->getOperand(0); - SDLoc dl(Vec); - MVT VecVT = Vec.getSimpleValueType(); - SDValue Idx = N->getOperand(1); - MVT EltVT = N->getSimpleValueType(0); - - assert((VecVT.getVectorElementType() == MVT::i1 && EltVT == MVT::i8) || - "Unexpected operands in ExtractBitFromMaskVector"); - - // variable index - if (!isa<ConstantSDNode>(Idx)) { - MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); - SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Vec); - SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - ExtVT.getVectorElementType(), Ext); - return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt); - } - - unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); - - MVT ScalarVT = MVT::getIntegerVT(VecVT.getSizeInBits()); - unsigned MaxShift = VecVT.getSizeInBits() - 1; - Vec = DAG.getNode(ISD::BITCAST, dl, ScalarVT, Vec); - Vec = DAG.getNode(ISD::SHL, dl, ScalarVT, Vec, - DAG.getConstant(MaxShift - IdxVal, ScalarVT)); - Vec = DAG.getNode(ISD::SRL, dl, ScalarVT, Vec, - DAG.getConstant(MaxShift, ScalarVT)); - - if (VecVT == MVT::v16i1) { - Vec = DAG.getNode(ISD::BITCAST, dl, MVT::i16, Vec); - return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Vec); - } - return DAG.getNode(ISD::BITCAST, dl, MVT::i8, Vec); -} - /// PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index /// generation and convert it from being a bunch of shuffles and extracts /// to a simple store and scalar loads to extract the elements. @@ -16426,10 +16875,6 @@ static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, SDValue InputVector = N->getOperand(0); - if (InputVector.getValueType().getVectorElementType() == MVT::i1 && - !DCI.isBeforeLegalize()) - return ExtractBitFromMaskVector(N, DAG); - // Detect whether we are trying to convert from mmx to i32 and the bitcast // from mmx to v2i32 has a single usage. if (InputVector.getNode()->getOpcode() == llvm::ISD::BITCAST && @@ -16975,12 +17420,13 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, // Simplify vector selection if the selector will be produced by CMPP*/PCMP*. if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC && // Check if SETCC has already been promoted - TLI.getSetCCResultType(*DAG.getContext(), VT) == Cond.getValueType()) { + TLI.getSetCCResultType(*DAG.getContext(), VT) == CondVT && + // Check that condition value type matches vselect operand type + CondVT == VT) { assert(Cond.getValueType().isVector() && "vector select expects a vector selector!"); - EVT IntVT = Cond.getValueType(); bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode()); bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); @@ -16995,7 +17441,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, ISD::CondCode NewCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType().isInteger()); - Cond = DAG.getSetCC(DL, IntVT, Cond.getOperand(0), Cond.getOperand(1), NewCC); + Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1), NewCC); std::swap(LHS, RHS); TValIsAllOnes = FValIsAllOnes; FValIsAllZeros = TValIsAllZeros; @@ -17008,16 +17454,91 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, if (TValIsAllOnes && FValIsAllZeros) Ret = Cond; else if (TValIsAllOnes) - Ret = DAG.getNode(ISD::OR, DL, IntVT, Cond, - DAG.getNode(ISD::BITCAST, DL, IntVT, RHS)); + Ret = DAG.getNode(ISD::OR, DL, CondVT, Cond, + DAG.getNode(ISD::BITCAST, DL, CondVT, RHS)); else if (FValIsAllZeros) - Ret = DAG.getNode(ISD::AND, DL, IntVT, Cond, - DAG.getNode(ISD::BITCAST, DL, IntVT, LHS)); + Ret = DAG.getNode(ISD::AND, DL, CondVT, Cond, + DAG.getNode(ISD::BITCAST, DL, CondVT, LHS)); return DAG.getNode(ISD::BITCAST, DL, VT, Ret); } } + // Try to fold this VSELECT into a MOVSS/MOVSD + if (N->getOpcode() == ISD::VSELECT && + Cond.getOpcode() == ISD::BUILD_VECTOR && !DCI.isBeforeLegalize()) { + if (VT == MVT::v4i32 || VT == MVT::v4f32 || + (Subtarget->hasSSE2() && (VT == MVT::v2i64 || VT == MVT::v2f64))) { + bool CanFold = false; + unsigned NumElems = Cond.getNumOperands(); + SDValue A = LHS; + SDValue B = RHS; + + if (isZero(Cond.getOperand(0))) { + CanFold = true; + + // fold (vselect <0,-1,-1,-1>, A, B) -> (movss A, B) + // fold (vselect <0,-1> -> (movsd A, B) + for (unsigned i = 1, e = NumElems; i != e && CanFold; ++i) + CanFold = isAllOnes(Cond.getOperand(i)); + } else if (isAllOnes(Cond.getOperand(0))) { + CanFold = true; + std::swap(A, B); + + // fold (vselect <-1,0,0,0>, A, B) -> (movss B, A) + // fold (vselect <-1,0> -> (movsd B, A) + for (unsigned i = 1, e = NumElems; i != e && CanFold; ++i) + CanFold = isZero(Cond.getOperand(i)); + } + + if (CanFold) { + if (VT == MVT::v4i32 || VT == MVT::v4f32) + return getTargetShuffleNode(X86ISD::MOVSS, DL, VT, A, B, DAG); + return getTargetShuffleNode(X86ISD::MOVSD, DL, VT, A, B, DAG); + } + + if (Subtarget->hasSSE2() && (VT == MVT::v4i32 || VT == MVT::v4f32)) { + // fold (v4i32: vselect <0,0,-1,-1>, A, B) -> + // (v4i32 (bitcast (movsd (v2i64 (bitcast A)), + // (v2i64 (bitcast B))))) + // + // fold (v4f32: vselect <0,0,-1,-1>, A, B) -> + // (v4f32 (bitcast (movsd (v2f64 (bitcast A)), + // (v2f64 (bitcast B))))) + // + // fold (v4i32: vselect <-1,-1,0,0>, A, B) -> + // (v4i32 (bitcast (movsd (v2i64 (bitcast B)), + // (v2i64 (bitcast A))))) + // + // fold (v4f32: vselect <-1,-1,0,0>, A, B) -> + // (v4f32 (bitcast (movsd (v2f64 (bitcast B)), + // (v2f64 (bitcast A))))) + + CanFold = (isZero(Cond.getOperand(0)) && + isZero(Cond.getOperand(1)) && + isAllOnes(Cond.getOperand(2)) && + isAllOnes(Cond.getOperand(3))); + + if (!CanFold && isAllOnes(Cond.getOperand(0)) && + isAllOnes(Cond.getOperand(1)) && + isZero(Cond.getOperand(2)) && + isZero(Cond.getOperand(3))) { + CanFold = true; + std::swap(LHS, RHS); + } + + if (CanFold) { + EVT NVT = (VT == MVT::v4i32) ? MVT::v2i64 : MVT::v2f64; + SDValue NewA = DAG.getNode(ISD::BITCAST, DL, NVT, LHS); + SDValue NewB = DAG.getNode(ISD::BITCAST, DL, NVT, RHS); + SDValue Select = getTargetShuffleNode(X86ISD::MOVSD, DL, NVT, NewA, + NewB, DAG); + return DAG.getNode(ISD::BITCAST, DL, VT, Select); + } + } + } + } + // If we know that this node is legal then we know that it is going to be // matched by one of the SSE/AVX BLEND instructions. These instructions only // depend on the highest bit in each word. Try to use SimplifyDemandedBits @@ -17466,7 +17987,7 @@ static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) { } /// \brief Returns a vector of 0s if the node in input is a vector logical -/// shift by a constant amount which is known to be bigger than or equal +/// shift by a constant amount which is known to be bigger than or equal /// to the vector element size in bits. static SDValue performShiftToAllZeros(SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) { @@ -17486,7 +18007,7 @@ static SDValue performShiftToAllZeros(SDNode *N, SelectionDAG &DAG, unsigned MaxAmount = VT.getVectorElementType().getSizeInBits(); // SSE2/AVX2 logical shifts always return a vector of 0s - // if the shift amount is bigger than or equal to + // if the shift amount is bigger than or equal to // the element size. The constant shift amount will be // encoded as a 8-bit immediate. if (ShiftAmt.trunc(8).uge(MaxAmount)) @@ -17571,18 +18092,42 @@ static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG, if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) || (cc0 == X86::COND_NE && cc1 == X86::COND_P)) { - bool is64BitFP = (CMP00.getValueType() == MVT::f64); - X86ISD::NodeType NTOperator = is64BitFP ? - X86ISD::FSETCCsd : X86ISD::FSETCCss; // FIXME: need symbolic constants for these magic numbers. // See X86ATTInstPrinter.cpp:printSSECC(). unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4; - SDValue OnesOrZeroesF = DAG.getNode(NTOperator, DL, MVT::f32, CMP00, CMP01, + if (Subtarget->hasAVX512()) { + SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00, + CMP01, DAG.getConstant(x86cc, MVT::i8)); + if (N->getValueType(0) != MVT::i1) + return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), + FSetCC); + return FSetCC; + } + SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL, + CMP00.getValueType(), CMP00, CMP01, DAG.getConstant(x86cc, MVT::i8)); - SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, MVT::i32, - OnesOrZeroesF); - SDValue ANDed = DAG.getNode(ISD::AND, DL, MVT::i32, OnesOrZeroesI, - DAG.getConstant(1, MVT::i32)); + + bool is64BitFP = (CMP00.getValueType() == MVT::f64); + MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32; + + if (is64BitFP && !Subtarget->is64Bit()) { + // On a 32-bit target, we cannot bitcast the 64-bit float to a + // 64-bit integer, since that's not a legal type. Since + // OnesOrZeroesF is all ones of all zeroes, we don't need all the + // bits, but can do this little dance to extract the lowest 32 bits + // and work with those going forward. + SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, + OnesOrZeroesF); + SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, + Vector64); + OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, + Vector32, DAG.getIntPtrConstant(0)); + IntVT = MVT::i32; + } + + SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, IntVT, OnesOrZeroesF); + SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI, + DAG.getConstant(1, IntVT)); SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); return OneBitOfTruth; } @@ -17715,9 +18260,7 @@ static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG, if (R.getNode()) return R; - // Create BLSI, BLSR, and BZHI instructions - // BLSI is X & (-X) - // BLSR is X & (X-1) + // Create BEXTR and BZHI instructions // BZHI is X & ((1 << Y) - 1) // BEXTR is ((X >> imm) & (2**size-1)) if (VT == MVT::i32 || VT == MVT::i64) { @@ -17725,28 +18268,6 @@ static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG, SDValue N1 = N->getOperand(1); SDLoc DL(N); - if (Subtarget->hasBMI()) { - // Check LHS for neg - if (N0.getOpcode() == ISD::SUB && N0.getOperand(1) == N1 && - isZero(N0.getOperand(0))) - return DAG.getNode(X86ISD::BLSI, DL, VT, N1); - - // Check RHS for neg - if (N1.getOpcode() == ISD::SUB && N1.getOperand(1) == N0 && - isZero(N1.getOperand(0))) - return DAG.getNode(X86ISD::BLSI, DL, VT, N0); - - // Check LHS for X-1 - if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1 && - isAllOnes(N0.getOperand(1))) - return DAG.getNode(X86ISD::BLSR, DL, VT, N1); - - // Check RHS for X-1 - if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N0 && - isAllOnes(N1.getOperand(1))) - return DAG.getNode(X86ISD::BLSR, DL, VT, N0); - } - if (Subtarget->hasBMI2()) { // Check for (and (add (shl 1, Y), -1), X) if (N0.getOpcode() == ISD::ADD && isAllOnes(N0.getOperand(1))) { @@ -17822,7 +18343,6 @@ static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG, static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) { - EVT VT = N->getValueType(0); if (DCI.isBeforeLegalizeOps()) return SDValue(); @@ -17832,6 +18352,7 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG, SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); + EVT VT = N->getValueType(0); // look for psign/blend if (VT == MVT::v2i64 || VT == MVT::v4i64) { @@ -17917,6 +18438,18 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG, return SDValue(); // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c) + MachineFunction &MF = DAG.getMachineFunction(); + bool OptForSize = MF.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); + + // SHLD/SHRD instructions have lower register pressure, but on some + // platforms they have higher latency than the equivalent + // series of shifts/or that would otherwise be generated. + // Don't fold (or (x << c) | (y >> (64 - c))) if SHLD/SHRD instructions + // have higher latencies and we are not optimizing for size. + if (!OptForSize && Subtarget->isSHLDSlow()) + return SDValue(); + if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) std::swap(N0, N1); if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) @@ -18010,7 +18543,6 @@ static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) { static SDValue PerformXorCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) { - EVT VT = N->getValueType(0); if (DCI.isBeforeLegalizeOps()) return SDValue(); @@ -18020,28 +18552,6 @@ static SDValue PerformXorCombine(SDNode *N, SelectionDAG &DAG, return RV; } - // Try forming BMI if it is available. - if (!Subtarget->hasBMI()) - return SDValue(); - - if (VT != MVT::i32 && VT != MVT::i64) - return SDValue(); - - assert(Subtarget->hasBMI() && "Creating BLSMSK requires BMI instructions"); - - // Create BLSMSK instructions by finding X ^ (X-1) - SDValue N0 = N->getOperand(0); - SDValue N1 = N->getOperand(1); - SDLoc DL(N); - - if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1 && - isAllOnes(N0.getOperand(1))) - return DAG.getNode(X86ISD::BLSMSK, DL, VT, N1); - - if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N0 && - isAllOnes(N1.getOperand(1))) - return DAG.getNode(X86ISD::BLSMSK, DL, VT, N0); - return SDValue(); } @@ -18845,6 +19355,17 @@ static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG, } } + if (N0.getOpcode() == ISD::TRUNCATE && + N0.hasOneUse() && + N0.getOperand(0).hasOneUse()) { + SDValue N00 = N0.getOperand(0); + if (N00.getOpcode() == X86ISD::SETCC_CARRY) { + return DAG.getNode(ISD::AND, dl, VT, + DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, + N00.getOperand(0), N00.getOperand(1)), + DAG.getConstant(1, VT)); + } + } if (VT.is256BitVector()) { SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget); if (R.getNode()) @@ -18856,10 +19377,13 @@ static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG, // Optimize x == -y --> x+y == 0 // x != -y --> x+y != 0 -static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG) { +static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG, + const X86Subtarget* Subtarget) { ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); + EVT VT = N->getValueType(0); + SDLoc DL(N); if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB) if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS.getOperand(0))) @@ -18877,17 +19401,51 @@ static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG) { return DAG.getSetCC(SDLoc(N), N->getValueType(0), addV, DAG.getConstant(0, addV.getValueType()), CC); } + + if (VT.getScalarType() == MVT::i1) { + bool IsSEXT0 = (LHS.getOpcode() == ISD::SIGN_EXTEND) && + (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1); + bool IsVZero0 = ISD::isBuildVectorAllZeros(LHS.getNode()); + if (!IsSEXT0 && !IsVZero0) + return SDValue(); + bool IsSEXT1 = (RHS.getOpcode() == ISD::SIGN_EXTEND) && + (RHS.getOperand(0).getValueType().getScalarType() == MVT::i1); + bool IsVZero1 = ISD::isBuildVectorAllZeros(RHS.getNode()); + + if (!IsSEXT1 && !IsVZero1) + return SDValue(); + + if (IsSEXT0 && IsVZero1) { + assert(VT == LHS.getOperand(0).getValueType() && "Uexpected operand type"); + if (CC == ISD::SETEQ) + return DAG.getNOT(DL, LHS.getOperand(0), VT); + return LHS.getOperand(0); + } + if (IsSEXT1 && IsVZero0) { + assert(VT == RHS.getOperand(0).getValueType() && "Uexpected operand type"); + if (CC == ISD::SETEQ) + return DAG.getNOT(DL, RHS.getOperand(0), VT); + return RHS.getOperand(0); + } + } + return SDValue(); } // Helper function of PerformSETCCCombine. It is to materialize "setb reg" // as "sbb reg,reg", since it can be extended without zext and produces // an all-ones bit which is more useful than 0/1 in some cases. -static SDValue MaterializeSETB(SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG) { - return DAG.getNode(ISD::AND, DL, MVT::i8, +static SDValue MaterializeSETB(SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG, + MVT VT) { + if (VT == MVT::i8) + return DAG.getNode(ISD::AND, DL, VT, + DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, + DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS), + DAG.getConstant(1, VT)); + assert (VT == MVT::i1 && "Unexpected type for SECCC node"); + return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, - DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS), - DAG.getConstant(1, MVT::i8)); + DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS)); } // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT @@ -18912,7 +19470,7 @@ static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG, EFLAGS.getNode()->getVTList(), EFLAGS.getOperand(1), EFLAGS.getOperand(0)); SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo()); - return MaterializeSETB(DL, NewEFLAGS, DAG); + return MaterializeSETB(DL, NewEFLAGS, DAG, N->getSimpleValueType(0)); } } @@ -18920,7 +19478,7 @@ static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG, // a zext and produces an all-ones bit which is more useful than 0/1 in some // cases. if (CC == X86::COND_B) - return MaterializeSETB(DL, EFLAGS, DAG); + return MaterializeSETB(DL, EFLAGS, DAG, N->getSimpleValueType(0)); SDValue Flags; @@ -19155,7 +19713,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, case ISD::SIGN_EXTEND: return PerformSExtCombine(N, DAG, DCI, Subtarget); case ISD::SIGN_EXTEND_INREG: return PerformSIGN_EXTEND_INREGCombine(N, DAG, Subtarget); case ISD::TRUNCATE: return PerformTruncateCombine(N, DAG,DCI,Subtarget); - case ISD::SETCC: return PerformISDSETCCCombine(N, DAG); + case ISD::SETCC: return PerformISDSETCCCombine(N, DAG, Subtarget); case X86ISD::SETCC: return PerformSETCCCombine(N, DAG, DCI, Subtarget); case X86ISD::BRCOND: return PerformBrCondCombine(N, DAG, DCI, Subtarget); case X86ISD::VZEXT: return performVZEXTCombine(N, DAG, DCI, Subtarget); @@ -19808,8 +20366,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, if (Res.second == 0) { // Map st(0) -> st(7) -> ST0 if (Constraint.size() == 7 && Constraint[0] == '{' && - std::tolower(Constraint[1]) == 's' && - std::tolower(Constraint[2]) == 't' && + tolower(Constraint[1]) == 's' && + tolower(Constraint[2]) == 't' && Constraint[3] == '(' && (Constraint[4] >= '0' && Constraint[4] <= '7') && Constraint[5] == ')' && diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h index bc3dd60..0f0d17b 100644 --- a/lib/Target/X86/X86ISelLowering.h +++ b/lib/Target/X86/X86ISelLowering.h @@ -15,16 +15,15 @@ #ifndef X86ISELLOWERING_H #define X86ISELLOWERING_H -#include "X86MachineFunctionInfo.h" -#include "X86RegisterInfo.h" #include "X86Subtarget.h" #include "llvm/CodeGen/CallingConvLower.h" -#include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetOptions.h" namespace llvm { + class X86TargetMachine; + namespace X86ISD { // X86 Specific DAG Nodes enum NodeType { @@ -94,6 +93,9 @@ namespace llvm { /// operand, usually produced by a CMP instruction. SETCC, + /// X86 Select + SELECT, + // Same as SETCC except it's materialized with a sbb and the value is all // one's or all zero's. SETCC_CARRY, // R = carry_bit ? ~0 : 0 @@ -101,7 +103,7 @@ namespace llvm { /// X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD. /// Operands are two FP values to compare; result is a mask of /// 0s or 1s. Generally DTRT for C/C++ with NaNs. - FSETCCss, FSETCCsd, + FSETCC, /// X86 MOVMSK{pd|ps}, extracts sign bits of two or four FP values, /// result in an integer GPR. Needs masking for scalar result. @@ -242,12 +244,9 @@ namespace llvm { /// the list of operands. TC_RETURN, - // VZEXT_MOVL - Vector move low and zero extend. + // VZEXT_MOVL - Vector move to low scalar and zero higher vector elements. VZEXT_MOVL, - // VSEXT_MOVL - Vector move low and sign extend. - VSEXT_MOVL, - // VZEXT - Vector integer zero-extend. VZEXT, @@ -292,9 +291,6 @@ namespace llvm { ADD, SUB, ADC, SBB, SMUL, INC, DEC, OR, XOR, AND, - BLSI, // BLSI - Extract lowest set isolated bit - BLSMSK, // BLSMSK - Get mask up to lowest set bit - BLSR, // BLSR - Reset lowest set bit BZHI, // BZHI - Zero high bits BEXTR, // BEXTR - Bit field extract @@ -309,12 +305,12 @@ namespace llvm { // TESTP - Vector packed fp sign bitwise comparisons. TESTP, - // TESTM - Vector "test" in AVX-512, the result is in a mask vector. + // TESTM, TESTNM - Vector "test" in AVX-512, the result is in a mask vector. TESTM, + TESTNM, // OR/AND test for masks KORTEST, - KTEST, // Several flavors of instructions with vector shuffle behaviors. PALIGNR, @@ -337,12 +333,15 @@ namespace llvm { VPERMILP, VPERMV, VPERMV3, + VPERMIV3, VPERMI, VPERM2X128, VBROADCAST, // masked broadcast VBROADCASTM, + // Insert/Extract vector element VINSERT, + VEXTRACT, // PMULUDQ - Vector multiply packed unsigned doubleword integers PMULUDQ, @@ -522,32 +521,32 @@ namespace llvm { //===--------------------------------------------------------------------===// // X86TargetLowering - X86 Implementation of the TargetLowering interface - class X86TargetLowering : public TargetLowering { + class X86TargetLowering final : public TargetLowering { public: explicit X86TargetLowering(X86TargetMachine &TM); - virtual unsigned getJumpTableEncoding() const; + unsigned getJumpTableEncoding() const override; - virtual MVT getScalarShiftAmountTy(EVT LHSTy) const { return MVT::i8; } + MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i8; } - virtual const MCExpr * + const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, - MCContext &Ctx) const; + MCContext &Ctx) const override; /// getPICJumpTableRelocaBase - Returns relocation base for the given PIC /// jumptable. - virtual SDValue getPICJumpTableRelocBase(SDValue Table, - SelectionDAG &DAG) const; - virtual const MCExpr * + SDValue getPICJumpTableRelocBase(SDValue Table, + SelectionDAG &DAG) const override; + const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, - unsigned JTI, MCContext &Ctx) const; + unsigned JTI, MCContext &Ctx) const override; /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate /// function arguments in the caller parameter area. For X86, aggregates /// that contains are placed at 16-byte boundaries while the rest are at /// 4-byte boundaries. - virtual unsigned getByValTypeAlignment(Type *Ty) const; + unsigned getByValTypeAlignment(Type *Ty) const override; /// getOptimalMemOpType - Returns the target specific optimal type for load /// and store operations as a result of memset, memcpy, and memmove @@ -560,10 +559,9 @@ namespace llvm { /// source is constant so it does not need to be loaded. /// It returns EVT::Other if the type should be determined using generic /// target-independent logic. - virtual EVT - getOptimalMemOpType(uint64_t Size, unsigned DstAlign, unsigned SrcAlign, - bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, - MachineFunction &MF) const; + EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign, unsigned SrcAlign, + bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, + MachineFunction &MF) const override; /// isSafeMemOpType - Returns true if it's safe to use load / store of the /// specified type to expand memcpy / memset inline. This is mostly true @@ -571,88 +569,91 @@ namespace llvm { /// targets without SSE2 f64 load / store are done with fldl / fstpl which /// also does type conversion. Note the specified type doesn't have to be /// legal as the hook is used before type legalization. - virtual bool isSafeMemOpType(MVT VT) const; + bool isSafeMemOpType(MVT VT) const override; /// allowsUnalignedMemoryAccesses - Returns true if the target allows /// unaligned memory accesses. of the specified type. Returns whether it /// is "fast" by reference in the second argument. - virtual bool allowsUnalignedMemoryAccesses(EVT VT, bool *Fast) const; + bool allowsUnalignedMemoryAccesses(EVT VT, unsigned AS, + bool *Fast) const override; /// LowerOperation - Provide custom lowering hooks for some operations. /// - virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; /// ReplaceNodeResults - Replace the results of node with an illegal result /// type with new values built out of custom code. /// - virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results, - SelectionDAG &DAG) const; + void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results, + SelectionDAG &DAG) const override; - virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; /// isTypeDesirableForOp - Return true if the target has native support for /// the specified value type and it is 'desirable' to use the type for the /// given node type. e.g. On x86 i16 is legal, but undesirable since i16 /// instruction encodings are longer and some i16 instructions are slow. - virtual bool isTypeDesirableForOp(unsigned Opc, EVT VT) const; + bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override; /// isTypeDesirable - Return true if the target has native support for the /// specified value type and it is 'desirable' to use the type. e.g. On x86 /// i16 is legal, but undesirable since i16 instruction encodings are longer /// and some i16 instructions are slow. - virtual bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const; + bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override; - virtual MachineBasicBlock * + MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr *MI, - MachineBasicBlock *MBB) const; + MachineBasicBlock *MBB) const override; /// getTargetNodeName - This method returns the name of a target specific /// DAG node. - virtual const char *getTargetNodeName(unsigned Opcode) const; + const char *getTargetNodeName(unsigned Opcode) const override; /// getSetCCResultType - Return the value type to use for ISD::SETCC. - virtual EVT getSetCCResultType(LLVMContext &Context, EVT VT) const; + EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override; /// computeMaskedBitsForTargetNode - Determine which of the bits specified /// in Mask are known to be either zero or one and return them in the /// KnownZero/KnownOne bitsets. - virtual void computeMaskedBitsForTargetNode(const SDValue Op, - APInt &KnownZero, - APInt &KnownOne, - const SelectionDAG &DAG, - unsigned Depth = 0) const; + void computeMaskedBitsForTargetNode(const SDValue Op, + APInt &KnownZero, + APInt &KnownOne, + const SelectionDAG &DAG, + unsigned Depth = 0) const override; // ComputeNumSignBitsForTargetNode - Determine the number of bits in the // operation that are sign bits. - virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, - unsigned Depth) const; + unsigned ComputeNumSignBitsForTargetNode(SDValue Op, + unsigned Depth) const override; - virtual bool - isGAPlusOffset(SDNode *N, const GlobalValue* &GA, int64_t &Offset) const; + bool isGAPlusOffset(SDNode *N, const GlobalValue* &GA, + int64_t &Offset) const override; SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const; - virtual bool ExpandInlineAsm(CallInst *CI) const; + bool ExpandInlineAsm(CallInst *CI) const override; - ConstraintType getConstraintType(const std::string &Constraint) const; + ConstraintType + getConstraintType(const std::string &Constraint) const override; /// Examine constraint string and operand type and determine a weight value. /// The operand object must already have been set up with the operand type. - virtual ConstraintWeight getSingleConstraintMatchWeight( - AsmOperandInfo &info, const char *constraint) const; + ConstraintWeight + getSingleConstraintMatchWeight(AsmOperandInfo &info, + const char *constraint) const override; - virtual const char *LowerXConstraint(EVT ConstraintVT) const; + const char *LowerXConstraint(EVT ConstraintVT) const override; /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops /// vector. If it is invalid, don't add anything to Ops. If hasMemory is /// true it means one of the asm constraint of the inline asm instruction /// being processed is 'm'. - virtual void LowerAsmOperandForConstraint(SDValue Op, - std::string &Constraint, - std::vector<SDValue> &Ops, - SelectionDAG &DAG) const; + void LowerAsmOperandForConstraint(SDValue Op, + std::string &Constraint, + std::vector<SDValue> &Ops, + SelectionDAG &DAG) const override; /// getRegForInlineAsmConstraint - Given a physical register constraint /// (e.g. {edx}), return the register number and the register class for the @@ -660,31 +661,34 @@ namespace llvm { /// error, this returns a register number of 0. std::pair<unsigned, const TargetRegisterClass*> getRegForInlineAsmConstraint(const std::string &Constraint, - MVT VT) const; + MVT VT) const override; /// isLegalAddressingMode - Return true if the addressing mode represented /// by AM is legal for this target, for a load/store of the specified type. - virtual bool isLegalAddressingMode(const AddrMode &AM, Type *Ty)const; + bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override; /// isLegalICmpImmediate - Return true if the specified immediate is legal /// icmp immediate, that is the target has icmp instructions which can /// compare a register against the immediate without having to materialize /// the immediate into a register. - virtual bool isLegalICmpImmediate(int64_t Imm) const; + bool isLegalICmpImmediate(int64_t Imm) const override; /// isLegalAddImmediate - Return true if the specified immediate is legal /// add immediate, that is the target has add instructions which can /// add a register and the immediate without having to materialize /// the immediate into a register. - virtual bool isLegalAddImmediate(int64_t Imm) const; + bool isLegalAddImmediate(int64_t Imm) const override; + + + bool isVectorShiftByScalarCheap(Type *Ty) const override; /// isTruncateFree - Return true if it's free to truncate a value of /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in /// register EAX to i16 by referencing its sub-register AX. - virtual bool isTruncateFree(Type *Ty1, Type *Ty2) const; - virtual bool isTruncateFree(EVT VT1, EVT VT2) const; + bool isTruncateFree(Type *Ty1, Type *Ty2) const override; + bool isTruncateFree(EVT VT1, EVT VT2) const override; - virtual bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const; + bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override; /// isZExtFree - Return true if any actual instruction that defines a /// value of type Ty1 implicit zero-extends the value to Ty2 in the result @@ -694,44 +698,44 @@ namespace llvm { /// does not necessarily apply to truncate instructions. e.g. on x86-64, /// all instructions that define 32-bit values implicit zero-extend the /// result out to 64 bits. - virtual bool isZExtFree(Type *Ty1, Type *Ty2) const; - virtual bool isZExtFree(EVT VT1, EVT VT2) const; - virtual bool isZExtFree(SDValue Val, EVT VT2) const; + bool isZExtFree(Type *Ty1, Type *Ty2) const override; + bool isZExtFree(EVT VT1, EVT VT2) const override; + bool isZExtFree(SDValue Val, EVT VT2) const override; /// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster /// than a pair of fmul and fadd instructions. fmuladd intrinsics will be /// expanded to FMAs when this method returns true, otherwise fmuladd is /// expanded to fmul + fadd. - virtual bool isFMAFasterThanFMulAndFAdd(EVT VT) const; + bool isFMAFasterThanFMulAndFAdd(EVT VT) const override; /// isNarrowingProfitable - Return true if it's profitable to narrow /// operations of type VT1 to VT2. e.g. on x86, it's profitable to narrow /// from i32 to i8 but not from i32 to i16. - virtual bool isNarrowingProfitable(EVT VT1, EVT VT2) const; + bool isNarrowingProfitable(EVT VT1, EVT VT2) const override; /// isFPImmLegal - Returns true if the target can instruction select the /// specified FP immediate natively. If false, the legalizer will /// materialize the FP immediate as a load from a constant pool. - virtual bool isFPImmLegal(const APFloat &Imm, EVT VT) const; + bool isFPImmLegal(const APFloat &Imm, EVT VT) const override; /// isShuffleMaskLegal - Targets can use this to indicate that they only /// support *some* VECTOR_SHUFFLE operations, those with specific masks. /// By default, if a target supports the VECTOR_SHUFFLE node, all mask /// values are assumed to be legal. - virtual bool isShuffleMaskLegal(const SmallVectorImpl<int> &Mask, - EVT VT) const; + bool isShuffleMaskLegal(const SmallVectorImpl<int> &Mask, + EVT VT) const override; /// isVectorClearMaskLegal - Similar to isShuffleMaskLegal. This is /// used by Targets can use this to indicate if there is a suitable /// VECTOR_SHUFFLE that can be used to replace a VAND with a constant /// pool entry. - virtual bool isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask, - EVT VT) const; + bool isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask, + EVT VT) const override; /// ShouldShrinkFPConstant - If true, then instruction selection should /// seek to shrink the FP constant of the specified type to a smaller type /// in order to save space and / or reduce runtime. - virtual bool ShouldShrinkFPConstant(EVT VT) const { + bool ShouldShrinkFPConstant(EVT VT) const override { // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more // expensive than a straight movsd. On the other hand, it's important to // shrink long double fp constant since fldt is very slow. @@ -752,7 +756,7 @@ namespace llvm { /// isTargetFTOL - Return true if the target uses the MSVC _ftol2 routine /// for fptoui. bool isTargetFTOL() const { - return Subtarget->isTargetWindows() && !Subtarget->is64Bit(); + return Subtarget->isTargetKnownWindowsMSVC() && !Subtarget->is64Bit(); } /// isIntegerTypeFTOL - Return true if the MSVC _ftol2 routine should be @@ -761,28 +765,39 @@ namespace llvm { return isTargetFTOL() && VT == MVT::i64; } + /// \brief Returns true if it is beneficial to convert a load of a constant + /// to just the constant itself. + bool shouldConvertConstantLoadToIntImm(const APInt &Imm, + Type *Ty) const override; + + /// Intel processors have a unified instruction and data cache + const char * getClearCacheBuiltinName() const { + return 0; // nothing to do, move along. + } + /// createFastISel - This method returns a target specific FastISel object, /// or null if the target does not support "fast" ISel. - virtual FastISel *createFastISel(FunctionLoweringInfo &funcInfo, - const TargetLibraryInfo *libInfo) const; + FastISel *createFastISel(FunctionLoweringInfo &funcInfo, + const TargetLibraryInfo *libInfo) const override; /// getStackCookieLocation - Return true if the target stores stack /// protector cookies at a fixed offset in some non-standard address /// space, and populates the address space and offset as /// appropriate. - virtual bool getStackCookieLocation(unsigned &AddressSpace, unsigned &Offset) const; + bool getStackCookieLocation(unsigned &AddressSpace, + unsigned &Offset) const override; SDValue BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, SDValue StackSlot, SelectionDAG &DAG) const; - virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const LLVM_OVERRIDE; + bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override; /// \brief Reset the operation actions based on target options. - virtual void resetOperationActions(); + void resetOperationActions() override; protected: std::pair<const TargetRegisterClass*, uint8_t> - findRepresentativeClass(MVT VT) const; + findRepresentativeClass(MVT VT) const override; private: /// Subtarget - Keep a pointer to the X86Subtarget around so that we can @@ -857,6 +872,7 @@ namespace llvm { SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; + SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; @@ -865,7 +881,6 @@ namespace llvm { SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG) const; @@ -874,9 +889,6 @@ namespace llvm { SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerFABS(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerFNEG(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const; SDValue LowerToBT(SDValue And, ISD::CondCode CC, SDLoc dl, SelectionDAG &DAG) const; SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; @@ -897,37 +909,34 @@ namespace llvm { SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const; - virtual SDValue + SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl, SelectionDAG &DAG, - SmallVectorImpl<SDValue> &InVals) const; - virtual SDValue - LowerCall(CallLoweringInfo &CLI, - SmallVectorImpl<SDValue> &InVals) const; + SmallVectorImpl<SDValue> &InVals) const override; + SDValue LowerCall(CallLoweringInfo &CLI, + SmallVectorImpl<SDValue> &InVals) const override; - virtual SDValue - LowerReturn(SDValue Chain, - CallingConv::ID CallConv, bool isVarArg, - const SmallVectorImpl<ISD::OutputArg> &Outs, - const SmallVectorImpl<SDValue> &OutVals, - SDLoc dl, SelectionDAG &DAG) const; + SDValue LowerReturn(SDValue Chain, + CallingConv::ID CallConv, bool isVarArg, + const SmallVectorImpl<ISD::OutputArg> &Outs, + const SmallVectorImpl<SDValue> &OutVals, + SDLoc dl, SelectionDAG &DAG) const override; - virtual bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const; + bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override; - virtual bool mayBeEmittedAsTailCall(CallInst *CI) const; + bool mayBeEmittedAsTailCall(CallInst *CI) const override; - virtual MVT - getTypeForExtArgOrReturn(MVT VT, ISD::NodeType ExtendKind) const; + MVT getTypeForExtArgOrReturn(MVT VT, + ISD::NodeType ExtendKind) const override; - virtual bool - CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, - bool isVarArg, - const SmallVectorImpl<ISD::OutputArg> &Outs, - LLVMContext &Context) const; + bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, + bool isVarArg, + const SmallVectorImpl<ISD::OutputArg> &Outs, + LLVMContext &Context) const override; - virtual const uint16_t *getScratchRegisters(CallingConv::ID CC) const; + const uint16_t *getScratchRegisters(CallingConv::ID CC) const override; /// Utility function to emit atomic-load-arith operations (and, or, xor, /// nand, max, min, umax, umin). It takes the corresponding instruction to @@ -973,6 +982,9 @@ namespace llvm { MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr *MI, MachineBasicBlock *MBB) const; + MachineBasicBlock *emitFMA3Instr(MachineInstr *MI, + MachineBasicBlock *MBB) const; + /// Emit nodes that will be selected as "test Op0,Op0", or something /// equivalent, for use with the given x86 condition code. SDValue EmitTest(SDValue Op0, unsigned X86CC, SelectionDAG &DAG) const; diff --git a/lib/Target/X86/X86InstrAVX512.td b/lib/Target/X86/X86InstrAVX512.td index cb19fbd..2c5edf6 100644 --- a/lib/Target/X86/X86InstrAVX512.td +++ b/lib/Target/X86/X86InstrAVX512.td @@ -90,16 +90,17 @@ def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "", [(set VR512:$dst, (v16f32 immAllZerosV))]>; } +let Predicates = [HasAVX512] in { def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>; def : Pat<(v16i32 immAllZerosV), (AVX512_512_SET0)>; def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>; -def : Pat<(v16f32 immAllZerosV), (AVX512_512_SET0)>; +} //===----------------------------------------------------------------------===// // AVX-512 - VECTOR INSERT // // -- 32x8 form -- -let neverHasSideEffects = 1, ExeDomain = SSEPackedSingle in { +let hasSideEffects = 0, ExeDomain = SSEPackedSingle in { def VINSERTF32x4rr : AVX512AIi8<0x18, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src1, VR128X:$src2, i8imm:$src3), "vinsertf32x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", @@ -112,7 +113,7 @@ def VINSERTF32x4rm : AVX512AIi8<0x18, MRMSrcMem, (outs VR512:$dst), } // -- 64x4 fp form -- -let neverHasSideEffects = 1, ExeDomain = SSEPackedDouble in { +let hasSideEffects = 0, ExeDomain = SSEPackedDouble in { def VINSERTF64x4rr : AVX512AIi8<0x1a, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src1, VR256X:$src2, i8imm:$src3), "vinsertf64x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", @@ -124,7 +125,7 @@ def VINSERTF64x4rm : AVX512AIi8<0x1a, MRMSrcMem, (outs VR512:$dst), []>, EVEX_4V, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT4>; } // -- 32x4 integer form -- -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { def VINSERTI32x4rr : AVX512AIi8<0x38, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src1, VR128X:$src2, i8imm:$src3), "vinserti32x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", @@ -137,7 +138,7 @@ def VINSERTI32x4rm : AVX512AIi8<0x38, MRMSrcMem, (outs VR512:$dst), } -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { // -- 64x4 form -- def VINSERTI64x4rr : AVX512AIi8<0x3a, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src1, VR256X:$src2, i8imm:$src3), @@ -162,12 +163,12 @@ def : Pat<(vinsert128_insert:$ins (v8i64 VR512:$src1), (v2i64 VR128X:$src2), def : Pat<(vinsert128_insert:$ins (v16i32 VR512:$src1), (v4i32 VR128X:$src2), (iPTR imm)), (VINSERTI32x4rr VR512:$src1, VR128X:$src2, (INSERT_get_vinsert128_imm VR512:$ins))>; - + def : Pat<(vinsert128_insert:$ins (v16f32 VR512:$src1), (loadv4f32 addr:$src2), (iPTR imm)), (VINSERTF32x4rm VR512:$src1, addr:$src2, (INSERT_get_vinsert128_imm VR512:$ins))>; def : Pat<(vinsert128_insert:$ins (v16i32 VR512:$src1), - (bc_v4i32 (loadv2i64 addr:$src2)), + (bc_v4i32 (loadv2i64 addr:$src2)), (iPTR imm)), (VINSERTI32x4rm VR512:$src1, addr:$src2, (INSERT_get_vinsert128_imm VR512:$ins))>; def : Pat<(vinsert128_insert:$ins (v8f64 VR512:$src1), (loadv2f64 addr:$src2), @@ -207,12 +208,12 @@ def : Pat<(vinsert256_insert:$ins (v16i32 VR512:$src1), // vinsertps - insert f32 to XMM def VINSERTPSzrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2, u32u8imm:$src3), - "vinsertps{z}\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", + "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR128X:$dst, (X86insrtps VR128X:$src1, VR128X:$src2, imm:$src3))]>, EVEX_4V; def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst), (ins VR128X:$src1, f32mem:$src2, u32u8imm:$src3), - "vinsertps{z}\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", + "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR128X:$dst, (X86insrtps VR128X:$src1, (v4f32 (scalar_to_vector (loadf32 addr:$src2))), imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>; @@ -220,7 +221,7 @@ def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst), //===----------------------------------------------------------------------===// // AVX-512 VECTOR EXTRACT //--- -let neverHasSideEffects = 1, ExeDomain = SSEPackedSingle in { +let hasSideEffects = 0, ExeDomain = SSEPackedSingle in { // -- 32x4 form -- def VEXTRACTF32x4rr : AVX512AIi8<0x19, MRMDestReg, (outs VR128X:$dst), (ins VR512:$src1, i8imm:$src2), @@ -243,7 +244,7 @@ def VEXTRACTF64x4mr : AVX512AIi8<0x1b, MRMDestMem, (outs), []>, EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT4>; } -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { // -- 32x4 form -- def VEXTRACTI32x4rr : AVX512AIi8<0x39, MRMDestReg, (outs VR128X:$dst), (ins VR512:$src1, i8imm:$src2), @@ -352,15 +353,15 @@ def : Pat<(insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0)), // vextractps - extract 32 bits from XMM def VEXTRACTPSzrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src1, u32u8imm:$src2), - "vextractps{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>, EVEX; def VEXTRACTPSzmr : AVX512AIi8<0x17, MRMDestMem, (outs), (ins f32mem:$dst, VR128X:$src1, u32u8imm:$src2), - "vextractps{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2), - addr:$dst)]>, EVEX; + addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>; //===---------------------------------------------------------------------===// // AVX-512 BROADCAST @@ -369,19 +370,19 @@ multiclass avx512_fp_broadcast<bits<8> opc, string OpcodeStr, RegisterClass DestRC, RegisterClass SrcRC, X86MemOperand x86memop> { def rr : AVX5128I<opc, MRMSrcReg, (outs DestRC:$dst), (ins SrcRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>, EVEX; def rm : AVX5128I<opc, MRMSrcMem, (outs DestRC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),[]>, EVEX; + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),[]>, EVEX; } let ExeDomain = SSEPackedSingle in { - defm VBROADCASTSSZ : avx512_fp_broadcast<0x18, "vbroadcastss{z}", VR512, + defm VBROADCASTSSZ : avx512_fp_broadcast<0x18, "vbroadcastss", VR512, VR128X, f32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; } let ExeDomain = SSEPackedDouble in { - defm VBROADCASTSDZ : avx512_fp_broadcast<0x19, "vbroadcastsd{z}", VR512, + defm VBROADCASTSDZ : avx512_fp_broadcast<0x19, "vbroadcastsd", VR512, VR128X, f64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; } @@ -399,12 +400,12 @@ def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src), multiclass avx512_int_broadcast_reg<bits<8> opc, string OpcodeStr, RegisterClass SrcRC, RegisterClass KRC> { def Zrr : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins SrcRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>, EVEX, EVEX_V512; def Zkrr : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins KRC:$mask, SrcRC:$src), !strconcat(OpcodeStr, - "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"), + " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"), []>, EVEX, EVEX_V512, EVEX_KZ; } @@ -420,6 +421,8 @@ def : Pat <(v8i64 (X86vzext VK8WM:$mask)), def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))), (VPBROADCASTDrZrr GR32:$src)>; +def : Pat<(v16i32 (X86VBroadcastm VK16WM:$mask, (i32 GR32:$src))), + (VPBROADCASTDrZkrr VK16WM:$mask, GR32:$src)>; def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))), (VPBROADCASTQrZrr GR64:$src)>; def : Pat<(v8i64 (X86VBroadcastm VK8WM:$mask, (i64 GR64:$src))), @@ -430,30 +433,37 @@ def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))), def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_i64_512 (i64 GR64:$src))), (VPBROADCASTQrZrr GR64:$src)>; +def : Pat<(v16i32 (int_x86_avx512_mask_pbroadcast_d_gpr_512 (i32 GR32:$src), + (v16i32 immAllZerosV), (i16 GR16:$mask))), + (VPBROADCASTDrZkrr (COPY_TO_REGCLASS GR16:$mask, VK16WM), GR32:$src)>; +def : Pat<(v8i64 (int_x86_avx512_mask_pbroadcast_q_gpr_512 (i64 GR64:$src), + (bc_v8i64 (v16i32 immAllZerosV)), (i8 GR8:$mask))), + (VPBROADCASTQrZkrr (COPY_TO_REGCLASS GR8:$mask, VK8WM), GR64:$src)>; + multiclass avx512_int_broadcast_rm<bits<8> opc, string OpcodeStr, X86MemOperand x86memop, PatFrag ld_frag, RegisterClass DstRC, ValueType OpVT, ValueType SrcVT, RegisterClass KRC> { def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins VR128X:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpVT (X86VBroadcast (SrcVT VR128X:$src))))]>, EVEX; def krr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask, VR128X:$src), !strconcat(OpcodeStr, - "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), [(set DstRC:$dst, (OpVT (X86VBroadcastm KRC:$mask, (SrcVT VR128X:$src))))]>, EVEX, EVEX_KZ; let mayLoad = 1 in { def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpVT (X86VBroadcast (ld_frag addr:$src))))]>, EVEX; def krm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask, x86memop:$src), !strconcat(OpcodeStr, - "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), [(set DstRC:$dst, (OpVT (X86VBroadcastm KRC:$mask, (ld_frag addr:$src))))]>, EVEX, EVEX_KZ; } @@ -503,7 +513,7 @@ multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr, RegisterClass DstRC, RegisterClass KRC, ValueType OpVT, ValueType SrcVT> { def rr : AVX512XS8I<opc, MRMDestReg, (outs DstRC:$dst), (ins KRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>, EVEX; } @@ -522,14 +532,14 @@ multiclass avx512_perm_imm<bits<8> opc, string OpcodeStr, RegisterClass RC, def ri : AVX512AIi8<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, i8imm:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (OpNode RC:$src1, (i8 imm:$src2))))]>, EVEX; def mi : AVX512AIi8<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1, i8imm:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (OpNode (mem_frag addr:$src1), (i8 imm:$src2))))]>, EVEX; @@ -548,14 +558,14 @@ multiclass avx512_perm<bits<8> opc, string OpcodeStr, RegisterClass RC, def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (X86VPermv RC:$src1, RC:$src2)))]>, EVEX_4V; def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (X86VPermv RC:$src1, (mem_frag addr:$src2))))]>, EVEX_4V; @@ -575,97 +585,104 @@ defm VPERMPDZ : avx512_perm<0x16, "vpermpd", VR512, memopv8f64, f512mem, // -- VPERM2I - 3 source operands form -- multiclass avx512_perm_3src<bits<8> opc, string OpcodeStr, RegisterClass RC, PatFrag mem_frag, X86MemOperand x86memop, - ValueType OpVT> { + SDNode OpNode, ValueType OpVT> { let Constraints = "$src1 = $dst" in { def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, RC:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), + " \t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, - (OpVT (X86VPermv3 RC:$src1, RC:$src2, RC:$src3)))]>, + (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>, EVEX_4V; def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src2, x86memop:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), + " \t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, - (OpVT (X86VPermv3 RC:$src1, RC:$src2, + (OpVT (OpNode RC:$src1, RC:$src2, (mem_frag addr:$src3))))]>, EVEX_4V; } } defm VPERMI2D : avx512_perm_3src<0x76, "vpermi2d", VR512, memopv16i32, i512mem, - v16i32>, EVEX_V512, EVEX_CD8<32, CD8VF>; + X86VPermiv3, v16i32>, EVEX_V512, EVEX_CD8<32, CD8VF>; defm VPERMI2Q : avx512_perm_3src<0x76, "vpermi2q", VR512, memopv8i64, i512mem, - v8i64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + X86VPermiv3, v8i64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; defm VPERMI2PS : avx512_perm_3src<0x77, "vpermi2ps", VR512, memopv16f32, i512mem, - v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>; + X86VPermiv3, v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>; defm VPERMI2PD : avx512_perm_3src<0x77, "vpermi2pd", VR512, memopv8f64, i512mem, - v8f64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; - + X86VPermiv3, v8f64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + +defm VPERMT2D : avx512_perm_3src<0x7E, "vpermt2d", VR512, memopv16i32, i512mem, + X86VPermv3, v16i32>, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPERMT2Q : avx512_perm_3src<0x7E, "vpermt2q", VR512, memopv8i64, i512mem, + X86VPermv3, v8i64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; +defm VPERMT2PS : avx512_perm_3src<0x7F, "vpermt2ps", VR512, memopv16f32, i512mem, + X86VPermv3, v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPERMT2PD : avx512_perm_3src<0x7F, "vpermt2pd", VR512, memopv8f64, i512mem, + X86VPermv3, v8f64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; //===----------------------------------------------------------------------===// // AVX-512 - BLEND using mask // -multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, Intrinsic Int, +multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, RegisterClass KRC, RegisterClass RC, X86MemOperand x86memop, PatFrag mem_frag, SDNode OpNode, ValueType vt> { def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), - (ins KRC:$mask, RC:$src1, RC:$src2), - !strconcat(OpcodeStr, - "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), - [(set RC:$dst, (OpNode KRC:$mask, (vt RC:$src2), - (vt RC:$src1)))]>, EVEX_4V, EVEX_K; - def rr_Int : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), - (ins KRC:$mask, RC:$src1, RC:$src2), - !strconcat(OpcodeStr, - "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), - [(set RC:$dst, (Int KRC:$mask, (vt RC:$src2), + (ins KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), + [(set RC:$dst, (OpNode KRC:$mask, (vt RC:$src2), (vt RC:$src1)))]>, EVEX_4V, EVEX_K; - - let mayLoad = 1 in { - def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), - (ins KRC:$mask, RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, - "\t{$src2, $src1, $mask, $dst|$dst, $mask, $src1, $src2}"), - []>, - EVEX_4V, EVEX_K; - - def rm_Int : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), - (ins KRC:$mask, RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, - "\t{$src2, $src1, $mask, $dst|$dst, $mask, $src1, $src2}"), - [(set RC:$dst, (Int KRC:$mask, (vt RC:$src1), - (mem_frag addr:$src2)))]>, - EVEX_4V, EVEX_K; - } + let mayLoad = 1 in + def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), + []>, EVEX_4V, EVEX_K; } let ExeDomain = SSEPackedSingle in defm VBLENDMPSZ : avx512_blendmask<0x65, "vblendmps", - int_x86_avx512_mskblend_ps_512, VK16WM, VR512, f512mem, memopv16f32, vselect, v16f32>, EVEX_CD8<32, CD8VF>, EVEX_V512; let ExeDomain = SSEPackedDouble in defm VBLENDMPDZ : avx512_blendmask<0x65, "vblendmpd", - int_x86_avx512_mskblend_pd_512, VK8WM, VR512, f512mem, memopv8f64, vselect, v8f64>, VEX_W, EVEX_CD8<64, CD8VF>, EVEX_V512; +def : Pat<(v16f32 (int_x86_avx512_mask_blend_ps_512 (v16f32 VR512:$src1), + (v16f32 VR512:$src2), (i16 GR16:$mask))), + (VBLENDMPSZrr (COPY_TO_REGCLASS GR16:$mask, VK16WM), + VR512:$src1, VR512:$src2)>; + +def : Pat<(v8f64 (int_x86_avx512_mask_blend_pd_512 (v8f64 VR512:$src1), + (v8f64 VR512:$src2), (i8 GR8:$mask))), + (VBLENDMPDZrr (COPY_TO_REGCLASS GR8:$mask, VK8WM), + VR512:$src1, VR512:$src2)>; + defm VPBLENDMDZ : avx512_blendmask<0x64, "vpblendmd", - int_x86_avx512_mskblend_d_512, VK16WM, VR512, f512mem, memopv16i32, vselect, v16i32>, EVEX_CD8<32, CD8VF>, EVEX_V512; defm VPBLENDMQZ : avx512_blendmask<0x64, "vpblendmq", - int_x86_avx512_mskblend_q_512, VK8WM, VR512, f512mem, memopv8i64, vselect, v8i64>, VEX_W, EVEX_CD8<64, CD8VF>, EVEX_V512; +def : Pat<(v16i32 (int_x86_avx512_mask_blend_d_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (i16 GR16:$mask))), + (VPBLENDMDZrr (COPY_TO_REGCLASS GR16:$mask, VK16), + VR512:$src1, VR512:$src2)>; + +def : Pat<(v8i64 (int_x86_avx512_mask_blend_q_512 (v8i64 VR512:$src1), + (v8i64 VR512:$src2), (i8 GR8:$mask))), + (VPBLENDMQZrr (COPY_TO_REGCLASS GR8:$mask, VK8), + VR512:$src1, VR512:$src2)>; + let Predicates = [HasAVX512] in { def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1), (v8f32 VR256X:$src2))), @@ -681,31 +698,71 @@ def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; } +//===----------------------------------------------------------------------===// +// Compare Instructions +//===----------------------------------------------------------------------===// + +// avx512_cmp_scalar - AVX512 CMPSS and CMPSD +multiclass avx512_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, + Operand CC, SDNode OpNode, ValueType VT, + PatFrag ld_frag, string asm, string asm_alt> { + def rr : AVX512Ii8<0xC2, MRMSrcReg, + (outs VK1:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm, + [(set VK1:$dst, (OpNode (VT RC:$src1), RC:$src2, imm:$cc))], + IIC_SSE_ALU_F32S_RR>, EVEX_4V; + def rm : AVX512Ii8<0xC2, MRMSrcMem, + (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm, + [(set VK1:$dst, (OpNode (VT RC:$src1), + (ld_frag addr:$src2), imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V; + let isAsmParserOnly = 1, hasSideEffects = 0 in { + def rri_alt : AVX512Ii8<0xC2, MRMSrcReg, + (outs VK1:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc), + asm_alt, [], IIC_SSE_ALU_F32S_RR>, EVEX_4V; + def rmi_alt : AVX512Ii8<0xC2, MRMSrcMem, + (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc), + asm_alt, [], IIC_SSE_ALU_F32P_RM>, EVEX_4V; + } +} + +let Predicates = [HasAVX512] in { +defm VCMPSSZ : avx512_cmp_scalar<FR32X, f32mem, AVXCC, X86cmpms, f32, loadf32, + "vcmp${cc}ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vcmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}">, + XS; +defm VCMPSDZ : avx512_cmp_scalar<FR64X, f64mem, AVXCC, X86cmpms, f64, loadf64, + "vcmp${cc}sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vcmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}">, + XD, VEX_W; +} multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, RegisterClass KRC, RegisterClass RC, X86MemOperand x86memop, PatFrag memop_frag, SDNode OpNode, ValueType vt> { def rr : AVX512BI<opc, MRMSrcReg, (outs KRC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2)))], IIC_SSE_ALU_F32P_RR>, EVEX_4V; def rm : AVX512BI<opc, MRMSrcMem, (outs KRC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set KRC:$dst, (OpNode (vt RC:$src1), (memop_frag addr:$src2)))], IIC_SSE_ALU_F32P_RM>, EVEX_4V; } defm VPCMPEQDZ : avx512_icmp_packed<0x76, "vpcmpeqd", VK16, VR512, i512mem, - memopv16i32, X86pcmpeqm, v16i32>, EVEX_V512; + memopv16i32, X86pcmpeqm, v16i32>, EVEX_V512, + EVEX_CD8<32, CD8VF>; defm VPCMPEQQZ : avx512_icmp_packed<0x29, "vpcmpeqq", VK8, VR512, i512mem, - memopv8i64, X86pcmpeqm, v8i64>, T8, EVEX_V512, VEX_W; + memopv8i64, X86pcmpeqm, v8i64>, T8PD, EVEX_V512, + VEX_W, EVEX_CD8<64, CD8VF>; defm VPCMPGTDZ : avx512_icmp_packed<0x66, "vpcmpgtd", VK16, VR512, i512mem, - memopv16i32, X86pcmpgtm, v16i32>, EVEX_V512; + memopv16i32, X86pcmpgtm, v16i32>, EVEX_V512, + EVEX_CD8<32, CD8VF>; defm VPCMPGTQZ : avx512_icmp_packed<0x37, "vpcmpgtq", VK8, VR512, i512mem, - memopv8i64, X86pcmpgtm, v8i64>, T8, EVEX_V512, VEX_W; + memopv8i64, X86pcmpgtm, v8i64>, T8PD, EVEX_V512, + VEX_W, EVEX_CD8<64, CD8VF>; def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), (COPY_TO_REGCLASS (VPCMPGTDZrr @@ -730,12 +787,12 @@ multiclass avx512_icmp_cc<bits<8> opc, RegisterClass KRC, [(set KRC:$dst, (OpNode (vt RC:$src1), (memop_frag addr:$src2), imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V; // Accept explicit immediate argument form instead of comparison code. - let neverHasSideEffects = 1 in { + let isAsmParserOnly = 1, hasSideEffects = 0 in { def rri_alt : AVX512AIi8<opc, MRMSrcReg, - (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc), + (outs KRC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc), asm_alt, [], IIC_SSE_ALU_F32P_RR>, EVEX_4V; def rmi_alt : AVX512AIi8<opc, MRMSrcMem, - (outs RC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc), + (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc), asm_alt, [], IIC_SSE_ALU_F32P_RM>, EVEX_4V; } } @@ -764,36 +821,43 @@ defm VPCMPUQZ : avx512_icmp_cc<0x1E, VK8, VR512, i512mem, memopv8i64, // avx512_cmp_packed - sse 1 & 2 compare packed instructions multiclass avx512_cmp_packed<RegisterClass KRC, RegisterClass RC, - X86MemOperand x86memop, Operand CC, - SDNode OpNode, ValueType vt, string asm, - string asm_alt, Domain d> { + X86MemOperand x86memop, ValueType vt, + string suffix, Domain d> { def rri : AVX512PIi8<0xC2, MRMSrcReg, - (outs KRC:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm, - [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2), imm:$cc))], d>; + (outs KRC:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc), + !strconcat("vcmp${cc}", suffix, + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + [(set KRC:$dst, (X86cmpm (vt RC:$src1), (vt RC:$src2), imm:$cc))], d>; + def rrib: AVX512PIi8<0xC2, MRMSrcReg, + (outs KRC:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc), + !strconcat("vcmp${cc}", suffix, + " \t{{sae}, $src2, $src1, $dst|$dst, $src1, $src2, {sae}}"), + [], d>, EVEX_B; def rmi : AVX512PIi8<0xC2, MRMSrcMem, - (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm, + (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, AVXCC:$cc), + !strconcat("vcmp${cc}", suffix, + " \t{$src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [(set KRC:$dst, - (OpNode (vt RC:$src1), (memop addr:$src2), imm:$cc))], d>; + (X86cmpm (vt RC:$src1), (memop addr:$src2), imm:$cc))], d>; // Accept explicit immediate argument form instead of comparison code. - let neverHasSideEffects = 1 in { + let isAsmParserOnly = 1, hasSideEffects = 0 in { def rri_alt : AVX512PIi8<0xC2, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc), - asm_alt, [], d>; + !strconcat("vcmp", suffix, + " \t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [], d>; def rmi_alt : AVX512PIi8<0xC2, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc), - asm_alt, [], d>; + !strconcat("vcmp", suffix, + " \t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [], d>; } } -defm VCMPPSZ : avx512_cmp_packed<VK16, VR512, f512mem, AVXCC, X86cmpm, v16f32, - "vcmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - "vcmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", - SSEPackedSingle>, EVEX_4V, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VCMPPDZ : avx512_cmp_packed<VK8, VR512, f512mem, AVXCC, X86cmpm, v8f64, - "vcmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - "vcmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", - SSEPackedDouble>, OpSize, EVEX_4V, VEX_W, EVEX_V512, +defm VCMPPSZ : avx512_cmp_packed<VK16, VR512, f512mem, v16f32, + "ps", SSEPackedSingle>, PS, EVEX_4V, EVEX_V512, + EVEX_CD8<32, CD8VF>; +defm VCMPPDZ : avx512_cmp_packed<VK8, VR512, f512mem, v8f64, + "pd", SSEPackedDouble>, PD, EVEX_4V, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)), @@ -811,7 +875,31 @@ def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), imm:$cc), VK8)>; - + +def : Pat<(i16 (int_x86_avx512_mask_cmp_ps_512 (v16f32 VR512:$src1), + (v16f32 VR512:$src2), imm:$cc, (i16 -1), + FROUND_NO_EXC)), + (COPY_TO_REGCLASS (VCMPPSZrrib VR512:$src1, VR512:$src2, + (I8Imm imm:$cc)), GR16)>; + +def : Pat<(i8 (int_x86_avx512_mask_cmp_pd_512 (v8f64 VR512:$src1), + (v8f64 VR512:$src2), imm:$cc, (i8 -1), + FROUND_NO_EXC)), + (COPY_TO_REGCLASS (VCMPPDZrrib VR512:$src1, VR512:$src2, + (I8Imm imm:$cc)), GR8)>; + +def : Pat<(i16 (int_x86_avx512_mask_cmp_ps_512 (v16f32 VR512:$src1), + (v16f32 VR512:$src2), imm:$cc, (i16 -1), + FROUND_CURRENT)), + (COPY_TO_REGCLASS (VCMPPSZrri VR512:$src1, VR512:$src2, + (I8Imm imm:$cc)), GR16)>; + +def : Pat<(i8 (int_x86_avx512_mask_cmp_pd_512 (v8f64 VR512:$src1), + (v8f64 VR512:$src2), imm:$cc, (i8 -1), + FROUND_CURRENT)), + (COPY_TO_REGCLASS (VCMPPDZrri VR512:$src1, VR512:$src2, + (I8Imm imm:$cc)), GR8)>; + // Mask register copy, including // - copy between mask registers // - load/store mask registers @@ -820,35 +908,35 @@ def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk, string OpcodeStr, RegisterClass KRC, ValueType vt, X86MemOperand x86memop> { - let neverHasSideEffects = 1 in { + let hasSideEffects = 0 in { def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>; let mayLoad = 1 in def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set KRC:$dst, (vt (load addr:$src)))]>; let mayStore = 1 in def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>; } } multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk, string OpcodeStr, RegisterClass KRC, RegisterClass GRC> { - let neverHasSideEffects = 1 in { + let hasSideEffects = 0 in { def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>; def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>; } } let Predicates = [HasAVX512] in { defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>, - VEX, TB; + VEX, PS; defm KMOVW : avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>, - VEX, TB; + VEX, PS; } let Predicates = [HasAVX512] in { @@ -862,8 +950,40 @@ let Predicates = [HasAVX512] in { def : Pat<(store (v16i1 VK16:$src), addr:$dst), (KMOVWmk addr:$dst, VK16:$src)>; - def : Pat<(store (v8i1 VK8:$src), addr:$dst), - (KMOVWmk addr:$dst, (v16i1 (COPY_TO_REGCLASS VK8:$src, VK16)))>; + def : Pat<(store VK8:$src, addr:$dst), + (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK8:$src, VK16))>; + + def : Pat<(i1 (load addr:$src)), + (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK1)>; + + def : Pat<(v8i1 (load addr:$src)), + (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK8)>; + + def : Pat<(i1 (trunc (i32 GR32:$src))), + (COPY_TO_REGCLASS (KMOVWkr (AND32ri $src, (i32 1))), VK1)>; + + def : Pat<(i1 (trunc (i8 GR8:$src))), + (COPY_TO_REGCLASS + (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit), (i32 1))), + VK1)>; + def : Pat<(i1 (trunc (i16 GR16:$src))), + (COPY_TO_REGCLASS + (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), $src, sub_16bit), (i32 1))), + VK1)>; + + def : Pat<(i32 (zext VK1:$src)), + (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1))>; + def : Pat<(i8 (zext VK1:$src)), + (EXTRACT_SUBREG + (AND32ri (KMOVWrk + (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), sub_8bit)>; + def : Pat<(i64 (zext VK1:$src)), + (AND64ri8 (SUBREG_TO_REG (i64 0), + (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_32bit), (i64 1))>; + def : Pat<(i16 (zext VK1:$src)), + (EXTRACT_SUBREG + (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), + sub_16bit)>; } // With AVX-512 only, 8-bit mask is promoted to 16-bit mask. let Predicates = [HasAVX512] in { @@ -876,6 +996,12 @@ let Predicates = [HasAVX512] in { (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), sub_8bit)>; + + def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))), + (COPY_TO_REGCLASS VK16:$src, VK1)>; + def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))), + (COPY_TO_REGCLASS VK8:$src, VK1)>; + } // Mask unary operation @@ -884,18 +1010,27 @@ multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr, RegisterClass KRC, SDPatternOperator OpNode> { let Predicates = [HasAVX512] in def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set KRC:$dst, (OpNode KRC:$src))]>; } multiclass avx512_mask_unop_w<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode> { defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>, - VEX, TB; + VEX, PS; } defm KNOT : avx512_mask_unop_w<0x44, "knot", not>; +multiclass avx512_mask_unop_int<string IntName, string InstName> { + let Predicates = [HasAVX512] in + def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w") + (i16 GR16:$src)), + (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr") + (v16i1 (COPY_TO_REGCLASS GR16:$src, VK16))), GR16)>; +} +defm : avx512_mask_unop_int<"knot", "KNOT">; + def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>; def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>; @@ -906,27 +1041,26 @@ def : Pat<(not VK8:$src), (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>; // Mask binary operation -// - KADD, KAND, KANDN, KOR, KXNOR, KXOR +// - KAND, KANDN, KOR, KXNOR, KXOR multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr, RegisterClass KRC, SDPatternOperator OpNode> { let Predicates = [HasAVX512] in def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>; } multiclass avx512_mask_binop_w<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode> { defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>, - VEX_4V, VEX_L, TB; + VEX_4V, VEX_L, PS; } def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>; def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>; let isCommutable = 1 in { - defm KADD : avx512_mask_binop_w<0x4a, "kadd", add>; defm KAND : avx512_mask_binop_w<0x41, "kand", and>; let isCommutable = 0 in defm KANDN : avx512_mask_binop_w<0x42, "kandn", andn>; @@ -935,19 +1069,33 @@ let isCommutable = 1 in { defm KXOR : avx512_mask_binop_w<0x47, "kxor", xor>; } +def : Pat<(xor VK1:$src1, VK1:$src2), + (COPY_TO_REGCLASS (KXORWrr (COPY_TO_REGCLASS VK1:$src1, VK16), + (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; + +def : Pat<(or VK1:$src1, VK1:$src2), + (COPY_TO_REGCLASS (KORWrr (COPY_TO_REGCLASS VK1:$src1, VK16), + (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; + +def : Pat<(and VK1:$src1, VK1:$src2), + (COPY_TO_REGCLASS (KANDWrr (COPY_TO_REGCLASS VK1:$src1, VK16), + (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; + multiclass avx512_mask_binop_int<string IntName, string InstName> { let Predicates = [HasAVX512] in - def : Pat<(!cast<Intrinsic>("int_x86_"##IntName##"_v16i1") - VK16:$src1, VK16:$src2), - (!cast<Instruction>(InstName##"Wrr") VK16:$src1, VK16:$src2)>; + def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w") + (i16 GR16:$src1), (i16 GR16:$src2)), + (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr") + (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), + (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; } -defm : avx512_mask_binop_int<"kadd", "KADD">; defm : avx512_mask_binop_int<"kand", "KAND">; defm : avx512_mask_binop_int<"kandn", "KANDN">; defm : avx512_mask_binop_int<"kor", "KOR">; defm : avx512_mask_binop_int<"kxnor", "KXNOR">; defm : avx512_mask_binop_int<"kxor", "KXOR">; + // With AVX-512, 8-bit mask is promoted to 16-bit mask. multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> { let Predicates = [HasAVX512] in @@ -965,44 +1113,53 @@ defm : avx512_binop_pat<xor, KXORWrr>; // Mask unpacking multiclass avx512_mask_unpck<bits<8> opc, string OpcodeStr, - RegisterClass KRC1, RegisterClass KRC2> { + RegisterClass KRC> { let Predicates = [HasAVX512] in - def rr : I<opc, MRMSrcReg, (outs KRC1:$dst), (ins KRC2:$src1, KRC2:$src2), + def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>; + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>; } multiclass avx512_mask_unpck_bw<bits<8> opc, string OpcodeStr> { - defm BW : avx512_mask_unpck<opc, !strconcat(OpcodeStr, "bw"), VK16, VK8>, - VEX_4V, VEX_L, OpSize, TB; + defm BW : avx512_mask_unpck<opc, !strconcat(OpcodeStr, "bw"), VK16>, + VEX_4V, VEX_L, PD; } defm KUNPCK : avx512_mask_unpck_bw<0x4b, "kunpck">; +def : Pat<(v16i1 (concat_vectors (v8i1 VK8:$src1), (v8i1 VK8:$src2))), + (KUNPCKBWrr (COPY_TO_REGCLASS VK8:$src2, VK16), + (COPY_TO_REGCLASS VK8:$src1, VK16))>; + multiclass avx512_mask_unpck_int<string IntName, string InstName> { let Predicates = [HasAVX512] in - def : Pat<(!cast<Intrinsic>("int_x86_"##IntName##"_v16i1") - VK8:$src1, VK8:$src2), - (!cast<Instruction>(InstName##"BWrr") VK8:$src1, VK8:$src2)>; + def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_bw") + (i16 GR16:$src1), (i16 GR16:$src2)), + (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"BWrr") + (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), + (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; } +defm : avx512_mask_unpck_int<"kunpck", "KUNPCK">; -defm : avx512_mask_unpck_int<"kunpck", "KUNPCK">; // Mask bit testing multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC, SDNode OpNode> { let Predicates = [HasAVX512], Defs = [EFLAGS] in def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1|$src1, $src2}"), [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>; } multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode> { defm W : avx512_mask_testop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>, - VEX, TB; + VEX, PS; } defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>; -defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest>; + +def : Pat<(X86cmp VK1:$src1, (i1 0)), + (KORTESTWrr (COPY_TO_REGCLASS VK1:$src1, VK16), + (COPY_TO_REGCLASS VK1:$src1, VK16))>; // Mask shift multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC, @@ -1010,18 +1167,18 @@ multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC, let Predicates = [HasAVX512] in def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, i8imm:$imm), !strconcat(OpcodeStr, - "\t{$imm, $src, $dst|$dst, $src, $imm}"), + " \t{$imm, $src, $dst|$dst, $src, $imm}"), [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>; } multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr, SDNode OpNode> { defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>, - VEX, OpSize, TA, VEX_W; + VEX, TAPD, VEX_W; } -defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", shl>; -defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", srl>; +defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86vshli>; +defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86vsrli>; // Mask setting all 0s or 1s multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> { @@ -1032,7 +1189,7 @@ multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> { } multiclass avx512_mask_setop_w<PatFrag Val> { - defm B : avx512_mask_setop<VK8, v8i1, Val>; + defm B : avx512_mask_setop<VK8, v8i1, Val>; defm W : avx512_mask_setop<VK16, v16i1, Val>; } @@ -1043,6 +1200,9 @@ defm KSET1 : avx512_mask_setop_w<immAllOnesV>; let Predicates = [HasAVX512] in { def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>; def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>; + def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>; + def : Pat<(i1 1), (COPY_TO_REGCLASS (KSET1W), VK1)>; + def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSET1W), VK1)>; } def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))), (v8i1 (COPY_TO_REGCLASS VK16:$src, VK8))>; @@ -1053,153 +1213,167 @@ def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))), def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))), (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>; +def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))), + (v8i1 (COPY_TO_REGCLASS (KSHIFTLWri (COPY_TO_REGCLASS VK8:$src, VK16), (I8Imm $imm)), VK8))>; + +def : Pat<(v8i1 (X86vsrli VK8:$src, (i8 imm:$imm))), + (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), (I8Imm $imm)), VK8))>; //===----------------------------------------------------------------------===// // AVX-512 - Aligned and unaligned load and store // -multiclass avx512_mov_packed<bits<8> opc, RegisterClass RC, RegisterClass KRC, +multiclass avx512_load<bits<8> opc, RegisterClass RC, RegisterClass KRC, X86MemOperand x86memop, PatFrag ld_frag, - string asm, Domain d> { -let neverHasSideEffects = 1 in + string asm, Domain d, + ValueType vt, bit IsReMaterializable = 1> { +let hasSideEffects = 0 in { def rr : AVX512PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [], d>, + !strconcat(asm, " \t{$src, $dst|$dst, $src}"), [], d>, EVEX; -let canFoldAsLoad = 1 in + def rrkz : AVX512PI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src), + !strconcat(asm, + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + [], d>, EVEX, EVEX_KZ; + } + let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in def rm : AVX512PI<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), - [(set RC:$dst, (ld_frag addr:$src))], d>, EVEX; -let Constraints = "$src1 = $dst" in { + !strconcat(asm, " \t{$src, $dst|$dst, $src}"), + [(set (vt RC:$dst), (ld_frag addr:$src))], d>, EVEX; + let Constraints = "$src1 = $dst", hasSideEffects = 0 in { def rrk : AVX512PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, KRC:$mask, RC:$src2), !strconcat(asm, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), [], d>, + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), [], d>, EVEX, EVEX_K; + let mayLoad = 1 in def rmk : AVX512PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, KRC:$mask, x86memop:$src2), !strconcat(asm, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), [], d>, EVEX, EVEX_K; -} + } + let mayLoad = 1 in + def rmkz : AVX512PI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, x86memop:$src2), + !strconcat(asm, + " \t{$src2, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src2}"), + [], d>, EVEX, EVEX_KZ; } -defm VMOVAPSZ : avx512_mov_packed<0x28, VR512, VK16WM, f512mem, alignedloadv16f32, - "vmovaps", SSEPackedSingle>, - EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VMOVAPDZ : avx512_mov_packed<0x28, VR512, VK8WM, f512mem, alignedloadv8f64, - "vmovapd", SSEPackedDouble>, - OpSize, EVEX_V512, VEX_W, - EVEX_CD8<64, CD8VF>; -defm VMOVUPSZ : avx512_mov_packed<0x10, VR512, VK16WM, f512mem, loadv16f32, - "vmovups", SSEPackedSingle>, - EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VMOVUPDZ : avx512_mov_packed<0x10, VR512, VK8WM, f512mem, loadv8f64, - "vmovupd", SSEPackedDouble>, - OpSize, EVEX_V512, VEX_W, - EVEX_CD8<64, CD8VF>; -def VMOVAPSZmr : AVX512PI<0x29, MRMDestMem, (outs), (ins f512mem:$dst, VR512:$src), - "vmovaps\t{$src, $dst|$dst, $src}", - [(alignedstore512 (v16f32 VR512:$src), addr:$dst)], - SSEPackedSingle>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; -def VMOVAPDZmr : AVX512PI<0x29, MRMDestMem, (outs), (ins f512mem:$dst, VR512:$src), - "vmovapd\t{$src, $dst|$dst, $src}", - [(alignedstore512 (v8f64 VR512:$src), addr:$dst)], - SSEPackedDouble>, EVEX, EVEX_V512, - OpSize, VEX_W, EVEX_CD8<64, CD8VF>; -def VMOVUPSZmr : AVX512PI<0x11, MRMDestMem, (outs), (ins f512mem:$dst, VR512:$src), - "vmovups\t{$src, $dst|$dst, $src}", - [(store (v16f32 VR512:$src), addr:$dst)], - SSEPackedSingle>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; -def VMOVUPDZmr : AVX512PI<0x11, MRMDestMem, (outs), (ins f512mem:$dst, VR512:$src), - "vmovupd\t{$src, $dst|$dst, $src}", - [(store (v8f64 VR512:$src), addr:$dst)], - SSEPackedDouble>, EVEX, EVEX_V512, - OpSize, VEX_W, EVEX_CD8<64, CD8VF>; - -let neverHasSideEffects = 1 in { - def VMOVDQA32rr : AVX512BI<0x6F, MRMSrcReg, (outs VR512:$dst), - (ins VR512:$src), - "vmovdqa32\t{$src, $dst|$dst, $src}", []>, - EVEX, EVEX_V512; - def VMOVDQA64rr : AVX512BI<0x6F, MRMSrcReg, (outs VR512:$dst), - (ins VR512:$src), - "vmovdqa64\t{$src, $dst|$dst, $src}", []>, - EVEX, EVEX_V512, VEX_W; -let mayStore = 1 in { - def VMOVDQA32mr : AVX512BI<0x7F, MRMDestMem, (outs), - (ins i512mem:$dst, VR512:$src), - "vmovdqa32\t{$src, $dst|$dst, $src}", []>, - EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; - def VMOVDQA64mr : AVX512BI<0x7F, MRMDestMem, (outs), - (ins i512mem:$dst, VR512:$src), - "vmovdqa64\t{$src, $dst|$dst, $src}", []>, - EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -} -let mayLoad = 1 in { -def VMOVDQA32rm : AVX512BI<0x6F, MRMSrcMem, (outs VR512:$dst), - (ins i512mem:$src), - "vmovdqa32\t{$src, $dst|$dst, $src}", []>, - EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; -def VMOVDQA64rm : AVX512BI<0x6F, MRMSrcMem, (outs VR512:$dst), - (ins i512mem:$src), - "vmovdqa64\t{$src, $dst|$dst, $src}", []>, - EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -} -} - -// 512-bit aligned load/store -def : Pat<(alignedloadv8i64 addr:$src), (VMOVDQA64rm addr:$src)>; -def : Pat<(alignedloadv16i32 addr:$src), (VMOVDQA32rm addr:$src)>; - -def : Pat<(alignedstore512 (v8i64 VR512:$src), addr:$dst), - (VMOVDQA64mr addr:$dst, VR512:$src)>; -def : Pat<(alignedstore512 (v16i32 VR512:$src), addr:$dst), - (VMOVDQA32mr addr:$dst, VR512:$src)>; - -multiclass avx512_mov_int<bits<8> load_opc, bits<8> store_opc, string asm, - RegisterClass RC, RegisterClass KRC, - PatFrag ld_frag, X86MemOperand x86memop> { -let neverHasSideEffects = 1 in - def rr : AVX512XSI<load_opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), []>, EVEX; -let canFoldAsLoad = 1 in - def rm : AVX512XSI<load_opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), - [(set RC:$dst, (ld_frag addr:$src))]>, EVEX; -let mayStore = 1 in - def mr : AVX512XSI<store_opc, MRMDestMem, (outs), - (ins x86memop:$dst, VR512:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), []>, EVEX; -let Constraints = "$src1 = $dst" in { - def rrk : AVX512XSI<load_opc, MRMSrcReg, (outs RC:$dst), - (ins RC:$src1, KRC:$mask, RC:$src2), - !strconcat(asm, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), []>, +multiclass avx512_store<bits<8> opc, RegisterClass RC, RegisterClass KRC, + X86MemOperand x86memop, PatFrag store_frag, + string asm, Domain d, ValueType vt> { + let isAsmParserOnly = 1, hasSideEffects = 0 in { + def rr_alt : AVX512PI<opc, MRMDestReg, (outs RC:$dst), (ins RC:$src), + !strconcat(asm, " \t{$src, $dst|$dst, $src}"), [], d>, + EVEX; + let Constraints = "$src1 = $dst" in + def alt_rrk : AVX512PI<opc, MRMDestReg, (outs RC:$dst), + (ins RC:$src1, KRC:$mask, RC:$src2), + !strconcat(asm, + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), [], d>, EVEX, EVEX_K; - def rmk : AVX512XSI<load_opc, MRMSrcMem, (outs RC:$dst), - (ins RC:$src1, KRC:$mask, x86memop:$src2), - !strconcat(asm, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), - []>, EVEX, EVEX_K; -} + def alt_rrkz : AVX512PI<opc, MRMDestReg, (outs RC:$dst), + (ins KRC:$mask, RC:$src), + !strconcat(asm, + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + [], d>, EVEX, EVEX_KZ; + } + let mayStore = 1 in { + def mr : AVX512PI<opc, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), + !strconcat(asm, " \t{$src, $dst|$dst, $src}"), + [(store_frag (vt RC:$src), addr:$dst)], d>, EVEX; + def mrk : AVX512PI<opc, MRMDestMem, (outs), + (ins x86memop:$dst, KRC:$mask, RC:$src), + !strconcat(asm, + " \t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"), + [], d>, EVEX, EVEX_K; + def mrkz : AVX512PI<opc, MRMDestMem, (outs), + (ins x86memop:$dst, KRC:$mask, RC:$src), + !strconcat(asm, + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + [], d>, EVEX, EVEX_KZ; + } } -defm VMOVDQU32 : avx512_mov_int<0x6F, 0x7F, "vmovdqu32", VR512, VK16WM, - memopv16i32, i512mem>, - EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VMOVDQU64 : avx512_mov_int<0x6F, 0x7F, "vmovdqu64", VR512, VK8WM, - memopv8i64, i512mem>, - EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; - -// 512-bit unaligned load/store -def : Pat<(loadv8i64 addr:$src), (VMOVDQU64rm addr:$src)>; -def : Pat<(loadv16i32 addr:$src), (VMOVDQU32rm addr:$src)>; - -def : Pat<(store (v8i64 VR512:$src), addr:$dst), - (VMOVDQU64mr addr:$dst, VR512:$src)>; -def : Pat<(store (v16i32 VR512:$src), addr:$dst), - (VMOVDQU32mr addr:$dst, VR512:$src)>; +defm VMOVAPSZ : avx512_load<0x28, VR512, VK16WM, f512mem, alignedloadv16f32, + "vmovaps", SSEPackedSingle, v16f32>, + avx512_store<0x29, VR512, VK16WM, f512mem, alignedstore512, + "vmovaps", SSEPackedSingle, v16f32>, + PS, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VMOVAPDZ : avx512_load<0x28, VR512, VK8WM, f512mem, alignedloadv8f64, + "vmovapd", SSEPackedDouble, v8f64>, + avx512_store<0x29, VR512, VK8WM, f512mem, alignedstore512, + "vmovapd", SSEPackedDouble, v8f64>, + PD, EVEX_V512, VEX_W, + EVEX_CD8<64, CD8VF>; +defm VMOVUPSZ : avx512_load<0x10, VR512, VK16WM, f512mem, loadv16f32, + "vmovups", SSEPackedSingle, v16f32>, + avx512_store<0x11, VR512, VK16WM, f512mem, store, + "vmovups", SSEPackedSingle, v16f32>, + PS, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VMOVUPDZ : avx512_load<0x10, VR512, VK8WM, f512mem, loadv8f64, + "vmovupd", SSEPackedDouble, v8f64, 0>, + avx512_store<0x11, VR512, VK8WM, f512mem, store, + "vmovupd", SSEPackedDouble, v8f64>, + PD, EVEX_V512, VEX_W, + EVEX_CD8<64, CD8VF>; +def: Pat<(v8f64 (int_x86_avx512_mask_loadu_pd_512 addr:$ptr, + (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), + (VMOVUPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; + +def: Pat<(v16f32 (int_x86_avx512_mask_loadu_ps_512 addr:$ptr, + (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), + (VMOVUPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; + +def: Pat<(int_x86_avx512_mask_storeu_ps_512 addr:$ptr, (v16f32 VR512:$src), + GR16:$mask), + (VMOVUPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), + VR512:$src)>; +def: Pat<(int_x86_avx512_mask_storeu_pd_512 addr:$ptr, (v8f64 VR512:$src), + GR8:$mask), + (VMOVUPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), + VR512:$src)>; + +defm VMOVDQA32: avx512_load<0x6F, VR512, VK16WM, i512mem, alignedloadv16i32, + "vmovdqa32", SSEPackedInt, v16i32>, + avx512_store<0x7F, VR512, VK16WM, i512mem, alignedstore512, + "vmovdqa32", SSEPackedInt, v16i32>, + PD, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VMOVDQA64: avx512_load<0x6F, VR512, VK8WM, i512mem, alignedloadv8i64, + "vmovdqa64", SSEPackedInt, v8i64>, + avx512_store<0x7F, VR512, VK8WM, i512mem, alignedstore512, + "vmovdqa64", SSEPackedInt, v8i64>, + PD, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; +defm VMOVDQU32: avx512_load<0x6F, VR512, VK16WM, i512mem, load, + "vmovdqu32", SSEPackedInt, v16i32>, + avx512_store<0x7F, VR512, VK16WM, i512mem, store, + "vmovdqu32", SSEPackedInt, v16i32>, + XS, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VMOVDQU64: avx512_load<0x6F, VR512, VK8WM, i512mem, load, + "vmovdqu64", SSEPackedInt, v8i64>, + avx512_store<0x7F, VR512, VK8WM, i512mem, store, + "vmovdqu64", SSEPackedInt, v8i64>, + XS, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; let AddedComplexity = 20 in { +def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src), + (bc_v8i64 (v16i32 immAllZerosV)))), + (VMOVDQU64rrkz VK8WM:$mask, VR512:$src)>; + +def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)), + (v8i64 VR512:$src))), + (VMOVDQU64rrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)), + VK8), VR512:$src)>; + +def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src), + (v16i32 immAllZerosV))), + (VMOVDQU32rrkz VK16WM:$mask, VR512:$src)>; + +def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV), + (v16i32 VR512:$src))), + (VMOVDQU32rrkz (KNOTWrr VK16WM:$mask), VR512:$src)>; + def : Pat<(v16f32 (vselect VK16WM:$mask, (v16f32 VR512:$src1), (v16f32 VR512:$src2))), (VMOVUPSZrrk VR512:$src2, VK16WM:$mask, VR512:$src1)>; @@ -1215,33 +1389,33 @@ def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src1), } // Move Int Doubleword to Packed Double Int // -def VMOVDI2PDIZrr : AVX512SI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", +def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src), + "vmovd\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, EVEX, VEX_LIG; -def VMOVDI2PDIZrm : AVX512SI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", +def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src), + "vmovd\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v4i32 (scalar_to_vector (loadi32 addr:$src))))], IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; -def VMOV64toPQIZrr : AVX512SI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", +def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src), + "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (scalar_to_vector GR64:$src)))], IIC_SSE_MOVDQ>, EVEX, VEX_W, VEX_LIG; let isCodeGenOnly = 1 in { -def VMOV64toSDZrr : AVX512SI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", +def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), + "vmovq\t{$src, $dst|$dst, $src}", [(set FR64:$dst, (bitconvert GR64:$src))], IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; -def VMOVSDto64Zrr : AVX512SI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", +def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), + "vmovq\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (bitconvert FR64:$src))], IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; } -def VMOVSDto64Zmr : AVX512SI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", +def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), + "vmovq\t{$src, $dst|$dst, $src}", [(store (i64 (bitconvert FR64:$src)), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>, EVEX_CD8<64, CD8VT1>; @@ -1249,68 +1423,68 @@ def VMOVSDto64Zmr : AVX512SI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$s // Move Int Doubleword to Single Scalar // let isCodeGenOnly = 1 in { -def VMOVDI2SSZrr : AVX512SI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", +def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src), + "vmovd\t{$src, $dst|$dst, $src}", [(set FR32X:$dst, (bitconvert GR32:$src))], IIC_SSE_MOVDQ>, EVEX, VEX_LIG; -def VMOVDI2SSZrm : AVX512SI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", +def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src), + "vmovd\t{$src, $dst|$dst, $src}", [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))], IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; } -// Move Packed Doubleword Int to Packed Double Int +// Move doubleword from xmm register to r/m32 // -def VMOVPDI2DIZrr : AVX512SI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", +def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src), + "vmovd\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (vector_extract (v4i32 VR128X:$src), (iPTR 0)))], IIC_SSE_MOVD_ToGP>, EVEX, VEX_LIG; -def VMOVPDI2DIZmr : AVX512SI<0x7E, MRMDestMem, (outs), +def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128X:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", + "vmovd\t{$src, $dst|$dst, $src}", [(store (i32 (vector_extract (v4i32 VR128X:$src), (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; -// Move Packed Doubleword Int first element to Doubleword Int +// Move quadword from xmm1 register to r/m64 // def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", + "vmovq\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (extractelt (v2i64 VR128X:$src), (iPTR 0)))], - IIC_SSE_MOVD_ToGP>, TB, OpSize, EVEX, VEX_LIG, VEX_W, + IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_LIG, VEX_W, Requires<[HasAVX512, In64BitMode]>; def VMOVPQIto64Zmr : I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", + "vmovq\t{$src, $dst|$dst, $src}", [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)), addr:$dst)], IIC_SSE_MOVDQ>, - EVEX, OpSize, VEX_LIG, VEX_W, TB, EVEX_CD8<64, CD8VT1>, + EVEX, PD, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>, Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>; // Move Scalar Single to Double Int // let isCodeGenOnly = 1 in { -def VMOVSS2DIZrr : AVX512SI<0x7E, MRMDestReg, (outs GR32:$dst), +def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32X:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", + "vmovd\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (bitconvert FR32X:$src))], IIC_SSE_MOVD_ToGP>, EVEX, VEX_LIG; -def VMOVSS2DIZmr : AVX512SI<0x7E, MRMDestMem, (outs), +def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32X:$src), - "vmovd{z}\t{$src, $dst|$dst, $src}", + "vmovd\t{$src, $dst|$dst, $src}", [(store (i32 (bitconvert FR32X:$src)), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; } // Move Quadword Int to Packed Quadword Int // -def VMOVQI2PQIZrm : AVX512SI<0x6E, MRMSrcMem, (outs VR128X:$dst), +def VMOVQI2PQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i64mem:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", + "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; @@ -1322,40 +1496,55 @@ def VMOVQI2PQIZrm : AVX512SI<0x6E, MRMSrcMem, (outs VR128X:$dst), multiclass avx512_move_scalar <string asm, RegisterClass RC, SDNode OpNode, ValueType vt, X86MemOperand x86memop, PatFrag mem_pat> { + let hasSideEffects = 0 in { def rr : SI<0x10, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, RC:$src2), - !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(asm, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128X:$dst, (vt (OpNode VR128X:$src1, (scalar_to_vector RC:$src2))))], IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG; + let Constraints = "$src1 = $dst" in + def rrk : SI<0x10, MRMSrcReg, (outs VR128X:$dst), + (ins VR128X:$src1, VK1WM:$mask, RC:$src2, RC:$src3), + !strconcat(asm, + " \t{$src3, $src2, $dst {${mask}}|$dst {${mask}}, $src2, $src3}"), + [], IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG, EVEX_K; def rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), + !strconcat(asm, " \t{$src, $dst|$dst, $src}"), [(set RC:$dst, (mem_pat addr:$src))], IIC_SSE_MOV_S_RM>, EVEX, VEX_LIG; def mr: SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), - !strconcat(asm, "\t{$src, $dst|$dst, $src}"), + !strconcat(asm, " \t{$src, $dst|$dst, $src}"), [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR>, EVEX, VEX_LIG; + } //hasSideEffects = 0 } let ExeDomain = SSEPackedSingle in -defm VMOVSSZ : avx512_move_scalar<"movss{z}", FR32X, X86Movss, v4f32, f32mem, +defm VMOVSSZ : avx512_move_scalar<"movss", FR32X, X86Movss, v4f32, f32mem, loadf32>, XS, EVEX_CD8<32, CD8VT1>; let ExeDomain = SSEPackedDouble in -defm VMOVSDZ : avx512_move_scalar<"movsd{z}", FR64X, X86Movsd, v2f64, f64mem, +defm VMOVSDZ : avx512_move_scalar<"movsd", FR64X, X86Movsd, v2f64, f64mem, loadf64>, XD, VEX_W, EVEX_CD8<64, CD8VT1>; +def : Pat<(f32 (X86select VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))), + (COPY_TO_REGCLASS (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X), + VK1WM:$mask, (f32 (IMPLICIT_DEF)), FR32X:$src1), FR32X)>; + +def : Pat<(f64 (X86select VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))), + (COPY_TO_REGCLASS (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X), + VK1WM:$mask, (f64 (IMPLICIT_DEF)), FR64X:$src1), FR64X)>; // For the disassembler -let isCodeGenOnly = 1 in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { def VMOVSSZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst), (ins VR128X:$src1, FR32X:$src2), - "movss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], + "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], IIC_SSE_MOV_S_RR>, XS, EVEX_4V, VEX_LIG; def VMOVSDZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst), (ins VR128X:$src1, FR64X:$src2), - "movsd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], + "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], IIC_SSE_MOV_S_RR>, XD, EVEX_4V, VEX_LIG, VEX_W; } @@ -1504,7 +1693,7 @@ let Predicates = [HasAVX512] in { let AddedComplexity = 15 in def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", + "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (X86vzmovl (v2i64 VR128X:$src))))], IIC_SSE_MOVQ_RR>, EVEX, VEX_W; @@ -1512,7 +1701,7 @@ def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst), let AddedComplexity = 20 in def VMOVZPQILo2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst), (ins i128mem:$src), - "vmovq{z}\t{$src, $dst|$dst, $src}", + "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (X86vzmovl (loadv2i64 addr:$src))))], IIC_SSE_MOVDQ>, EVEX, VEX_W, @@ -1536,6 +1725,8 @@ let Predicates = [HasAVX512] in { (VMOVZPQILo2PQIZrm addr:$src)>; def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))), (VMOVZPQILo2PQIZrr VR128X:$src)>; + def : Pat<(v2i64 (X86vzload addr:$src)), + (VMOVZPQILo2PQIZrm addr:$src)>; } // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext. @@ -1563,104 +1754,251 @@ def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))), // AVX-512 - Integer arithmetic // multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, - ValueType OpVT, RegisterClass RC, PatFrag memop_frag, + ValueType OpVT, RegisterClass KRC, + RegisterClass RC, PatFrag memop_frag, X86MemOperand x86memop, PatFrag scalar_mfrag, X86MemOperand x86scalar_mop, string BrdcstStr, OpndItins itins, bit IsCommutable = 0> { let isCommutable = IsCommutable in - def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), - (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))], - itins.rr>, EVEX_4V; - def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), - (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (memop_frag addr:$src2))))], - itins.rm>, EVEX_4V; - def rmb : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), - (ins RC:$src1, x86scalar_mop:$src2), - !strconcat(OpcodeStr, "\t{${src2}", BrdcstStr, - ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"), - [(set RC:$dst, (OpNode RC:$src1, - (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))))], - itins.rm>, EVEX_4V, EVEX_B; -} -multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, - ValueType DstVT, ValueType SrcVT, RegisterClass RC, - PatFrag memop_frag, X86MemOperand x86memop, - OpndItins itins, - bit IsCommutable = 0> { + def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), + (ins RC:$src1, RC:$src2), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))], + itins.rr>, EVEX_4V; + let AddedComplexity = 30 in { + let Constraints = "$src0 = $dst" in + def rrk : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), + (ins RC:$src0, KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + [(set RC:$dst, (OpVT (vselect KRC:$mask, + (OpNode (OpVT RC:$src1), (OpVT RC:$src2)), + RC:$src0)))], + itins.rr>, EVEX_4V, EVEX_K; + def rrkz : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst {${mask}} {z}" , + "|$dst {${mask}} {z}, $src1, $src2}"), + [(set RC:$dst, (OpVT (vselect KRC:$mask, + (OpNode (OpVT RC:$src1), (OpVT RC:$src2)), + (OpVT immAllZerosV))))], + itins.rr>, EVEX_4V, EVEX_KZ; + } + + let mayLoad = 1 in { + def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (memop_frag addr:$src2))))], + itins.rm>, EVEX_4V; + let AddedComplexity = 30 in { + let Constraints = "$src0 = $dst" in + def rmk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src0, KRC:$mask, RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + [(set RC:$dst, (OpVT (vselect KRC:$mask, + (OpNode (OpVT RC:$src1), (memop_frag addr:$src2)), + RC:$src0)))], + itins.rm>, EVEX_4V, EVEX_K; + def rmkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"), + [(set RC:$dst, (OpVT (vselect KRC:$mask, + (OpNode (OpVT RC:$src1), (memop_frag addr:$src2)), + (OpVT immAllZerosV))))], + itins.rm>, EVEX_4V, EVEX_KZ; + } + def rmb : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src1, x86scalar_mop:$src2), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"), + [(set RC:$dst, (OpNode RC:$src1, + (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))))], + itins.rm>, EVEX_4V, EVEX_B; + let AddedComplexity = 30 in { + let Constraints = "$src0 = $dst" in + def rmbk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src0, KRC:$mask, RC:$src1, x86scalar_mop:$src2), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst {${mask}}|$dst {${mask}}, $src1, ${src2}", + BrdcstStr, "}"), + [(set RC:$dst, (OpVT (vselect KRC:$mask, + (OpNode (OpVT RC:$src1), + (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))), + RC:$src0)))], + itins.rm>, EVEX_4V, EVEX_B, EVEX_K; + def rmbkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, ${src2}", + BrdcstStr, "}"), + [(set RC:$dst, (OpVT (vselect KRC:$mask, + (OpNode (OpVT RC:$src1), + (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))), + (OpVT immAllZerosV))))], + itins.rm>, EVEX_4V, EVEX_B, EVEX_KZ; + } + } +} + +multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, ValueType DstVT, + ValueType SrcVT, RegisterClass KRC, RegisterClass RC, + PatFrag memop_frag, X86MemOperand x86memop, + PatFrag scalar_mfrag, X86MemOperand x86scalar_mop, + string BrdcstStr, OpndItins itins, bit IsCommutable = 0> { let isCommutable = IsCommutable in - def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), + { + def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, EVEX_4V, VEX_W; - def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), - (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, EVEX_4V, VEX_W; + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX_4V; + def rrk : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + [], itins.rr>, EVEX_4V, EVEX_K; + def rrkz : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst {${mask}} {z}" , + "|$dst {${mask}} {z}, $src1, $src2}"), + [], itins.rr>, EVEX_4V, EVEX_KZ; + } + let mayLoad = 1 in { + def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX_4V; + def rmk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + [], itins.rm>, EVEX_4V, EVEX_K; + def rmkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86memop:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"), + [], itins.rm>, EVEX_4V, EVEX_KZ; + def rmb : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src1, x86scalar_mop:$src2), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"), + [], itins.rm>, EVEX_4V, EVEX_B; + def rmbk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst {${mask}}|$dst {${mask}}, $src1, ${src2}", + BrdcstStr, "}"), + [], itins.rm>, EVEX_4V, EVEX_B, EVEX_K; + def rmbkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, ${src2}", + BrdcstStr, "}"), + [], itins.rm>, EVEX_4V, EVEX_B, EVEX_KZ; + } } -defm VPADDDZ : avx512_binop_rm<0xFE, "vpaddd", add, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 1>, - EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPADDDZ : avx512_binop_rm<0xFE, "vpaddd", add, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPSUBDZ : avx512_binop_rm<0xFA, "vpsubd", sub, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 0>, - EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPSUBDZ : avx512_binop_rm<0xFA, "vpsubd", sub, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 0>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPMULLDZ : avx512_binop_rm<0x40, "vpmulld", mul, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 1>, - T8, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPMULLDZ : avx512_binop_rm<0x40, "vpmulld", mul, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 1>, T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPADDQZ : avx512_binop_rm<0xD4, "vpaddq", add, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_INTALU_ITINS_P, 1>, - EVEX_CD8<64, CD8VF>, EVEX_V512, VEX_W; +defm VPADDQZ : avx512_binop_rm<0xD4, "vpaddq", add, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 1>, EVEX_CD8<64, CD8VF>, EVEX_V512, VEX_W; -defm VPSUBQZ : avx512_binop_rm<0xFB, "vpsubq", sub, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_INTALU_ITINS_P, 0>, - EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; +defm VPSUBQZ : avx512_binop_rm<0xFB, "vpsubq", sub, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 0>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -defm VPMULDQZ : avx512_binop_rm2<0x28, "vpmuldq", v8i64, v16i32, - VR512, memopv8i64, i512mem, SSE_INTALU_ITINS_P, 1>, T8, - EVEX_V512, EVEX_CD8<64, CD8VF>; +defm VPMULDQZ : avx512_binop_rm2<0x28, "vpmuldq", v8i64, v16i32, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 1>, T8PD, EVEX_V512, + EVEX_CD8<64, CD8VF>, VEX_W; -defm VPMULUDQZ : avx512_binop_rm2<0xF4, "vpmuludq", v8i64, v16i32, - VR512, memopv8i64, i512mem, SSE_INTMUL_ITINS_P, 1>, EVEX_V512, - EVEX_CD8<64, CD8VF>; +defm VPMULUDQZ : avx512_binop_rm2<0xF4, "vpmuludq", v8i64, v16i32, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTMUL_ITINS_P, 1>, EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W; def : Pat<(v8i64 (X86pmuludq (v16i32 VR512:$src1), (v16i32 VR512:$src2))), (VPMULUDQZrr VR512:$src1, VR512:$src2)>; -defm VPMAXUDZ : avx512_binop_rm<0x3F, "vpmaxud", X86umax, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 1>, - T8, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPMAXUQZ : avx512_binop_rm<0x3F, "vpmaxuq", X86umax, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_INTALU_ITINS_P, 0>, - T8, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; - -defm VPMAXSDZ : avx512_binop_rm<0x3D, "vpmaxsd", X86smax, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 1>, - EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPMAXSQZ : avx512_binop_rm<0x3D, "vpmaxsq", X86smax, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_INTALU_ITINS_P, 0>, - T8, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; - -defm VPMINUDZ : avx512_binop_rm<0x3B, "vpminud", X86umin, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 1>, - T8, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPMINUQZ : avx512_binop_rm<0x3B, "vpminuq", X86umin, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_INTALU_ITINS_P, 0>, - T8, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; - -defm VPMINSDZ : avx512_binop_rm<0x39, "vpminsd", X86smin, v16i32, VR512, memopv16i32, - i512mem, loadi32, i32mem, "{1to16}", SSE_INTALU_ITINS_P, 1>, - T8, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPMINSQZ : avx512_binop_rm<0x39, "vpminsq", X86smin, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_INTALU_ITINS_P, 0>, - T8, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; - +def : Pat<(v8i64 (int_x86_avx512_mask_pmulu_dq_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPMULUDQZrr VR512:$src1, VR512:$src2)>; +def : Pat<(v8i64 (int_x86_avx512_mask_pmul_dq_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPMULDQZrr VR512:$src1, VR512:$src2)>; + +defm VPMAXUDZ : avx512_binop_rm<0x3F, "vpmaxud", X86umax, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 1>, + T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPMAXUQZ : avx512_binop_rm<0x3F, "vpmaxuq", X86umax, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 0>, + T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + +defm VPMAXSDZ : avx512_binop_rm<0x3D, "vpmaxsd", X86smax, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 1>, + T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPMAXSQZ : avx512_binop_rm<0x3D, "vpmaxsq", X86smax, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 0>, + T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + +defm VPMINUDZ : avx512_binop_rm<0x3B, "vpminud", X86umin, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 1>, + T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPMINUQZ : avx512_binop_rm<0x3B, "vpminuq", X86umin, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 0>, + T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + +defm VPMINSDZ : avx512_binop_rm<0x39, "vpminsd", X86smin, v16i32, VK16WM, VR512, + memopv16i32, i512mem, loadi32, i32mem, "{1to16}", + SSE_INTALU_ITINS_P, 1>, + T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VPMINSQZ : avx512_binop_rm<0x39, "vpminsq", X86smin, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_INTALU_ITINS_P, 0>, + T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + +def : Pat <(v16i32 (int_x86_avx512_mask_pmaxs_d_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))), + (VPMAXSDZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v16i32 (int_x86_avx512_mask_pmaxu_d_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))), + (VPMAXUDZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v8i64 (int_x86_avx512_mask_pmaxs_q_512 (v8i64 VR512:$src1), + (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPMAXSQZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v8i64 (int_x86_avx512_mask_pmaxu_q_512 (v8i64 VR512:$src1), + (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPMAXUQZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v16i32 (int_x86_avx512_mask_pmins_d_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))), + (VPMINSDZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v16i32 (int_x86_avx512_mask_pminu_d_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))), + (VPMINUDZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v8i64 (int_x86_avx512_mask_pmins_q_512 (v8i64 VR512:$src1), + (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPMINSQZrr VR512:$src1, VR512:$src2)>; +def : Pat <(v8i64 (int_x86_avx512_mask_pminu_q_512 (v8i64 VR512:$src1), + (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPMINUQZrr VR512:$src1, VR512:$src2)>; //===----------------------------------------------------------------------===// // AVX-512 - Unpack Instructions //===----------------------------------------------------------------------===// @@ -1684,28 +2022,28 @@ multiclass avx512_unpack_fp<bits<8> opc, SDNode OpNode, ValueType vt, defm VUNPCKHPSZ: avx512_unpack_fp<0x15, X86Unpckh, v16f32, memopv8f64, VR512, f512mem, "vunpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedSingle>, EVEX_V512, EVEX_CD8<32, CD8VF>; + SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>; defm VUNPCKHPDZ: avx512_unpack_fp<0x15, X86Unpckh, v8f64, memopv8f64, VR512, f512mem, "vunpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedDouble>, OpSize, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; defm VUNPCKLPSZ: avx512_unpack_fp<0x14, X86Unpckl, v16f32, memopv8f64, VR512, f512mem, "vunpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedSingle>, EVEX_V512, EVEX_CD8<32, CD8VF>; + SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>; defm VUNPCKLPDZ: avx512_unpack_fp<0x14, X86Unpckl, v8f64, memopv8f64, VR512, f512mem, "vunpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedDouble>, OpSize, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; multiclass avx512_unpack_int<bits<8> opc, string OpcodeStr, SDNode OpNode, ValueType OpVT, RegisterClass RC, PatFrag memop_frag, X86MemOperand x86memop> { def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))], IIC_SSE_UNPCK>, EVEX_4V; def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (bitconvert (memop_frag addr:$src2)))))], IIC_SSE_UNPCK>, EVEX_4V; @@ -1732,29 +2070,29 @@ multiclass avx512_pshuf_imm<bits<8> opc, string OpcodeStr, RegisterClass RC, def ri : AVX512Ii8<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, i8imm:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (OpNode RC:$src1, (i8 imm:$src2))))]>, EVEX; def mi : AVX512Ii8<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1, i8imm:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpVT (OpNode (mem_frag addr:$src1), (i8 imm:$src2))))]>, EVEX; } defm VPSHUFDZ : avx512_pshuf_imm<0x70, "vpshufd", VR512, X86PShufd, memopv16i32, - i512mem, v16i32>, OpSize, EVEX_V512, EVEX_CD8<32, CD8VF>; + i512mem, v16i32>, PD, EVEX_V512, EVEX_CD8<32, CD8VF>; let ExeDomain = SSEPackedSingle in defm VPERMILPSZ : avx512_pshuf_imm<0x04, "vpermilps", VR512, X86VPermilp, - memopv16f32, i512mem, v16f32>, OpSize, TA, EVEX_V512, + memopv16f32, i512mem, v16f32>, TAPD, EVEX_V512, EVEX_CD8<32, CD8VF>; let ExeDomain = SSEPackedDouble in defm VPERMILPDZ : avx512_pshuf_imm<0x05, "vpermilpd", VR512, X86VPermilp, - memopv8f64, i512mem, v8f64>, OpSize, TA, EVEX_V512, + memopv8f64, i512mem, v8f64>, TAPD, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VF>; def : Pat<(v16i32 (X86VPermilp VR512:$src1, (i8 imm:$imm))), @@ -1766,30 +2104,30 @@ def : Pat<(v8i64 (X86VPermilp VR512:$src1, (i8 imm:$imm))), // AVX-512 Logical Instructions //===----------------------------------------------------------------------===// -defm VPANDDZ : avx512_binop_rm<0xDB, "vpandd", and, v16i32, VR512, memopv16i32, +defm VPANDDZ : avx512_binop_rm<0xDB, "vpandd", and, v16i32, VK16WM, VR512, memopv16i32, i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPANDQZ : avx512_binop_rm<0xDB, "vpandq", and, v8i64, VR512, memopv8i64, +defm VPANDQZ : avx512_binop_rm<0xDB, "vpandq", and, v8i64, VK8WM, VR512, memopv8i64, i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 1>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -defm VPORDZ : avx512_binop_rm<0xEB, "vpord", or, v16i32, VR512, memopv16i32, +defm VPORDZ : avx512_binop_rm<0xEB, "vpord", or, v16i32, VK16WM, VR512, memopv16i32, i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPORQZ : avx512_binop_rm<0xEB, "vporq", or, v8i64, VR512, memopv8i64, +defm VPORQZ : avx512_binop_rm<0xEB, "vporq", or, v8i64, VK8WM, VR512, memopv8i64, i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 1>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -defm VPXORDZ : avx512_binop_rm<0xEF, "vpxord", xor, v16i32, VR512, memopv16i32, +defm VPXORDZ : avx512_binop_rm<0xEF, "vpxord", xor, v16i32, VK16WM, VR512, memopv16i32, i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPXORQZ : avx512_binop_rm<0xEF, "vpxorq", xor, v8i64, VR512, memopv8i64, +defm VPXORQZ : avx512_binop_rm<0xEF, "vpxorq", xor, v8i64, VK8WM, VR512, memopv8i64, i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 1>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -defm VPANDNDZ : avx512_binop_rm<0xDF, "vpandnd", X86andnp, v16i32, VR512, +defm VPANDNDZ : avx512_binop_rm<0xDF, "vpandnd", X86andnp, v16i32, VK16WM, VR512, memopv16i32, i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 0>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VPANDNQZ : avx512_binop_rm<0xDF, "vpandnq", X86andnp, v8i64, VR512, memopv8i64, - i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 0>, - EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; +defm VPANDNQZ : avx512_binop_rm<0xDF, "vpandnq", X86andnp, v8i64, VK8WM, VR512, + memopv8i64, i512mem, loadi64, i64mem, "{1to8}", + SSE_BIT_ITINS_P, 0>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; //===----------------------------------------------------------------------===// // AVX-512 FP arithmetic @@ -1797,10 +2135,10 @@ defm VPANDNQZ : avx512_binop_rm<0xDF, "vpandnq", X86andnp, v8i64, VR512, memopv8 multiclass avx512_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, SizeItins itins> { - defm SSZ : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss{z}"), OpNode, FR32X, + defm SSZ : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"), OpNode, FR32X, f32mem, itins.s, 0>, XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; - defm SDZ : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd{z}"), OpNode, FR64X, + defm SDZ : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"), OpNode, FR64X, f64mem, itins.d, 0>, XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; } @@ -1817,82 +2155,138 @@ defm VDIV : avx512_binop_s<0x5E, "div", fdiv, SSE_ALU_ITINS_S>; } multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, + RegisterClass KRC, RegisterClass RC, ValueType vt, X86MemOperand x86memop, PatFrag mem_frag, X86MemOperand x86scalar_mop, PatFrag scalar_mfrag, string BrdcstStr, Domain d, OpndItins itins, bit commutable> { - let isCommutable = commutable in + let isCommutable = commutable in { def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], itins.rr, d>, - EVEX_4V, TB; + EVEX_4V; + + def rrk: PI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}} |$dst {${mask}}, $src1, $src2}"), + [], itins.rr, d>, EVEX_4V, EVEX_K; + + def rrkz: PI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src1, RC:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"), + [], itins.rr, d>, EVEX_4V, EVEX_KZ; + } + let mayLoad = 1 in { def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpNode RC:$src1, (mem_frag addr:$src2)))], - itins.rm, d>, EVEX_4V, TB; + itins.rm, d>, EVEX_4V; + def rmb : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86scalar_mop:$src2), - !strconcat(OpcodeStr, "\t{${src2}", BrdcstStr, - ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"), + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, + ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"), [(set RC:$dst, (OpNode RC:$src1, (vt (X86VBroadcast (scalar_mfrag addr:$src2)))))], - itins.rm, d>, EVEX_4V, EVEX_B, TB; - } + itins.rm, d>, EVEX_4V, EVEX_B; + + def rmk : PI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86memop:$src2), !strconcat(OpcodeStr, + "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + [], itins.rm, d>, EVEX_4V, EVEX_K; + + def rmkz : PI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86memop:$src2), !strconcat(OpcodeStr, + "\t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"), + [], itins.rm, d>, EVEX_4V, EVEX_KZ; + + def rmbk : PI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), !strconcat(OpcodeStr, + " \t{${src2}", BrdcstStr, + ", $src1, $dst {${mask}}|$dst {${mask}}, $src1, ${src2}", BrdcstStr, "}"), + [], itins.rm, d>, EVEX_4V, EVEX_B, EVEX_K; + + def rmbkz : PI<opc, MRMSrcMem, (outs RC:$dst), + (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), !strconcat(OpcodeStr, + " \t{${src2}", BrdcstStr, + ", $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, ${src2}", + BrdcstStr, "}"), + [], itins.rm, d>, EVEX_4V, EVEX_B, EVEX_KZ; + } } -defm VADDPSZ : avx512_fp_packed<0x58, "addps", fadd, VR512, v16f32, f512mem, +defm VADDPSZ : avx512_fp_packed<0x58, "addps", fadd, VK16WM, VR512, v16f32, f512mem, memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, - SSE_ALU_ITINS_P.s, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>; + SSE_ALU_ITINS_P.s, 1>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; -defm VADDPDZ : avx512_fp_packed<0x58, "addpd", fadd, VR512, v8f64, f512mem, +defm VADDPDZ : avx512_fp_packed<0x58, "addpd", fadd, VK8WM, VR512, v8f64, f512mem, memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble, SSE_ALU_ITINS_P.d, 1>, - EVEX_V512, OpSize, VEX_W, EVEX_CD8<64, CD8VF>; + EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; -defm VMULPSZ : avx512_fp_packed<0x59, "mulps", fmul, VR512, v16f32, f512mem, +defm VMULPSZ : avx512_fp_packed<0x59, "mulps", fmul, VK16WM, VR512, v16f32, f512mem, memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, - SSE_ALU_ITINS_P.s, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VMULPDZ : avx512_fp_packed<0x59, "mulpd", fmul, VR512, v8f64, f512mem, + SSE_ALU_ITINS_P.s, 1>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; +defm VMULPDZ : avx512_fp_packed<0x59, "mulpd", fmul, VK8WM, VR512, v8f64, f512mem, memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble, SSE_ALU_ITINS_P.d, 1>, - EVEX_V512, OpSize, VEX_W, EVEX_CD8<64, CD8VF>; + EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; -defm VMINPSZ : avx512_fp_packed<0x5D, "minps", X86fmin, VR512, v16f32, f512mem, +defm VMINPSZ : avx512_fp_packed<0x5D, "minps", X86fmin, VK16WM, VR512, v16f32, f512mem, memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, SSE_ALU_ITINS_P.s, 1>, - EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VMAXPSZ : avx512_fp_packed<0x5F, "maxps", X86fmax, VR512, v16f32, f512mem, + EVEX_V512, PS, EVEX_CD8<32, CD8VF>; +defm VMAXPSZ : avx512_fp_packed<0x5F, "maxps", X86fmax, VK16WM, VR512, v16f32, f512mem, memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, SSE_ALU_ITINS_P.s, 1>, - EVEX_V512, EVEX_CD8<32, CD8VF>; + EVEX_V512, PS, EVEX_CD8<32, CD8VF>; -defm VMINPDZ : avx512_fp_packed<0x5D, "minpd", X86fmin, VR512, v8f64, f512mem, +defm VMINPDZ : avx512_fp_packed<0x5D, "minpd", X86fmin, VK8WM, VR512, v8f64, f512mem, memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble, SSE_ALU_ITINS_P.d, 1>, - EVEX_V512, OpSize, VEX_W, EVEX_CD8<64, CD8VF>; -defm VMAXPDZ : avx512_fp_packed<0x5F, "maxpd", X86fmax, VR512, v8f64, f512mem, + EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; +defm VMAXPDZ : avx512_fp_packed<0x5F, "maxpd", X86fmax, VK8WM, VR512, v8f64, f512mem, memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble, SSE_ALU_ITINS_P.d, 1>, - EVEX_V512, OpSize, VEX_W, EVEX_CD8<64, CD8VF>; + EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; -defm VSUBPSZ : avx512_fp_packed<0x5C, "subps", fsub, VR512, v16f32, f512mem, +defm VSUBPSZ : avx512_fp_packed<0x5C, "subps", fsub, VK16WM, VR512, v16f32, f512mem, memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, - SSE_ALU_ITINS_P.s, 0>, EVEX_V512, EVEX_CD8<32, CD8VF>; -defm VDIVPSZ : avx512_fp_packed<0x5E, "divps", fdiv, VR512, v16f32, f512mem, + SSE_ALU_ITINS_P.s, 0>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; +defm VDIVPSZ : avx512_fp_packed<0x5E, "divps", fdiv, VK16WM, VR512, v16f32, f512mem, memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, - SSE_ALU_ITINS_P.s, 0>, EVEX_V512, EVEX_CD8<32, CD8VF>; + SSE_ALU_ITINS_P.s, 0>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; -defm VSUBPDZ : avx512_fp_packed<0x5C, "subpd", fsub, VR512, v8f64, f512mem, +defm VSUBPDZ : avx512_fp_packed<0x5C, "subpd", fsub, VK8WM, VR512, v8f64, f512mem, memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble, SSE_ALU_ITINS_P.d, 0>, - EVEX_V512, OpSize, VEX_W, EVEX_CD8<64, CD8VF>; -defm VDIVPDZ : avx512_fp_packed<0x5E, "divpd", fdiv, VR512, v8f64, f512mem, + EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; +defm VDIVPDZ : avx512_fp_packed<0x5E, "divpd", fdiv, VK8WM, VR512, v8f64, f512mem, memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble, SSE_ALU_ITINS_P.d, 0>, - EVEX_V512, OpSize, VEX_W, EVEX_CD8<64, CD8VF>; - + EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; + +def : Pat<(v16f32 (int_x86_avx512_mask_max_ps_512 (v16f32 VR512:$src1), + (v16f32 VR512:$src2), (bc_v16f32 (v16i32 immAllZerosV)), + (i16 -1), FROUND_CURRENT)), + (VMAXPSZrr VR512:$src1, VR512:$src2)>; + +def : Pat<(v8f64 (int_x86_avx512_mask_max_pd_512 (v8f64 VR512:$src1), + (v8f64 VR512:$src2), (bc_v8f64 (v16i32 immAllZerosV)), + (i8 -1), FROUND_CURRENT)), + (VMAXPDZrr VR512:$src1, VR512:$src2)>; + +def : Pat<(v16f32 (int_x86_avx512_mask_min_ps_512 (v16f32 VR512:$src1), + (v16f32 VR512:$src2), (bc_v16f32 (v16i32 immAllZerosV)), + (i16 -1), FROUND_CURRENT)), + (VMINPSZrr VR512:$src1, VR512:$src2)>; + +def : Pat<(v8f64 (int_x86_avx512_mask_min_pd_512 (v8f64 VR512:$src1), + (v8f64 VR512:$src2), (bc_v8f64 (v16i32 immAllZerosV)), + (i8 -1), FROUND_CURRENT)), + (VMINPDZrr VR512:$src1, VR512:$src2)>; //===----------------------------------------------------------------------===// // AVX-512 VPTESTM instructions //===----------------------------------------------------------------------===// @@ -1900,24 +2294,41 @@ defm VDIVPDZ : avx512_fp_packed<0x5E, "divpd", fdiv, VR512, v8f64, f512mem, multiclass avx512_vptest<bits<8> opc, string OpcodeStr, RegisterClass KRC, RegisterClass RC, X86MemOperand x86memop, PatFrag memop_frag, SDNode OpNode, ValueType vt> { - def rr : AVX5128I<opc, MRMSrcReg, + def rr : AVX512PI<opc, MRMSrcReg, (outs KRC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2)))]>, EVEX_4V; - def rm : AVX5128I<opc, MRMSrcMem, + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2)))], + SSEPackedInt>, EVEX_4V; + def rm : AVX512PI<opc, MRMSrcMem, (outs KRC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set KRC:$dst, (OpNode (vt RC:$src1), - (bitconvert (memop_frag addr:$src2))))]>, EVEX_4V; + (bitconvert (memop_frag addr:$src2))))], SSEPackedInt>, EVEX_4V; } defm VPTESTMDZ : avx512_vptest<0x27, "vptestmd", VK16, VR512, f512mem, - memopv16i32, X86testm, v16i32>, EVEX_V512, + memopv16i32, X86testm, v16i32>, T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; defm VPTESTMQZ : avx512_vptest<0x27, "vptestmq", VK8, VR512, f512mem, - memopv8i64, X86testm, v8i64>, EVEX_V512, VEX_W, + memopv8i64, X86testm, v8i64>, T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; +let Predicates = [HasCDI] in { +defm VPTESTNMDZ : avx512_vptest<0x27, "vptestnmd", VK16, VR512, f512mem, + memopv16i32, X86testnm, v16i32>, T8XS, EVEX_V512, + EVEX_CD8<32, CD8VF>; +defm VPTESTNMQZ : avx512_vptest<0x27, "vptestnmq", VK8, VR512, f512mem, + memopv8i64, X86testnm, v8i64>, T8XS, EVEX_V512, VEX_W, + EVEX_CD8<64, CD8VF>; +} + +def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1), + (v16i32 VR512:$src2), (i16 -1))), + (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>; + +def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1), + (v8i64 VR512:$src2), (i8 -1))), + (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>; //===----------------------------------------------------------------------===// // AVX-512 Shift instructions //===----------------------------------------------------------------------===// @@ -1927,23 +2338,23 @@ multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM, RegisterClass KRC> { def ri : AVX512BIi8<opc, ImmFormR, (outs RC:$dst), (ins RC:$src1, i8imm:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (vt (OpNode RC:$src1, (i8 imm:$src2))))], SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V; def rik : AVX512BIi8<opc, ImmFormR, (outs RC:$dst), (ins KRC:$mask, RC:$src1, i8imm:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), [], SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V, EVEX_K; def mi: AVX512BIi8<opc, ImmFormM, (outs RC:$dst), (ins x86memop:$src1, i8imm:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (OpNode (mem_frag addr:$src1), (i8 imm:$src2)))], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V; def mik: AVX512BIi8<opc, ImmFormM, (outs RC:$dst), (ins KRC:$mask, x86memop:$src1, i8imm:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), [], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V, EVEX_K; } @@ -1953,24 +2364,24 @@ multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode, // src2 is always 128-bit def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, VR128X:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (vt (OpNode RC:$src1, (SrcVT VR128X:$src2))))], SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V; def rrk : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src1, VR128X:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), [], SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V, EVEX_K; def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, i128mem:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (vt (OpNode RC:$src1, (bc_frag (memopv2i64 addr:$src2)))))], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V; def rmk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), (ins KRC:$mask, RC:$src1, i128mem:$src2), !strconcat(OpcodeStr, - "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), + " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"), [], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V, EVEX_K; } @@ -2024,13 +2435,13 @@ multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode, X86MemOperand x86memop, PatFrag mem_frag> { def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (vt (OpNode RC:$src1, (vt RC:$src2))))]>, EVEX_4V; def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (vt (OpNode RC:$src1, (mem_frag addr:$src2))))]>, EVEX_4V; @@ -2062,10 +2473,10 @@ defm VPSRAVQZ : avx512_var_shift<0x46, "vpsravq", sra, VR512, v8i64, multiclass avx512_movddup<string OpcodeStr, RegisterClass RC, ValueType VT, X86MemOperand x86memop, PatFrag memop_frag> { def rr : AVX512PDI<0x12, MRMSrcReg, (outs RC:$dst), (ins RC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set RC:$dst, (VT (X86Movddup RC:$src)))]>, EVEX; def rm : AVX512PDI<0x12, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set RC:$dst, (VT (X86Movddup (memop_frag addr:$src))))]>, EVEX; } @@ -2082,11 +2493,11 @@ multiclass avx512_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr, ValueType vt, RegisterClass RC, PatFrag mem_frag, X86MemOperand x86memop> { def rr : AVX512XSI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set RC:$dst, (vt (OpNode RC:$src)))]>, EVEX; let mayLoad = 1 in def rm : AVX512XSI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set RC:$dst, (OpNode (mem_frag addr:$src)))]>, EVEX; } @@ -2109,12 +2520,12 @@ def : Pat<(v16i32 (X86Movsldup (memopv16i32 addr:$src))), //===----------------------------------------------------------------------===// def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), - "vmovlhps{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))], IIC_SSE_MOV_LH>, EVEX_4V; def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), - "vmovhlps{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))], IIC_SSE_MOV_LH>, EVEX_4V; @@ -2140,18 +2551,18 @@ multiclass avx512_fma3p_rm<bits<8> opc, string OpcodeStr, string BrdcstStr, SDNode OpNode, ValueType OpVT> { def r: AVX512FMA3<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, RC:$src3), - !strconcat(OpcodeStr,"\t{$src3, $src2, $dst|$dst, $src2, $src3}"), + !strconcat(OpcodeStr," \t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpVT(OpNode RC:$src1, RC:$src2, RC:$src3)))]>; let mayLoad = 1 in def m: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src2, x86memop:$src3), - !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), + !strconcat(OpcodeStr, " \t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, (mem_frag addr:$src3))))]>; def mb: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src2, x86scalar_mop:$src3), - !strconcat(OpcodeStr, "\t{${src3}", BrdcstStr, + !strconcat(OpcodeStr, " \t{${src3}", BrdcstStr, ", $src2, $dst|$dst, $src2, ${src3}", BrdcstStr, "}"), [(set RC:$dst, (OpNode RC:$src1, RC:$src2, (OpVT (X86VBroadcast (scalar_mfrag addr:$src3)))))]>, EVEX_B; @@ -2219,11 +2630,11 @@ multiclass avx512_fma3p_m132<bits<8> opc, string OpcodeStr, let mayLoad = 1 in def m: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src3, x86memop:$src2), - !strconcat(OpcodeStr, "\t{$src2, $src3, $dst|$dst, $src3, $src2}"), + !strconcat(OpcodeStr, " \t{$src2, $src3, $dst|$dst, $src3, $src2}"), [(set RC:$dst, (OpVT (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3)))]>; def mb: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src3, x86scalar_mop:$src2), - !strconcat(OpcodeStr, "\t{${src2}", BrdcstStr, + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, ", $src3, $dst|$dst, $src3, ${src2}", BrdcstStr, "}"), [(set RC:$dst, (OpNode RC:$src1, (OpVT (X86VBroadcast (scalar_mfrag addr:$src2))), RC:$src3))]>, EVEX_B; @@ -2294,14 +2705,14 @@ multiclass avx512_fma3s_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, def r : AVX512FMA3<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, RC:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), + " \t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpVT (OpNode RC:$src2, RC:$src1, RC:$src3)))]>; let mayLoad = 1 in def m : AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src2, f128mem:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), + " \t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpVT (OpNode RC:$src2, RC:$src1, (mem_frag addr:$src3))))]>; @@ -2309,21 +2720,21 @@ multiclass avx512_fma3s_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, } // Constraints = "$src1 = $dst" -defm VFMADDSSZ : avx512_fma3s_rm<0xA9, "vfmadd213ss{z}", X86Fmadd, FR32X, +defm VFMADDSSZ : avx512_fma3s_rm<0xA9, "vfmadd213ss", X86Fmadd, FR32X, f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>; -defm VFMADDSDZ : avx512_fma3s_rm<0xA9, "vfmadd213sd{z}", X86Fmadd, FR64X, +defm VFMADDSDZ : avx512_fma3s_rm<0xA9, "vfmadd213sd", X86Fmadd, FR64X, f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>; -defm VFMSUBSSZ : avx512_fma3s_rm<0xAB, "vfmsub213ss{z}", X86Fmsub, FR32X, +defm VFMSUBSSZ : avx512_fma3s_rm<0xAB, "vfmsub213ss", X86Fmsub, FR32X, f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>; -defm VFMSUBSDZ : avx512_fma3s_rm<0xAB, "vfmsub213sd{z}", X86Fmsub, FR64X, +defm VFMSUBSDZ : avx512_fma3s_rm<0xAB, "vfmsub213sd", X86Fmsub, FR64X, f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>; -defm VFNMADDSSZ : avx512_fma3s_rm<0xAD, "vfnmadd213ss{z}", X86Fnmadd, FR32X, +defm VFNMADDSSZ : avx512_fma3s_rm<0xAD, "vfnmadd213ss", X86Fnmadd, FR32X, f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>; -defm VFNMADDSDZ : avx512_fma3s_rm<0xAD, "vfnmadd213sd{z}", X86Fnmadd, FR64X, +defm VFNMADDSDZ : avx512_fma3s_rm<0xAD, "vfnmadd213sd", X86Fnmadd, FR64X, f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>; -defm VFNMSUBSSZ : avx512_fma3s_rm<0xAF, "vfnmsub213ss{z}", X86Fnmsub, FR32X, +defm VFNMSUBSSZ : avx512_fma3s_rm<0xAF, "vfnmsub213ss", X86Fnmsub, FR32X, f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>; -defm VFNMSUBSDZ : avx512_fma3s_rm<0xAF, "vfnmsub213sd{z}", X86Fnmsub, FR64X, +defm VFNMSUBSDZ : avx512_fma3s_rm<0xAF, "vfnmsub213sd", X86Fnmsub, FR64X, f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>; //===----------------------------------------------------------------------===// @@ -2332,25 +2743,25 @@ defm VFNMSUBSDZ : avx512_fma3s_rm<0xAF, "vfnmsub213sd{z}", X86Fnmsub, FR64X, multiclass avx512_vcvtsi<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, X86MemOperand x86memop, string asm> { -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src), - !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, + !strconcat(asm," \t{$src, $src1, $dst|$dst, $src1, $src}"), []>, EVEX_4V; let mayLoad = 1 in def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins DstRC:$src1, x86memop:$src), - !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, + !strconcat(asm," \t{$src, $src1, $dst|$dst, $src1, $src}"), []>, EVEX_4V; -} // neverHasSideEffects = 1 +} // hasSideEffects = 0 } let Predicates = [HasAVX512] in { -defm VCVTSI2SSZ : avx512_vcvtsi<0x2A, GR32, FR32X, i32mem, "cvtsi2ss{l}{z}">, +defm VCVTSI2SSZ : avx512_vcvtsi<0x2A, GR32, FR32X, i32mem, "cvtsi2ss{l}">, XS, VEX_LIG, EVEX_CD8<32, CD8VT1>; -defm VCVTSI642SSZ : avx512_vcvtsi<0x2A, GR64, FR32X, i64mem, "cvtsi2ss{q}{z}">, +defm VCVTSI642SSZ : avx512_vcvtsi<0x2A, GR64, FR32X, i64mem, "cvtsi2ss{q}">, XS, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>; -defm VCVTSI2SDZ : avx512_vcvtsi<0x2A, GR32, FR64X, i32mem, "cvtsi2sd{l}{z}">, +defm VCVTSI2SDZ : avx512_vcvtsi<0x2A, GR32, FR64X, i32mem, "cvtsi2sd{l}">, XD, VEX_LIG, EVEX_CD8<32, CD8VT1>; -defm VCVTSI642SDZ : avx512_vcvtsi<0x2A, GR64, FR64X, i64mem, "cvtsi2sd{q}{z}">, +defm VCVTSI642SDZ : avx512_vcvtsi<0x2A, GR64, FR64X, i64mem, "cvtsi2sd{q}">, XD, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>; def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))), @@ -2371,13 +2782,13 @@ def : Pat<(f64 (sint_to_fp GR32:$src)), def : Pat<(f64 (sint_to_fp GR64:$src)), (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; -defm VCVTUSI2SSZ : avx512_vcvtsi<0x7B, GR32, FR32X, i32mem, "cvtusi2ss{l}{z}">, +defm VCVTUSI2SSZ : avx512_vcvtsi<0x7B, GR32, FR32X, i32mem, "cvtusi2ss{l}">, XS, VEX_LIG, EVEX_CD8<32, CD8VT1>; -defm VCVTUSI642SSZ : avx512_vcvtsi<0x7B, GR64, FR32X, i64mem, "cvtusi2ss{q}{z}">, +defm VCVTUSI642SSZ : avx512_vcvtsi<0x7B, GR64, FR32X, i64mem, "cvtusi2ss{q}">, XS, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>; -defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, GR32, FR64X, i32mem, "cvtusi2sd{l}{z}">, +defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, GR32, FR64X, i32mem, "cvtusi2sd{l}">, XD, VEX_LIG, EVEX_CD8<32, CD8VT1>; -defm VCVTUSI642SDZ : avx512_vcvtsi<0x7B, GR64, FR64X, i64mem, "cvtusi2sd{q}{z}">, +defm VCVTUSI642SDZ : avx512_vcvtsi<0x7B, GR64, FR64X, i64mem, "cvtusi2sd{q}">, XD, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>; def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))), @@ -2405,161 +2816,167 @@ def : Pat<(f64 (uint_to_fp GR64:$src)), multiclass avx512_cvt_s_int<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, Intrinsic Int, Operand memop, ComplexPattern mem_cpat, string asm> { -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), - !strconcat(asm,"\t{$src, $dst|$dst, $src}"), - [(set DstRC:$dst, (Int SrcRC:$src))]>, EVEX, VEX_LIG; + !strconcat(asm," \t{$src, $dst|$dst, $src}"), + [(set DstRC:$dst, (Int SrcRC:$src))]>, EVEX, VEX_LIG, + Requires<[HasAVX512]>; let mayLoad = 1 in def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src), - !strconcat(asm,"\t{$src, $dst|$dst, $src}"), []>, EVEX, VEX_LIG; -} // neverHasSideEffects = 1 + !strconcat(asm," \t{$src, $dst|$dst, $src}"), []>, EVEX, VEX_LIG, + Requires<[HasAVX512]>; +} // hasSideEffects = 0 } let Predicates = [HasAVX512] in { // Convert float/double to signed/unsigned int 32/64 defm VCVTSS2SIZ: avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse_cvtss2si, - ssmem, sse_load_f32, "cvtss2si{z}">, + ssmem, sse_load_f32, "cvtss2si">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTSS2SI64Z: avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse_cvtss2si64, - ssmem, sse_load_f32, "cvtss2si{z}">, + ssmem, sse_load_f32, "cvtss2si">, XS, VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTSS2USIZ: avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtss2usi, - ssmem, sse_load_f32, "cvtss2usi{z}">, + ssmem, sse_load_f32, "cvtss2usi">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTSS2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64, int_x86_avx512_cvtss2usi64, ssmem, - sse_load_f32, "cvtss2usi{z}">, XS, VEX_W, + sse_load_f32, "cvtss2usi">, XS, VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTSD2SIZ: avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse2_cvtsd2si, - sdmem, sse_load_f64, "cvtsd2si{z}">, + sdmem, sse_load_f64, "cvtsd2si">, XD, EVEX_CD8<64, CD8VT1>; defm VCVTSD2SI64Z: avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse2_cvtsd2si64, - sdmem, sse_load_f64, "cvtsd2si{z}">, + sdmem, sse_load_f64, "cvtsd2si">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; defm VCVTSD2USIZ: avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtsd2usi, - sdmem, sse_load_f64, "cvtsd2usi{z}">, + sdmem, sse_load_f64, "cvtsd2usi">, XD, EVEX_CD8<64, CD8VT1>; defm VCVTSD2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64, int_x86_avx512_cvtsd2usi64, sdmem, - sse_load_f64, "cvtsd2usi{z}">, XD, VEX_W, + sse_load_f64, "cvtsd2usi">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; -defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, - int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}{z}", - SSE_CVT_Scalar, 0>, XS, EVEX_4V; -defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, - int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}{z}", - SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; -defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, - int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}{z}", - SSE_CVT_Scalar, 0>, XD, EVEX_4V; -defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, - int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}{z}", - SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; - -defm Int_VCVTUSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, - int_x86_avx512_cvtusi2ss, i32mem, loadi32, "cvtusi2ss{l}{z}", - SSE_CVT_Scalar, 0>, XS, EVEX_4V; -defm Int_VCVTUSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, - int_x86_avx512_cvtusi642ss, i64mem, loadi64, "cvtusi2ss{q}{z}", - SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; -defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, - int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}{z}", - SSE_CVT_Scalar, 0>, XD, EVEX_4V; -defm Int_VCVTUSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, - int_x86_avx512_cvtusi642sd, i64mem, loadi64, "cvtusi2sd{q}{z}", - SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; +let isCodeGenOnly = 1 in { + defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, + int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", + SSE_CVT_Scalar, 0>, XS, EVEX_4V; + defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, + int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", + SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; + defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, + int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", + SSE_CVT_Scalar, 0>, XD, EVEX_4V; + defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, + int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", + SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; + + defm Int_VCVTUSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, + int_x86_avx512_cvtusi2ss, i32mem, loadi32, "cvtusi2ss{l}", + SSE_CVT_Scalar, 0>, XS, EVEX_4V; + defm Int_VCVTUSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, + int_x86_avx512_cvtusi642ss, i64mem, loadi64, "cvtusi2ss{q}", + SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; + defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, + int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}", + SSE_CVT_Scalar, 0>, XD, EVEX_4V; + defm Int_VCVTUSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, + int_x86_avx512_cvtusi642sd, i64mem, loadi64, "cvtusi2sd{q}", + SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; +} // isCodeGenOnly = 1 // Convert float/double to signed/unsigned int 32/64 with truncation -defm Int_VCVTTSS2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse_cvttss2si, - ssmem, sse_load_f32, "cvttss2si{z}">, - XS, EVEX_CD8<32, CD8VT1>; -defm Int_VCVTTSS2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, - int_x86_sse_cvttss2si64, ssmem, sse_load_f32, - "cvttss2si{z}">, XS, VEX_W, - EVEX_CD8<32, CD8VT1>; -defm Int_VCVTTSD2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse2_cvttsd2si, - sdmem, sse_load_f64, "cvttsd2si{z}">, XD, - EVEX_CD8<64, CD8VT1>; -defm Int_VCVTTSD2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, - int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, - "cvttsd2si{z}">, XD, VEX_W, - EVEX_CD8<64, CD8VT1>; -defm Int_VCVTTSS2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, - int_x86_avx512_cvttss2usi, ssmem, sse_load_f32, - "cvttss2si{z}">, XS, EVEX_CD8<32, CD8VT1>; -defm Int_VCVTTSS2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, - int_x86_avx512_cvttss2usi64, ssmem, - sse_load_f32, "cvttss2usi{z}">, XS, VEX_W, - EVEX_CD8<32, CD8VT1>; -defm Int_VCVTTSD2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, - int_x86_avx512_cvttsd2usi, - sdmem, sse_load_f64, "cvttsd2usi{z}">, XD, - EVEX_CD8<64, CD8VT1>; -defm Int_VCVTTSD2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, - int_x86_avx512_cvttsd2usi64, sdmem, - sse_load_f64, "cvttsd2usi{z}">, XD, VEX_W, - EVEX_CD8<64, CD8VT1>; -} +let isCodeGenOnly = 1 in { + defm Int_VCVTTSS2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse_cvttss2si, + ssmem, sse_load_f32, "cvttss2si">, + XS, EVEX_CD8<32, CD8VT1>; + defm Int_VCVTTSS2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, + int_x86_sse_cvttss2si64, ssmem, sse_load_f32, + "cvttss2si">, XS, VEX_W, + EVEX_CD8<32, CD8VT1>; + defm Int_VCVTTSD2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse2_cvttsd2si, + sdmem, sse_load_f64, "cvttsd2si">, XD, + EVEX_CD8<64, CD8VT1>; + defm Int_VCVTTSD2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, + int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, + "cvttsd2si">, XD, VEX_W, + EVEX_CD8<64, CD8VT1>; + defm Int_VCVTTSS2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, + int_x86_avx512_cvttss2usi, ssmem, sse_load_f32, + "cvttss2usi">, XS, EVEX_CD8<32, CD8VT1>; + defm Int_VCVTTSS2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, + int_x86_avx512_cvttss2usi64, ssmem, + sse_load_f32, "cvttss2usi">, XS, VEX_W, + EVEX_CD8<32, CD8VT1>; + defm Int_VCVTTSD2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, + int_x86_avx512_cvttsd2usi, + sdmem, sse_load_f64, "cvttsd2usi">, XD, + EVEX_CD8<64, CD8VT1>; + defm Int_VCVTTSD2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, + int_x86_avx512_cvttsd2usi64, sdmem, + sse_load_f64, "cvttsd2usi">, XD, VEX_W, + EVEX_CD8<64, CD8VT1>; +} // isCodeGenOnly = 1 multiclass avx512_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag, string asm> { def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), - !strconcat(asm,"\t{$src, $dst|$dst, $src}"), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpNode SrcRC:$src))]>, EVEX; def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), - !strconcat(asm,"\t{$src, $dst|$dst, $src}"), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))]>, EVEX; } defm VCVTTSS2SIZ : avx512_cvt_s<0x2C, FR32X, GR32, fp_to_sint, f32mem, - loadf32, "cvttss2si{z}">, XS, + loadf32, "cvttss2si">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTTSS2USIZ : avx512_cvt_s<0x78, FR32X, GR32, fp_to_uint, f32mem, - loadf32, "cvttss2usi{z}">, XS, + loadf32, "cvttss2usi">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTTSS2SI64Z : avx512_cvt_s<0x2C, FR32X, GR64, fp_to_sint, f32mem, - loadf32, "cvttss2si{z}">, XS, VEX_W, + loadf32, "cvttss2si">, XS, VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTTSS2USI64Z : avx512_cvt_s<0x78, FR32X, GR64, fp_to_uint, f32mem, - loadf32, "cvttss2usi{z}">, XS, VEX_W, + loadf32, "cvttss2usi">, XS, VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTTSD2SIZ : avx512_cvt_s<0x2C, FR64X, GR32, fp_to_sint, f64mem, - loadf64, "cvttsd2si{z}">, XD, + loadf64, "cvttsd2si">, XD, EVEX_CD8<64, CD8VT1>; defm VCVTTSD2USIZ : avx512_cvt_s<0x78, FR64X, GR32, fp_to_uint, f64mem, - loadf64, "cvttsd2usi{z}">, XD, + loadf64, "cvttsd2usi">, XD, EVEX_CD8<64, CD8VT1>; defm VCVTTSD2SI64Z : avx512_cvt_s<0x2C, FR64X, GR64, fp_to_sint, f64mem, - loadf64, "cvttsd2si{z}">, XD, VEX_W, + loadf64, "cvttsd2si">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; defm VCVTTSD2USI64Z : avx512_cvt_s<0x78, FR64X, GR64, fp_to_uint, f64mem, - loadf64, "cvttsd2usi{z}">, XD, VEX_W, + loadf64, "cvttsd2usi">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; +} // HasAVX512 //===----------------------------------------------------------------------===// // AVX-512 Convert form float to double and back //===----------------------------------------------------------------------===// -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { def VCVTSS2SDZrr : AVX512XSI<0x5A, MRMSrcReg, (outs FR64X:$dst), (ins FR32X:$src1, FR32X:$src2), - "vcvtss2sd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>; let mayLoad = 1 in def VCVTSS2SDZrm : AVX512XSI<0x5A, MRMSrcMem, (outs FR64X:$dst), (ins FR32X:$src1, f32mem:$src2), - "vcvtss2sd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>, EVEX_CD8<32, CD8VT1>; // Convert scalar double to scalar single def VCVTSD2SSZrr : AVX512XDI<0x5A, MRMSrcReg, (outs FR32X:$dst), (ins FR64X:$src1, FR64X:$src2), - "vcvtsd2ss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX_4V, VEX_LIG, VEX_W, Sched<[WriteCvtF2F]>; let mayLoad = 1 in def VCVTSD2SSZrm : AVX512XDI<0x5A, MRMSrcMem, (outs FR32X:$dst), (ins FR64X:$src1, f64mem:$src2), - "vcvtsd2ss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX_4V, VEX_LIG, VEX_W, Sched<[WriteCvtF2FLd, ReadAfterLd]>, EVEX_CD8<64, CD8VT1>; } @@ -2580,41 +2997,71 @@ def : Pat<(extloadf32 addr:$src), def : Pat<(f32 (fround FR64X:$src)), (VCVTSD2SSZrr FR64X:$src, FR64X:$src)>, Requires<[HasAVX512]>; -multiclass avx512_vcvt_fp<bits<8> opc, string asm, RegisterClass SrcRC, +multiclass avx512_vcvt_fp_with_rc<bits<8> opc, string asm, RegisterClass SrcRC, RegisterClass DstRC, SDNode OpNode, PatFrag mem_frag, X86MemOperand x86memop, ValueType OpVT, ValueType InVT, Domain d> { -let neverHasSideEffects = 1 in { +let hasSideEffects = 0 in { def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), - !strconcat(asm,"\t{$src, $dst|$dst, $src}"), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpVT (OpNode (InVT SrcRC:$src))))], d>, EVEX; + def rrb : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc), + !strconcat(asm," \t{$rc, $src, $dst|$dst, $src, $rc}"), + [], d>, EVEX, EVEX_B, EVEX_RC; let mayLoad = 1 in def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), - !strconcat(asm,"\t{$src, $dst|$dst, $src}"), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))], d>, EVEX; -} // neverHasSideEffects = 1 +} // hasSideEffects = 0 } -defm VCVTPD2PSZ : avx512_vcvt_fp<0x5A, "vcvtpd2ps", VR512, VR256X, fround, +multiclass avx512_vcvt_fp<bits<8> opc, string asm, RegisterClass SrcRC, + RegisterClass DstRC, SDNode OpNode, PatFrag mem_frag, + X86MemOperand x86memop, ValueType OpVT, ValueType InVT, + Domain d> { +let hasSideEffects = 0 in { + def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), + [(set DstRC:$dst, + (OpVT (OpNode (InVT SrcRC:$src))))], d>, EVEX; + let mayLoad = 1 in + def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), + [(set DstRC:$dst, + (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))], d>, EVEX; +} // hasSideEffects = 0 +} + +defm VCVTPD2PSZ : avx512_vcvt_fp_with_rc<0x5A, "vcvtpd2ps", VR512, VR256X, fround, memopv8f64, f512mem, v8f32, v8f64, - SSEPackedSingle>, EVEX_V512, VEX_W, OpSize, + SSEPackedSingle>, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm VCVTPS2PDZ : avx512_vcvt_fp<0x5A, "vcvtps2pd", VR256X, VR512, fextend, memopv4f64, f256mem, v8f64, v8f32, - SSEPackedDouble>, EVEX_V512, EVEX_CD8<32, CD8VH>; + SSEPackedDouble>, EVEX_V512, PS, + EVEX_CD8<32, CD8VH>; def : Pat<(v8f64 (extloadv8f32 addr:$src)), (VCVTPS2PDZrm addr:$src)>; + +def : Pat<(v8f32 (int_x86_avx512_mask_cvtpd2ps_512 (v8f64 VR512:$src), + (bc_v8f32(v8i32 immAllZerosV)), (i8 -1), (i32 FROUND_CURRENT))), + (VCVTPD2PSZrr VR512:$src)>; + +def : Pat<(v8f32 (int_x86_avx512_mask_cvtpd2ps_512 (v8f64 VR512:$src), + (bc_v8f32(v8i32 immAllZerosV)), (i8 -1), imm:$rc)), + (VCVTPD2PSZrrb VR512:$src, imm:$rc)>; //===----------------------------------------------------------------------===// // AVX-512 Vector convert from sign integer to float/double //===----------------------------------------------------------------------===// -defm VCVTDQ2PSZ : avx512_vcvt_fp<0x5B, "vcvtdq2ps", VR512, VR512, sint_to_fp, +defm VCVTDQ2PSZ : avx512_vcvt_fp_with_rc<0x5B, "vcvtdq2ps", VR512, VR512, sint_to_fp, memopv8i64, i512mem, v16f32, v16i32, - SSEPackedSingle>, EVEX_V512, EVEX_CD8<32, CD8VF>; + SSEPackedSingle>, EVEX_V512, PS, + EVEX_CD8<32, CD8VF>; defm VCVTDQ2PDZ : avx512_vcvt_fp<0xE6, "vcvtdq2pd", VR256X, VR512, sint_to_fp, memopv4i64, i256mem, v8f64, v8i32, @@ -2628,25 +3075,35 @@ defm VCVTTPS2DQZ : avx512_vcvt_fp<0x5B, "vcvttps2dq", VR512, VR512, fp_to_sint, defm VCVTTPD2DQZ : avx512_vcvt_fp<0xE6, "vcvttpd2dq", VR512, VR256X, fp_to_sint, memopv8f64, f512mem, v8i32, v8f64, - SSEPackedDouble>, EVEX_V512, OpSize, VEX_W, + SSEPackedDouble>, EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; defm VCVTTPS2UDQZ : avx512_vcvt_fp<0x78, "vcvttps2udq", VR512, VR512, fp_to_uint, memopv16f32, f512mem, v16i32, v16f32, - SSEPackedSingle>, EVEX_V512, + SSEPackedSingle>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; +// cvttps2udq (src, 0, mask-all-ones, sae-current) +def : Pat<(v16i32 (int_x86_avx512_mask_cvttps2udq_512 (v16f32 VR512:$src), + (v16i32 immAllZerosV), (i16 -1), FROUND_CURRENT)), + (VCVTTPS2UDQZrr VR512:$src)>; + defm VCVTTPD2UDQZ : avx512_vcvt_fp<0x78, "vcvttpd2udq", VR512, VR256X, fp_to_uint, memopv8f64, f512mem, v8i32, v8f64, - SSEPackedDouble>, EVEX_V512, VEX_W, + SSEPackedDouble>, EVEX_V512, PS, VEX_W, EVEX_CD8<64, CD8VF>; +// cvttpd2udq (src, 0, mask-all-ones, sae-current) +def : Pat<(v8i32 (int_x86_avx512_mask_cvttpd2udq_512 (v8f64 VR512:$src), + (v8i32 immAllZerosV), (i8 -1), FROUND_CURRENT)), + (VCVTTPD2UDQZrr VR512:$src)>; + defm VCVTUDQ2PDZ : avx512_vcvt_fp<0x7A, "vcvtudq2pd", VR256X, VR512, uint_to_fp, memopv4i64, f256mem, v8f64, v8i32, SSEPackedDouble>, EVEX_V512, XS, EVEX_CD8<32, CD8VH>; -defm VCVTUDQ2PSZ : avx512_vcvt_fp<0x7A, "vcvtudq2ps", VR512, VR512, uint_to_fp, +defm VCVTUDQ2PSZ : avx512_vcvt_fp_with_rc<0x7A, "vcvtudq2ps", VR512, VR512, uint_to_fp, memopv16i32, f512mem, v16f32, v16i32, SSEPackedSingle>, EVEX_V512, XD, EVEX_CD8<32, CD8VF>; @@ -2656,22 +3113,65 @@ def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))), (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; -def : Pat<(int_x86_avx512_cvtdq2_ps_512 VR512:$src), - (VCVTDQ2PSZrr VR512:$src)>; -def : Pat<(int_x86_avx512_cvtdq2_ps_512 (bitconvert (memopv8i64 addr:$src))), - (VCVTDQ2PSZrm addr:$src)>; +def : Pat<(v16f32 (int_x86_avx512_mask_cvtdq2ps_512 (v16i32 VR512:$src), + (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), imm:$rc)), + (VCVTDQ2PSZrrb VR512:$src, imm:$rc)>; +def : Pat<(v8f64 (int_x86_avx512_mask_cvtdq2pd_512 (v8i32 VR256X:$src), + (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), + (VCVTDQ2PDZrr VR256X:$src)>; +def : Pat<(v16f32 (int_x86_avx512_mask_cvtudq2ps_512 (v16i32 VR512:$src), + (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), imm:$rc)), + (VCVTUDQ2PSZrrb VR512:$src, imm:$rc)>; +def : Pat<(v8f64 (int_x86_avx512_mask_cvtudq2pd_512 (v8i32 VR256X:$src), + (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), + (VCVTUDQ2PDZrr VR256X:$src)>; + +multiclass avx512_vcvt_fp2int<bits<8> opc, string asm, RegisterClass SrcRC, + RegisterClass DstRC, PatFrag mem_frag, + X86MemOperand x86memop, Domain d> { +let hasSideEffects = 0 in { + def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), + [], d>, EVEX; + def rrb : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc), + !strconcat(asm," \t{$rc, $src, $dst|$dst, $src, $rc}"), + [], d>, EVEX, EVEX_B, EVEX_RC; + let mayLoad = 1 in + def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), + !strconcat(asm," \t{$src, $dst|$dst, $src}"), + [], d>, EVEX; +} // hasSideEffects = 0 +} + +defm VCVTPS2DQZ : avx512_vcvt_fp2int<0x5B, "vcvtps2dq", VR512, VR512, + memopv16f32, f512mem, SSEPackedSingle>, PD, + EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VCVTPD2DQZ : avx512_vcvt_fp2int<0xE6, "vcvtpd2dq", VR512, VR256X, + memopv8f64, f512mem, SSEPackedDouble>, XD, VEX_W, + EVEX_V512, EVEX_CD8<64, CD8VF>; + +def : Pat <(v16i32 (int_x86_avx512_mask_cvtps2dq_512 (v16f32 VR512:$src), + (v16i32 immAllZerosV), (i16 -1), imm:$rc)), + (VCVTPS2DQZrrb VR512:$src, imm:$rc)>; + +def : Pat <(v8i32 (int_x86_avx512_mask_cvtpd2dq_512 (v8f64 VR512:$src), + (v8i32 immAllZerosV), (i8 -1), imm:$rc)), + (VCVTPD2DQZrrb VR512:$src, imm:$rc)>; -def VCVTPS2DQZrr : AVX512BI<0x5B, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - "vcvtps2dq\t{$src, $dst|$dst, $src}", - [(set VR512:$dst, - (int_x86_avx512_cvt_ps2dq_512 VR512:$src))], - IIC_SSE_CVT_PS_RR>, EVEX, EVEX_V512; -def VCVTPS2DQZrm : AVX512BI<0x5B, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src), - "vcvtps2dq\t{$src, $dst|$dst, $src}", - [(set VR512:$dst, - (int_x86_avx512_cvt_ps2dq_512 (memopv16f32 addr:$src)))], - IIC_SSE_CVT_PS_RM>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VCVTPS2UDQZ : avx512_vcvt_fp2int<0x79, "vcvtps2udq", VR512, VR512, + memopv16f32, f512mem, SSEPackedSingle>, + PS, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VCVTPD2UDQZ : avx512_vcvt_fp2int<0x79, "vcvtpd2udq", VR512, VR256X, + memopv8f64, f512mem, SSEPackedDouble>, VEX_W, + PS, EVEX_V512, EVEX_CD8<64, CD8VF>; +def : Pat <(v16i32 (int_x86_avx512_mask_cvtps2udq_512 (v16f32 VR512:$src), + (v16i32 immAllZerosV), (i16 -1), imm:$rc)), + (VCVTPS2UDQZrrb VR512:$src, imm:$rc)>; + +def : Pat <(v8i32 (int_x86_avx512_mask_cvtpd2udq_512 (v8f64 VR512:$src), + (v8i32 immAllZerosV), (i8 -1), imm:$rc)), + (VCVTPD2UDQZrrb VR512:$src, imm:$rc)>; let Predicates = [HasAVX512] in { def : Pat<(v8f32 (fround (loadv8f64 addr:$src))), @@ -2683,234 +3183,279 @@ let Predicates = [HasAVX512] in { //===----------------------------------------------------------------------===// // Half precision conversion instructions //===----------------------------------------------------------------------===// -multiclass avx512_f16c_ph2ps<RegisterClass destRC, RegisterClass srcRC, - X86MemOperand x86memop, Intrinsic Int> { +multiclass avx512_cvtph2ps<RegisterClass destRC, RegisterClass srcRC, + X86MemOperand x86memop> { def rr : AVX5128I<0x13, MRMSrcReg, (outs destRC:$dst), (ins srcRC:$src), "vcvtph2ps\t{$src, $dst|$dst, $src}", - [(set destRC:$dst, (Int srcRC:$src))]>, EVEX; - let neverHasSideEffects = 1, mayLoad = 1 in + []>, EVEX; + let hasSideEffects = 0, mayLoad = 1 in def rm : AVX5128I<0x13, MRMSrcMem, (outs destRC:$dst), (ins x86memop:$src), "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, EVEX; } -multiclass avx512_f16c_ps2ph<RegisterClass destRC, RegisterClass srcRC, - X86MemOperand x86memop, Intrinsic Int> { +multiclass avx512_cvtps2ph<RegisterClass destRC, RegisterClass srcRC, + X86MemOperand x86memop> { def rr : AVX512AIi8<0x1D, MRMDestReg, (outs destRC:$dst), (ins srcRC:$src1, i32i8imm:$src2), - "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set destRC:$dst, (Int srcRC:$src1, imm:$src2))]>, EVEX; - let neverHasSideEffects = 1, mayStore = 1 in + "vcvtps2ph \t{$src2, $src1, $dst|$dst, $src1, $src2}", + []>, EVEX; + let hasSideEffects = 0, mayStore = 1 in def mr : AVX512AIi8<0x1D, MRMDestMem, (outs), (ins x86memop:$dst, srcRC:$src1, i32i8imm:$src2), - "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX; + "vcvtps2ph \t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX; } -defm VCVTPH2PSZ : avx512_f16c_ph2ps<VR512, VR256X, f256mem, - int_x86_avx512_vcvtph2ps_512>, EVEX_V512, +defm VCVTPH2PSZ : avx512_cvtph2ps<VR512, VR256X, f256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>; -defm VCVTPS2PHZ : avx512_f16c_ps2ph<VR256X, VR512, f256mem, - int_x86_avx512_vcvtps2ph_512>, EVEX_V512, +defm VCVTPS2PHZ : avx512_cvtps2ph<VR256X, VR512, f256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>; +def : Pat<(v16i16 (int_x86_avx512_mask_vcvtps2ph_512 (v16f32 VR512:$src), + imm:$rc, (bc_v16i16(v8i32 immAllZerosV)), (i16 -1))), + (VCVTPS2PHZrr VR512:$src, imm:$rc)>; + +def : Pat<(v16f32 (int_x86_avx512_mask_vcvtph2ps_512 (v16i16 VR256X:$src), + (bc_v16f32(v16i32 immAllZerosV)), (i16 -1), (i32 FROUND_CURRENT))), + (VCVTPH2PSZrr VR256X:$src)>; + let Defs = [EFLAGS], Predicates = [HasAVX512] in { defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32, - "ucomiss{z}">, TB, EVEX, VEX_LIG, + "ucomiss">, PS, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64, - "ucomisd{z}">, TB, OpSize, EVEX, + "ucomisd">, PD, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; let Pattern = []<dag> in { defm VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, undef, v4f32, f128mem, load, - "comiss{z}">, TB, EVEX, VEX_LIG, + "comiss">, PS, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, undef, v2f64, f128mem, load, - "comisd{z}">, TB, OpSize, EVEX, + "comisd">, PD, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; } - defm Int_VUCOMISSZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem, - load, "ucomiss">, TB, EVEX, VEX_LIG, - EVEX_CD8<32, CD8VT1>; - defm Int_VUCOMISDZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem, - load, "ucomisd">, TB, OpSize, EVEX, - VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; - - defm Int_VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem, - load, "comiss">, TB, EVEX, VEX_LIG, - EVEX_CD8<32, CD8VT1>; - defm Int_VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem, - load, "comisd">, TB, OpSize, EVEX, - VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; + let isCodeGenOnly = 1 in { + defm Int_VUCOMISSZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem, + load, "ucomiss">, PS, EVEX, VEX_LIG, + EVEX_CD8<32, CD8VT1>; + defm Int_VUCOMISDZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem, + load, "ucomisd">, PD, EVEX, + VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; + + defm Int_VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem, + load, "comiss">, PS, EVEX, VEX_LIG, + EVEX_CD8<32, CD8VT1>; + defm Int_VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem, + load, "comisd">, PD, EVEX, + VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; + } } -/// avx512_unop_p - AVX-512 unops in packed form. -multiclass avx512_fp_unop_p<bits<8> opc, string OpcodeStr, SDNode OpNode> { - def PSZr : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - !strconcat(OpcodeStr, - "ps\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, (v16f32 (OpNode VR512:$src)))]>, - EVEX, EVEX_V512; - def PSZm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins f256mem:$src), - !strconcat(OpcodeStr, - "ps\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, (OpNode (memopv16f32 addr:$src)))]>, - EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; - def PDZr : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - !strconcat(OpcodeStr, - "pd\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, (v8f64 (OpNode VR512:$src)))]>, - EVEX, EVEX_V512, VEX_W; - def PDZm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src), - !strconcat(OpcodeStr, - "pd\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, (OpNode (memopv16f32 addr:$src)))]>, - EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -} - -/// avx512_fp_unop_p_int - AVX-512 intrinsics unops in packed forms. -multiclass avx512_fp_unop_p_int<bits<8> opc, string OpcodeStr, - Intrinsic V16F32Int, Intrinsic V8F64Int> { - def PSZr_Int : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - !strconcat(OpcodeStr, - "ps\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, (V16F32Int VR512:$src))]>, - EVEX, EVEX_V512; - def PSZm_Int : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src), - !strconcat(OpcodeStr, - "ps\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, - (V16F32Int (memopv16f32 addr:$src)))]>, EVEX, - EVEX_V512, EVEX_CD8<32, CD8VF>; - def PDZr_Int : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - !strconcat(OpcodeStr, - "pd\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, (V8F64Int VR512:$src))]>, - EVEX, EVEX_V512, VEX_W; - def PDZm_Int : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src), - !strconcat(OpcodeStr, - "pd\t{$src, $dst|$dst, $src}"), - [(set VR512:$dst, - (V8F64Int (memopv8f64 addr:$src)))]>, - EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; -} - -/// avx512_fp_unop_s - AVX-512 unops in scalar form. -multiclass avx512_fp_unop_s<bits<8> opc, string OpcodeStr> { +/// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd +multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, RegisterClass RC, + X86MemOperand x86memop> { let hasSideEffects = 0 in { - def SSZr : AVX5128I<opc, MRMSrcReg, (outs FR32X:$dst), - (ins FR32X:$src1, FR32X:$src2), + def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), + (ins RC:$src1, RC:$src2), !strconcat(OpcodeStr, - "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, EVEX_4V; + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; let mayLoad = 1 in { - def SSZm : AVX5128I<opc, MRMSrcMem, (outs FR32X:$dst), - (ins FR32X:$src1, f32mem:$src2), + def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src1, x86memop:$src2), !strconcat(OpcodeStr, - "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, EVEX_4V, EVEX_CD8<32, CD8VT1>; - def SSZm_Int : AVX5128I<opc, MRMSrcMem, (outs VR128X:$dst), - (ins VR128X:$src1, ssmem:$src2), - !strconcat(OpcodeStr, - "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, EVEX_4V, EVEX_CD8<32, CD8VT1>; + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; } - def SDZr : AVX5128I<opc, MRMSrcReg, (outs FR64X:$dst), - (ins FR64X:$src1, FR64X:$src2), +} +} + +defm VRCP14SS : avx512_fp14_s<0x4D, "vrcp14ss", FR32X, f32mem>, + EVEX_CD8<32, CD8VT1>; +defm VRCP14SD : avx512_fp14_s<0x4D, "vrcp14sd", FR64X, f64mem>, + VEX_W, EVEX_CD8<64, CD8VT1>; +defm VRSQRT14SS : avx512_fp14_s<0x4F, "vrsqrt14ss", FR32X, f32mem>, + EVEX_CD8<32, CD8VT1>; +defm VRSQRT14SD : avx512_fp14_s<0x4F, "vrsqrt14sd", FR64X, f64mem>, + VEX_W, EVEX_CD8<64, CD8VT1>; + +def : Pat <(v4f32 (int_x86_avx512_rcp14_ss (v4f32 VR128X:$src1), + (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))), + (COPY_TO_REGCLASS (VRCP14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X), + (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; + +def : Pat <(v2f64 (int_x86_avx512_rcp14_sd (v2f64 VR128X:$src1), + (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))), + (COPY_TO_REGCLASS (VRCP14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X), + (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; + +def : Pat <(v4f32 (int_x86_avx512_rsqrt14_ss (v4f32 VR128X:$src1), + (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))), + (COPY_TO_REGCLASS (VRSQRT14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X), + (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; + +def : Pat <(v2f64 (int_x86_avx512_rsqrt14_sd (v2f64 VR128X:$src1), + (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))), + (COPY_TO_REGCLASS (VRSQRT14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X), + (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; + +/// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd +multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode, + RegisterClass RC, X86MemOperand x86memop, + PatFrag mem_frag, ValueType OpVt> { + def r : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), + !strconcat(OpcodeStr, + " \t{$src, $dst|$dst, $src}"), + [(set RC:$dst, (OpVt (OpNode RC:$src)))]>, + EVEX; + def m : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), + [(set RC:$dst, (OpVt (OpNode (mem_frag addr:$src))))]>, + EVEX; +} +defm VRSQRT14PSZ : avx512_fp14_p<0x4E, "vrsqrt14ps", X86frsqrt, VR512, f512mem, + memopv16f32, v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VRSQRT14PDZ : avx512_fp14_p<0x4E, "vrsqrt14pd", X86frsqrt, VR512, f512mem, + memopv8f64, v8f64>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; +defm VRCP14PSZ : avx512_fp14_p<0x4C, "vrcp14ps", X86frcp, VR512, f512mem, + memopv16f32, v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VRCP14PDZ : avx512_fp14_p<0x4C, "vrcp14pd", X86frcp, VR512, f512mem, + memopv8f64, v8f64>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; + +def : Pat <(v16f32 (int_x86_avx512_rsqrt14_ps_512 (v16f32 VR512:$src), + (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), + (VRSQRT14PSZr VR512:$src)>; +def : Pat <(v8f64 (int_x86_avx512_rsqrt14_pd_512 (v8f64 VR512:$src), + (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), + (VRSQRT14PDZr VR512:$src)>; + +def : Pat <(v16f32 (int_x86_avx512_rcp14_ps_512 (v16f32 VR512:$src), + (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), + (VRCP14PSZr VR512:$src)>; +def : Pat <(v8f64 (int_x86_avx512_rcp14_pd_512 (v8f64 VR512:$src), + (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), + (VRCP14PDZr VR512:$src)>; + +/// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd +multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr, RegisterClass RC, + X86MemOperand x86memop> { + let hasSideEffects = 0, Predicates = [HasERI] in { + def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), + (ins RC:$src1, RC:$src2), !strconcat(OpcodeStr, - "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, - EVEX_4V, VEX_W; + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; + def rrb : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), + (ins RC:$src1, RC:$src2), + !strconcat(OpcodeStr, + " \t{{sae}, $src2, $src1, $dst|$dst, $src1, $src2, {sae}}"), + []>, EVEX_4V, EVEX_B; let mayLoad = 1 in { - def SDZm : AVX5128I<opc, MRMSrcMem, (outs FR64X:$dst), - (ins FR64X:$src1, f64mem:$src2), + def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), + (ins RC:$src1, x86memop:$src2), !strconcat(OpcodeStr, - "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, - EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; - def SDZm_Int : AVX5128I<opc, MRMSrcMem, (outs VR128X:$dst), - (ins VR128X:$src1, sdmem:$src2), - !strconcat(OpcodeStr, - "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; } } } -defm VRCP14 : avx512_fp_unop_s<0x4D, "vrcp14">, - avx512_fp_unop_p<0x4C, "vrcp14", X86frcp>, - avx512_fp_unop_p_int<0x4C, "vrcp14", - int_x86_avx512_rcp14_ps_512, int_x86_avx512_rcp14_pd_512>; - -defm VRSQRT14 : avx512_fp_unop_s<0x4F, "vrsqrt14">, - avx512_fp_unop_p<0x4E, "vrsqrt14", X86frsqrt>, - avx512_fp_unop_p_int<0x4E, "vrsqrt14", - int_x86_avx512_rsqrt14_ps_512, int_x86_avx512_rsqrt14_pd_512>; - -def : Pat<(int_x86_avx512_rsqrt14_ss VR128X:$src), - (COPY_TO_REGCLASS (VRSQRT14SSZr (f32 (IMPLICIT_DEF)), - (COPY_TO_REGCLASS VR128X:$src, FR32)), - VR128X)>; -def : Pat<(int_x86_avx512_rsqrt14_ss sse_load_f32:$src), - (VRSQRT14SSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>; - -def : Pat<(int_x86_avx512_rcp14_ss VR128X:$src), - (COPY_TO_REGCLASS (VRCP14SSZr (f32 (IMPLICIT_DEF)), - (COPY_TO_REGCLASS VR128X:$src, FR32)), - VR128X)>; -def : Pat<(int_x86_avx512_rcp14_ss sse_load_f32:$src), - (VRCP14SSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>; - -let AddedComplexity = 20, Predicates = [HasERI] in { -defm VRCP28 : avx512_fp_unop_s<0xCB, "vrcp28">, - avx512_fp_unop_p<0xCA, "vrcp28", X86frcp>, - avx512_fp_unop_p_int<0xCA, "vrcp28", - int_x86_avx512_rcp28_ps_512, int_x86_avx512_rcp28_pd_512>; - -defm VRSQRT28 : avx512_fp_unop_s<0xCD, "vrsqrt28">, - avx512_fp_unop_p<0xCC, "vrsqrt28", X86frsqrt>, - avx512_fp_unop_p_int<0xCC, "vrsqrt28", - int_x86_avx512_rsqrt28_ps_512, int_x86_avx512_rsqrt28_pd_512>; -} - -let Predicates = [HasERI] in { - def : Pat<(int_x86_avx512_rsqrt28_ss VR128X:$src), - (COPY_TO_REGCLASS (VRSQRT28SSZr (f32 (IMPLICIT_DEF)), - (COPY_TO_REGCLASS VR128X:$src, FR32)), - VR128X)>; - def : Pat<(int_x86_avx512_rsqrt28_ss sse_load_f32:$src), - (VRSQRT28SSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>; - - def : Pat<(int_x86_avx512_rcp28_ss VR128X:$src), - (COPY_TO_REGCLASS (VRCP28SSZr (f32 (IMPLICIT_DEF)), - (COPY_TO_REGCLASS VR128X:$src, FR32)), - VR128X)>; - def : Pat<(int_x86_avx512_rcp28_ss sse_load_f32:$src), - (VRCP28SSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>; +defm VRCP28SS : avx512_fp28_s<0xCB, "vrcp28ss", FR32X, f32mem>, + EVEX_CD8<32, CD8VT1>; +defm VRCP28SD : avx512_fp28_s<0xCB, "vrcp28sd", FR64X, f64mem>, + VEX_W, EVEX_CD8<64, CD8VT1>; +defm VRSQRT28SS : avx512_fp28_s<0xCD, "vrsqrt28ss", FR32X, f32mem>, + EVEX_CD8<32, CD8VT1>; +defm VRSQRT28SD : avx512_fp28_s<0xCD, "vrsqrt28sd", FR64X, f64mem>, + VEX_W, EVEX_CD8<64, CD8VT1>; + +def : Pat <(v4f32 (int_x86_avx512_rcp28_ss (v4f32 VR128X:$src1), + (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1), + FROUND_NO_EXC)), + (COPY_TO_REGCLASS (VRCP28SSrrb (COPY_TO_REGCLASS VR128X:$src1, FR32X), + (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; + +def : Pat <(v2f64 (int_x86_avx512_rcp28_sd (v2f64 VR128X:$src1), + (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1), + FROUND_NO_EXC)), + (COPY_TO_REGCLASS (VRCP28SDrrb (COPY_TO_REGCLASS VR128X:$src1, FR64X), + (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; + +def : Pat <(v4f32 (int_x86_avx512_rsqrt28_ss (v4f32 VR128X:$src1), + (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1), + FROUND_NO_EXC)), + (COPY_TO_REGCLASS (VRSQRT28SSrrb (COPY_TO_REGCLASS VR128X:$src1, FR32X), + (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; + +def : Pat <(v2f64 (int_x86_avx512_rsqrt28_sd (v2f64 VR128X:$src1), + (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1), + FROUND_NO_EXC)), + (COPY_TO_REGCLASS (VRSQRT28SDrrb (COPY_TO_REGCLASS VR128X:$src1, FR64X), + (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; + +/// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd +multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, + RegisterClass RC, X86MemOperand x86memop> { + let hasSideEffects = 0, Predicates = [HasERI] in { + def r : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), + !strconcat(OpcodeStr, + " \t{$src, $dst|$dst, $src}"), + []>, EVEX; + def rb : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), + !strconcat(OpcodeStr, + " \t{{sae}, $src, $dst|$dst, $src, {sae}}"), + []>, EVEX, EVEX_B; + def m : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), + []>, EVEX; + } } +defm VRSQRT28PSZ : avx512_fp28_p<0xCC, "vrsqrt28ps", VR512, f512mem>, + EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VRSQRT28PDZ : avx512_fp28_p<0xCC, "vrsqrt28pd", VR512, f512mem>, + VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; +defm VRCP28PSZ : avx512_fp28_p<0xCA, "vrcp28ps", VR512, f512mem>, + EVEX_V512, EVEX_CD8<32, CD8VF>; +defm VRCP28PDZ : avx512_fp28_p<0xCA, "vrcp28pd", VR512, f512mem>, + VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; + +def : Pat <(v16f32 (int_x86_avx512_rsqrt28_ps (v16f32 VR512:$src), + (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), FROUND_NO_EXC)), + (VRSQRT28PSZrb VR512:$src)>; +def : Pat <(v8f64 (int_x86_avx512_rsqrt28_pd (v8f64 VR512:$src), + (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1), FROUND_NO_EXC)), + (VRSQRT28PDZrb VR512:$src)>; + +def : Pat <(v16f32 (int_x86_avx512_rcp28_ps (v16f32 VR512:$src), + (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), FROUND_NO_EXC)), + (VRCP28PSZrb VR512:$src)>; +def : Pat <(v8f64 (int_x86_avx512_rcp28_pd (v8f64 VR512:$src), + (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1), FROUND_NO_EXC)), + (VRCP28PDZrb VR512:$src)>; + multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, Intrinsic V16F32Int, Intrinsic V8F64Int, OpndItins itins_s, OpndItins itins_d> { def PSZrr :AVX512PSI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), [(set VR512:$dst, (v16f32 (OpNode VR512:$src)))], itins_s.rr>, EVEX, EVEX_V512; let mayLoad = 1 in def PSZrm : AVX512PSI<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), [(set VR512:$dst, (OpNode (v16f32 (bitconvert (memopv16f32 addr:$src)))))], itins_s.rm>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>; def PDZrr : AVX512PDI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), [(set VR512:$dst, (v8f64 (OpNode VR512:$src)))], itins_d.rr>, EVEX, EVEX_V512; let mayLoad = 1 in def PDZrm : AVX512PDI<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), [(set VR512:$dst, (OpNode (v8f64 (bitconvert (memopv16f32 addr:$src)))))], itins_d.rm>, EVEX, EVEX_V512, EVEX_CD8<64, CD8VF>; +let isCodeGenOnly = 1 in { def PSZr_Int : AVX512PSI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src), !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), @@ -2929,7 +3474,8 @@ multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), [(set VR512:$dst, (V8F64Int (memopv8f64 addr:$src)))]>, - EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; +} // isCodeGenOnly = 1 } multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, @@ -2938,12 +3484,13 @@ multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, def SSZr : SI<opc, MRMSrcReg, (outs FR32X:$dst), (ins FR32X:$src1, FR32X:$src2), !strconcat(OpcodeStr, - "ss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [], itins_s.rr>, XS, EVEX_4V; + let isCodeGenOnly = 1 in def SSZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), !strconcat(OpcodeStr, - "ss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128X:$dst, (F32Int VR128X:$src1, VR128X:$src2))], itins_s.rr>, XS, EVEX_4V; @@ -2951,12 +3498,13 @@ multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, def SSZm : SI<opc, MRMSrcMem, (outs FR32X:$dst), (ins FR32X:$src1, f32mem:$src2), !strconcat(OpcodeStr, - "ss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [], itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>; + let isCodeGenOnly = 1 in def SSZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst), (ins VR128X:$src1, ssmem:$src2), !strconcat(OpcodeStr, - "ss{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128X:$dst, (F32Int VR128X:$src1, sse_load_f32:$src2))], itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>; @@ -2964,12 +3512,13 @@ multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, def SDZr : SI<opc, MRMSrcReg, (outs FR64X:$dst), (ins FR64X:$src1, FR64X:$src2), !strconcat(OpcodeStr, - "sd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, + "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, XD, EVEX_4V, VEX_W; + let isCodeGenOnly = 1 in def SDZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), !strconcat(OpcodeStr, - "sd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128X:$dst, (F64Int VR128X:$src1, VR128X:$src2))], itins_s.rr>, XD, EVEX_4V, VEX_W; @@ -2977,12 +3526,13 @@ multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, def SDZm : SI<opc, MRMSrcMem, (outs FR64X:$dst), (ins FR64X:$src1, f64mem:$src2), !strconcat(OpcodeStr, - "sd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, + "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; + let isCodeGenOnly = 1 in def SDZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst), (ins VR128X:$src1, sdmem:$src2), !strconcat(OpcodeStr, - "sd{z}\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128X:$dst, (F64Int VR128X:$src1, sse_load_f64:$src2))]>, XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; @@ -3010,15 +3560,15 @@ let Predicates = [HasAVX512] in { Requires<[OptForSize]>; def : Pat<(f32 (X86frsqrt FR32X:$src)), - (VRSQRT14SSZr (f32 (IMPLICIT_DEF)), FR32X:$src)>; + (VRSQRT14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>; def : Pat<(f32 (X86frsqrt (load addr:$src))), - (VRSQRT14SSZm (f32 (IMPLICIT_DEF)), addr:$src)>, + (VRSQRT14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[OptForSize]>; def : Pat<(f32 (X86frcp FR32X:$src)), - (VRCP14SSZr (f32 (IMPLICIT_DEF)), FR32X:$src)>; + (VRCP14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>; def : Pat<(f32 (X86frcp (load addr:$src))), - (VRCP14SSZm (f32 (IMPLICIT_DEF)), addr:$src)>, + (VRCP14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[OptForSize]>; def : Pat<(int_x86_sse_sqrt_ss VR128X:$src), @@ -3094,6 +3644,7 @@ let ExeDomain = GenericDomain in { []>; // Intrinsic operation, reg. + let isCodeGenOnly = 1 in def SSr_Int : AVX512AIi8<opcss, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2, i32i8imm:$src3), !strconcat(OpcodeStr, @@ -3118,6 +3669,7 @@ let ExeDomain = GenericDomain in { []>, VEX_W; // Intrinsic operation, reg. + let isCodeGenOnly = 1 in def SDr_Int : AVX512AIi8<opcsd, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2, i32i8imm:$src3), !strconcat(OpcodeStr, @@ -3136,18 +3688,70 @@ let ExeDomain = GenericDomain in { } // ExeDomain = GenericDomain } -let Predicates = [HasAVX512] in { - defm VRNDSCALE : avx512_fp_binop_rm<0x0A, 0x0B, "vrndscale", - int_x86_avx512_rndscale_ss, - int_x86_avx512_rndscale_sd>, EVEX_4V; +multiclass avx512_rndscale<bits<8> opc, string OpcodeStr, + X86MemOperand x86memop, RegisterClass RC, + PatFrag mem_frag, Domain d> { +let ExeDomain = d in { + // Intrinsic operation, reg. + // Vector intrinsic operation, reg + def r : AVX512AIi8<opc, MRMSrcReg, + (outs RC:$dst), (ins RC:$src1, i32i8imm:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX; - defm VRNDSCALEZ : avx512_fp_unop_rm<0x08, 0x09, "vrndscale", f256mem, VR512, - memopv16f32, memopv8f64, - int_x86_avx512_rndscale_ps_512, - int_x86_avx512_rndscale_pd_512, CD8VF>, - EVEX, EVEX_V512; + // Vector intrinsic operation, mem + def m : AVX512AIi8<opc, MRMSrcMem, + (outs RC:$dst), (ins x86memop:$src1, i32i8imm:$src2), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX; +} // ExeDomain } + +defm VRNDSCALEPSZ : avx512_rndscale<0x08, "vrndscaleps", f512mem, VR512, + memopv16f32, SSEPackedSingle>, EVEX_V512, + EVEX_CD8<32, CD8VF>; + +def : Pat<(v16f32 (int_x86_avx512_mask_rndscale_ps_512 (v16f32 VR512:$src1), + imm:$src2, (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), + FROUND_CURRENT)), + (VRNDSCALEPSZr VR512:$src1, imm:$src2)>; + + +defm VRNDSCALEPDZ : avx512_rndscale<0x09, "vrndscalepd", f512mem, VR512, + memopv8f64, SSEPackedDouble>, EVEX_V512, + VEX_W, EVEX_CD8<64, CD8VF>; + +def : Pat<(v8f64 (int_x86_avx512_mask_rndscale_pd_512 (v8f64 VR512:$src1), + imm:$src2, (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1), + FROUND_CURRENT)), + (VRNDSCALEPDZr VR512:$src1, imm:$src2)>; + +multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, + Operand x86memop, RegisterClass RC, Domain d> { +let ExeDomain = d in { + def r : AVX512AIi8<opc, MRMSrcReg, + (outs RC:$dst), (ins RC:$src1, RC:$src2, i32i8imm:$src3), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX_4V; + + def m : AVX512AIi8<opc, MRMSrcMem, + (outs RC:$dst), (ins RC:$src1, x86memop:$src2, i32i8imm:$src3), + !strconcat(OpcodeStr, + " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX_4V; +} // ExeDomain +} + +defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", ssmem, FR32X, + SSEPackedSingle>, EVEX_CD8<32, CD8VT1>; + +defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", sdmem, FR64X, + SSEPackedDouble>, EVEX_CD8<64, CD8VT1>; + def : Pat<(ffloor FR32X:$src), (VRNDSCALESSr (f32 (IMPLICIT_DEF)), FR32X:$src, (i32 0x1))>; def : Pat<(f64 (ffloor FR64X:$src)), @@ -3170,26 +3774,26 @@ def : Pat<(f64 (ftrunc FR64X:$src)), (VRNDSCALESDr (f64 (IMPLICIT_DEF)), FR64X:$src, (i32 0x3))>; def : Pat<(v16f32 (ffloor VR512:$src)), - (VRNDSCALEZPSr VR512:$src, (i32 0x1))>; + (VRNDSCALEPSZr VR512:$src, (i32 0x1))>; def : Pat<(v16f32 (fnearbyint VR512:$src)), - (VRNDSCALEZPSr VR512:$src, (i32 0xC))>; + (VRNDSCALEPSZr VR512:$src, (i32 0xC))>; def : Pat<(v16f32 (fceil VR512:$src)), - (VRNDSCALEZPSr VR512:$src, (i32 0x2))>; + (VRNDSCALEPSZr VR512:$src, (i32 0x2))>; def : Pat<(v16f32 (frint VR512:$src)), - (VRNDSCALEZPSr VR512:$src, (i32 0x4))>; + (VRNDSCALEPSZr VR512:$src, (i32 0x4))>; def : Pat<(v16f32 (ftrunc VR512:$src)), - (VRNDSCALEZPSr VR512:$src, (i32 0x3))>; + (VRNDSCALEPSZr VR512:$src, (i32 0x3))>; def : Pat<(v8f64 (ffloor VR512:$src)), - (VRNDSCALEZPDr VR512:$src, (i32 0x1))>; + (VRNDSCALEPDZr VR512:$src, (i32 0x1))>; def : Pat<(v8f64 (fnearbyint VR512:$src)), - (VRNDSCALEZPDr VR512:$src, (i32 0xC))>; + (VRNDSCALEPDZr VR512:$src, (i32 0xC))>; def : Pat<(v8f64 (fceil VR512:$src)), - (VRNDSCALEZPDr VR512:$src, (i32 0x2))>; + (VRNDSCALEPDZr VR512:$src, (i32 0x2))>; def : Pat<(v8f64 (frint VR512:$src)), - (VRNDSCALEZPDr VR512:$src, (i32 0x4))>; + (VRNDSCALEPDZr VR512:$src, (i32 0x4))>; def : Pat<(v8f64 (ftrunc VR512:$src)), - (VRNDSCALEZPDr VR512:$src, (i32 0x3))>; + (VRNDSCALEPDZr VR512:$src, (i32 0x3))>; //------------------------------------------------- // Integer truncate and extend operations @@ -3200,17 +3804,17 @@ multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, RegisterClass KRC, X86MemOperand x86memop> { def rr : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst), (ins srcRC:$src), - !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr," \t{$src, $dst|$dst, $src}"), []>, EVEX; def krr : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst), (ins KRC:$mask, srcRC:$src), !strconcat(OpcodeStr, - "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), []>, EVEX, EVEX_KZ; def mr : AVX512XS8I<opc, MRMDestMem, (outs), (ins x86memop:$dst, srcRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>, EVEX; } defm VPMOVQB : avx512_trunc_sat<0x32, "vpmovqb", VR128X, VR512, VK8WM, @@ -3266,11 +3870,11 @@ multiclass avx512_extend<bits<8> opc, string OpcodeStr, RegisterClass DstRC, def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpVT (OpNode (InVT SrcRC:$src))))]>, EVEX; def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"), + !strconcat(OpcodeStr," \t{$src, $dst|$dst, $src}"), [(set DstRC:$dst, (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))]>, EVEX; @@ -3318,18 +3922,23 @@ let mayLoad = 1, def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst, KRC:$mask_wb), (ins RC:$src1, KRC:$mask, memop:$src2), !strconcat(OpcodeStr, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), []>, EVEX, EVEX_K; } + +let ExeDomain = SSEPackedDouble in { defm VGATHERDPDZ : avx512_gather<0x92, "vgatherdpd", VK8WM, VR512, vy64xmem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; -defm VGATHERDPSZ : avx512_gather<0x92, "vgatherdps", VK16WM, VR512, vz32mem>, - EVEX_V512, EVEX_CD8<32, CD8VT1>; - defm VGATHERQPDZ : avx512_gather<0x93, "vgatherqpd", VK8WM, VR512, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; +} + +let ExeDomain = SSEPackedSingle in { +defm VGATHERDPSZ : avx512_gather<0x92, "vgatherdps", VK16WM, VR512, vz32mem>, + EVEX_V512, EVEX_CD8<32, CD8VT1>; defm VGATHERQPSZ : avx512_gather<0x93, "vgatherqps", VK8WM, VR256X, vz64mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; +} defm VPGATHERDQZ : avx512_gather<0x90, "vpgatherdq", VK8WM, VR512, vy64xmem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; @@ -3347,20 +3956,24 @@ let mayStore = 1, Constraints = "$mask = $mask_wb" in def mr : AVX5128I<opc, MRMDestMem, (outs KRC:$mask_wb), (ins memop:$dst, KRC:$mask, RC:$src2), !strconcat(OpcodeStr, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), []>, EVEX, EVEX_K; } +let ExeDomain = SSEPackedDouble in { defm VSCATTERDPDZ : avx512_scatter<0xA2, "vscatterdpd", VK8WM, VR512, vy64xmem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; -defm VSCATTERDPSZ : avx512_scatter<0xA2, "vscatterdps", VK16WM, VR512, vz32mem>, - EVEX_V512, EVEX_CD8<32, CD8VT1>; - defm VSCATTERQPDZ : avx512_scatter<0xA3, "vscatterqpd", VK8WM, VR512, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; +} + +let ExeDomain = SSEPackedSingle in { +defm VSCATTERDPSZ : avx512_scatter<0xA2, "vscatterdps", VK16WM, VR512, vz32mem>, + EVEX_V512, EVEX_CD8<32, CD8VT1>; defm VSCATTERQPSZ : avx512_scatter<0xA3, "vscatterqps", VK8WM, VR256X, vz64mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; - +} + defm VPSCATTERDQZ : avx512_scatter<0xA0, "vpscatterdq", VK8WM, VR512, vy64xmem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; defm VPSCATTERDDZ : avx512_scatter<0xA0, "vpscatterdd", VK16WM, VR512, vz32mem>, @@ -3380,23 +3993,23 @@ multiclass avx512_shufp<RegisterClass RC, X86MemOperand x86memop, def rmi : AVX512PIi8<0xC6, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), + " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2), (i8 imm:$src3))))], d, IIC_SSE_SHUFP>, EVEX_4V, Sched<[WriteShuffleLd, ReadAfterLd]>; def rri : AVX512PIi8<0xC6, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), + " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2, (i8 imm:$src3))))], d, IIC_SSE_SHUFP>, EVEX_4V, Sched<[WriteShuffle]>; } defm VSHUFPSZ : avx512_shufp<VR512, f512mem, v16f32, "vshufps", memopv16f32, - SSEPackedSingle>, EVEX_V512, EVEX_CD8<32, CD8VF>; + SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>; defm VSHUFPDZ : avx512_shufp<VR512, f512mem, v8f64, "vshufpd", memopv8f64, - SSEPackedDouble>, OpSize, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; + SSEPackedDouble>, PD, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; def : Pat<(v16i32 (X86Shufp VR512:$src1, VR512:$src2, (i8 imm:$imm))), (VSHUFPSZrri VR512:$src1, VR512:$src2, imm:$imm)>; @@ -3415,13 +4028,13 @@ multiclass avx512_alignr<string OpcodeStr, RegisterClass RC, def rri : AVX512AIi8<0x03, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), + " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>, EVEX_4V; let mayLoad = 1 in def rmi : AVX512AIi8<0x03, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$src3), !strconcat(OpcodeStr, - "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), + " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>, EVEX_4V; } defm VALIGND : avx512_alignr<"valignd", VR512, i512mem>, @@ -3438,54 +4051,111 @@ def : Pat<(v16i32 (X86PAlignr VR512:$src1, VR512:$src2, (i8 imm:$imm))), def : Pat<(v8i64 (X86PAlignr VR512:$src1, VR512:$src2, (i8 imm:$imm))), (VALIGNQrri VR512:$src2, VR512:$src1, imm:$imm)>; -multiclass avx512_vpabs<bits<8> opc, string OpcodeStr, RegisterClass RC, - X86MemOperand x86memop> { - def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>, - EVEX; - def rm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), - (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>, - EVEX; +// Helper fragments to match sext vXi1 to vXiY. +def v16i1sextv16i32 : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>; +def v8i1sextv8i64 : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>; + +multiclass avx512_vpabs<bits<8> opc, string OpcodeStr, ValueType OpVT, + RegisterClass KRC, RegisterClass RC, + X86MemOperand x86memop, X86MemOperand x86scalar_mop, + string BrdcstStr> { + def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), + []>, EVEX; + def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src), + !strconcat(OpcodeStr, " \t{$src, $dst {${mask}}|$dst {${mask}}, $src}"), + []>, EVEX, EVEX_K; + def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src), + !strconcat(OpcodeStr, + " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"), + []>, EVEX, EVEX_KZ; + let mayLoad = 1 in { + def rm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), + (ins x86memop:$src), + !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), + []>, EVEX; + def rmk : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), + (ins KRC:$mask, x86memop:$src), + !strconcat(OpcodeStr, + " \t{$src, $dst {${mask}}|$dst {${mask}}, $src}"), + []>, EVEX, EVEX_K; + def rmkz : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), + (ins KRC:$mask, x86memop:$src), + !strconcat(OpcodeStr, + " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"), + []>, EVEX, EVEX_KZ; + def rmb : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), + (ins x86scalar_mop:$src), + !strconcat(OpcodeStr, " \t{${src}", BrdcstStr, + ", $dst|$dst, ${src}", BrdcstStr, "}"), + []>, EVEX, EVEX_B; + def rmbk : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), + (ins KRC:$mask, x86scalar_mop:$src), + !strconcat(OpcodeStr, " \t{${src}", BrdcstStr, + ", $dst {${mask}}|$dst {${mask}}, ${src}", BrdcstStr, "}"), + []>, EVEX, EVEX_B, EVEX_K; + def rmbkz : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), + (ins KRC:$mask, x86scalar_mop:$src), + !strconcat(OpcodeStr, " \t{${src}", BrdcstStr, + ", $dst {${mask}} {z}|$dst {${mask}} {z}, ${src}", + BrdcstStr, "}"), + []>, EVEX, EVEX_B, EVEX_KZ; + } } -defm VPABSD : avx512_vpabs<0x1E, "vpabsd", VR512, i512mem>, EVEX_V512, - EVEX_CD8<32, CD8VF>; -defm VPABSQ : avx512_vpabs<0x1F, "vpabsq", VR512, i512mem>, EVEX_V512, VEX_W, - EVEX_CD8<64, CD8VF>; +defm VPABSDZ : avx512_vpabs<0x1E, "vpabsd", v16i32, VK16WM, VR512, + i512mem, i32mem, "{1to16}">, EVEX_V512, + EVEX_CD8<32, CD8VF>; +defm VPABSQZ : avx512_vpabs<0x1F, "vpabsq", v8i64, VK8WM, VR512, + i512mem, i64mem, "{1to8}">, EVEX_V512, VEX_W, + EVEX_CD8<64, CD8VF>; + +def : Pat<(xor + (bc_v16i32 (v16i1sextv16i32)), + (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))), + (VPABSDZrr VR512:$src)>; +def : Pat<(xor + (bc_v8i64 (v8i1sextv8i64)), + (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))), + (VPABSQZrr VR512:$src)>; + +def : Pat<(v16i32 (int_x86_avx512_mask_pabs_d_512 (v16i32 VR512:$src), + (v16i32 immAllZerosV), (i16 -1))), + (VPABSDZrr VR512:$src)>; +def : Pat<(v8i64 (int_x86_avx512_mask_pabs_q_512 (v8i64 VR512:$src), + (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))), + (VPABSQZrr VR512:$src)>; multiclass avx512_conflict<bits<8> opc, string OpcodeStr, - RegisterClass RC, RegisterClass KRC, PatFrag memop_frag, - X86MemOperand x86memop, PatFrag scalar_mfrag, - X86MemOperand x86scalar_mop, string BrdcstStr, - Intrinsic Int, Intrinsic maskInt, Intrinsic maskzInt> { + RegisterClass RC, RegisterClass KRC, + X86MemOperand x86memop, + X86MemOperand x86scalar_mop, string BrdcstStr> { def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), - !strconcat(OpcodeStr, "\t{$src, ${dst} |${dst}, $src}"), - [(set RC:$dst, (Int RC:$src))]>, EVEX; + !strconcat(OpcodeStr, " \t{$src, ${dst} |${dst}, $src}"), + []>, EVEX; def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - !strconcat(OpcodeStr, "\t{$src, ${dst}|${dst}, $src}"), - [(set RC:$dst, (Int (memop_frag addr:$src)))]>, EVEX; + !strconcat(OpcodeStr, " \t{$src, ${dst}|${dst}, $src}"), + []>, EVEX; def rmb : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins x86scalar_mop:$src), - !strconcat(OpcodeStr, "\t{${src}", BrdcstStr, + !strconcat(OpcodeStr, " \t{${src}", BrdcstStr, ", ${dst}|${dst}, ${src}", BrdcstStr, "}"), []>, EVEX, EVEX_B; def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src), !strconcat(OpcodeStr, - "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), - [(set RC:$dst, (maskzInt KRC:$mask, RC:$src))]>, EVEX, EVEX_KZ; + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + []>, EVEX, EVEX_KZ; def rmkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins KRC:$mask, x86memop:$src), !strconcat(OpcodeStr, - "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), - [(set RC:$dst, (maskzInt KRC:$mask, (memop_frag addr:$src)))]>, - EVEX, EVEX_KZ; + " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), + []>, EVEX, EVEX_KZ; def rmbkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins KRC:$mask, x86scalar_mop:$src), - !strconcat(OpcodeStr, "\t{${src}", BrdcstStr, + !strconcat(OpcodeStr, " \t{${src}", BrdcstStr, ", ${dst} {${mask}} {z}|${dst} {${mask}} {z}, ${src}", BrdcstStr, "}"), []>, EVEX, EVEX_KZ, EVEX_B; @@ -3494,16 +4164,16 @@ multiclass avx512_conflict<bits<8> opc, string OpcodeStr, def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, KRC:$mask, RC:$src2), !strconcat(OpcodeStr, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), - [(set RC:$dst, (maskInt RC:$src1, KRC:$mask, RC:$src2))]>, EVEX, EVEX_K; + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), + []>, EVEX, EVEX_K; def rmk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, KRC:$mask, x86memop:$src2), !strconcat(OpcodeStr, - "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), - [(set RC:$dst, (maskInt RC:$src1, KRC:$mask, (memop_frag addr:$src2)))]>, EVEX, EVEX_K; + " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), + []>, EVEX, EVEX_K; def rmbk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, KRC:$mask, x86scalar_mop:$src2), - !strconcat(OpcodeStr, "\t{${src2}", BrdcstStr, + !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr, ", ${dst} {${mask}}|${dst} {${mask}}, ${src2}", BrdcstStr, "}"), []>, EVEX, EVEX_K, EVEX_B; } @@ -3511,16 +4181,22 @@ multiclass avx512_conflict<bits<8> opc, string OpcodeStr, let Predicates = [HasCDI] in { defm VPCONFLICTD : avx512_conflict<0xC4, "vpconflictd", VR512, VK16WM, - memopv16i32, i512mem, loadi32, i32mem, "{1to16}", - int_x86_avx512_conflict_d_512, - int_x86_avx512_conflict_d_mask_512, - int_x86_avx512_conflict_d_maskz_512>, + i512mem, i32mem, "{1to16}">, EVEX_V512, EVEX_CD8<32, CD8VF>; + defm VPCONFLICTQ : avx512_conflict<0xC4, "vpconflictq", VR512, VK8WM, - memopv8i64, i512mem, loadi64, i64mem, "{1to8}", - int_x86_avx512_conflict_q_512, - int_x86_avx512_conflict_q_mask_512, - int_x86_avx512_conflict_q_maskz_512>, + i512mem, i64mem, "{1to8}">, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; + } + +def : Pat<(int_x86_avx512_mask_conflict_d_512 VR512:$src2, VR512:$src1, + GR16:$mask), + (VPCONFLICTDrrk VR512:$src1, + (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>; + +def : Pat<(int_x86_avx512_mask_conflict_q_512 VR512:$src2, VR512:$src1, + GR8:$mask), + (VPCONFLICTQrrk VR512:$src1, + (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>; diff --git a/lib/Target/X86/X86InstrArithmetic.td b/lib/Target/X86/X86InstrArithmetic.td index 7fc9c44..368e14b 100644 --- a/lib/Target/X86/X86InstrArithmetic.td +++ b/lib/Target/X86/X86InstrArithmetic.td @@ -18,19 +18,19 @@ let SchedRW = [WriteLEA] in { let neverHasSideEffects = 1 in def LEA16r : I<0x8D, MRMSrcMem, (outs GR16:$dst), (ins i32mem:$src), - "lea{w}\t{$src|$dst}, {$dst|$src}", [], IIC_LEA_16>, OpSize; + "lea{w}\t{$src|$dst}, {$dst|$src}", [], IIC_LEA_16>, OpSize16; let isReMaterializable = 1 in def LEA32r : I<0x8D, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "lea{l}\t{$src|$dst}, {$dst|$src}", [(set GR32:$dst, lea32addr:$src)], IIC_LEA>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; def LEA64_32r : I<0x8D, MRMSrcMem, (outs GR32:$dst), (ins lea64_32mem:$src), "lea{l}\t{$src|$dst}, {$dst|$src}", [(set GR32:$dst, lea64_32addr:$src)], IIC_LEA>, - Requires<[In64BitMode]>; + OpSize32, Requires<[In64BitMode]>; let isReMaterializable = 1 in def LEA64r : RI<0x8D, MRMSrcMem, (outs GR64:$dst), (ins lea64mem:$src), @@ -68,13 +68,13 @@ def MUL8r : I<0xF6, MRM4r, (outs), (ins GR8:$src), "mul{b}\t$src", let Defs = [AX,DX,EFLAGS], Uses = [AX], neverHasSideEffects = 1 in def MUL16r : I<0xF7, MRM4r, (outs), (ins GR16:$src), "mul{w}\t$src", - [], IIC_MUL16_REG>, OpSize, Sched<[WriteIMul]>; + [], IIC_MUL16_REG>, OpSize16, Sched<[WriteIMul]>; // EAX,EDX = EAX*GR32 let Defs = [EAX,EDX,EFLAGS], Uses = [EAX], neverHasSideEffects = 1 in def MUL32r : I<0xF7, MRM4r, (outs), (ins GR32:$src), "mul{l}\t$src", [/*(set EAX, EDX, EFLAGS, (X86umul_flag EAX, GR32:$src))*/], - IIC_MUL32_REG>, Sched<[WriteIMul]>; + IIC_MUL32_REG>, OpSize32, Sched<[WriteIMul]>; // RAX,RDX = RAX*GR64 let Defs = [RAX,RDX,EFLAGS], Uses = [RAX], neverHasSideEffects = 1 in def MUL64r : RI<0xF7, MRM4r, (outs), (ins GR64:$src), @@ -95,12 +95,12 @@ let mayLoad = 1, neverHasSideEffects = 1 in { let Defs = [AX,DX,EFLAGS], Uses = [AX] in def MUL16m : I<0xF7, MRM4m, (outs), (ins i16mem:$src), "mul{w}\t$src", - [], IIC_MUL16_MEM>, OpSize, SchedLoadReg<WriteIMulLd>; + [], IIC_MUL16_MEM>, OpSize16, SchedLoadReg<WriteIMulLd>; // EAX,EDX = EAX*[mem32] let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in def MUL32m : I<0xF7, MRM4m, (outs), (ins i32mem:$src), "mul{l}\t$src", - [], IIC_MUL32_MEM>, SchedLoadReg<WriteIMulLd>; + [], IIC_MUL32_MEM>, OpSize32, SchedLoadReg<WriteIMulLd>; // RAX,RDX = RAX*[mem64] let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def MUL64m : RI<0xF7, MRM4m, (outs), (ins i64mem:$src), @@ -115,11 +115,11 @@ def IMUL8r : I<0xF6, MRM5r, (outs), (ins GR8:$src), "imul{b}\t$src", [], // AX,DX = AX*GR16 let Defs = [AX,DX,EFLAGS], Uses = [AX] in def IMUL16r : I<0xF7, MRM5r, (outs), (ins GR16:$src), "imul{w}\t$src", [], - IIC_IMUL16_RR>, OpSize, Sched<[WriteIMul]>; + IIC_IMUL16_RR>, OpSize16, Sched<[WriteIMul]>; // EAX,EDX = EAX*GR32 let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in def IMUL32r : I<0xF7, MRM5r, (outs), (ins GR32:$src), "imul{l}\t$src", [], - IIC_IMUL32_RR>, Sched<[WriteIMul]>; + IIC_IMUL32_RR>, OpSize32, Sched<[WriteIMul]>; // RAX,RDX = RAX*GR64 let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def IMUL64r : RI<0xF7, MRM5r, (outs), (ins GR64:$src), "imul{q}\t$src", [], @@ -133,12 +133,13 @@ def IMUL8m : I<0xF6, MRM5m, (outs), (ins i8mem :$src), // AX,DX = AX*[mem16] let Defs = [AX,DX,EFLAGS], Uses = [AX] in def IMUL16m : I<0xF7, MRM5m, (outs), (ins i16mem:$src), - "imul{w}\t$src", [], IIC_IMUL16_MEM>, OpSize, + "imul{w}\t$src", [], IIC_IMUL16_MEM>, OpSize16, SchedLoadReg<WriteIMulLd>; // EAX,EDX = EAX*[mem32] let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in def IMUL32m : I<0xF7, MRM5m, (outs), (ins i32mem:$src), - "imul{l}\t$src", [], IIC_IMUL32_MEM>, SchedLoadReg<WriteIMulLd>; + "imul{l}\t$src", [], IIC_IMUL32_MEM>, OpSize32, + SchedLoadReg<WriteIMulLd>; // RAX,RDX = RAX*[mem64] let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def IMUL64m : RI<0xF7, MRM5m, (outs), (ins i64mem:$src), @@ -157,12 +158,12 @@ def IMUL16rr : I<0xAF, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src1,GR16:$src2), "imul{w}\t{$src2, $dst|$dst, $src2}", [(set GR16:$dst, EFLAGS, (X86smul_flag GR16:$src1, GR16:$src2))], IIC_IMUL16_RR>, - TB, OpSize; + TB, OpSize16; def IMUL32rr : I<0xAF, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src1,GR32:$src2), "imul{l}\t{$src2, $dst|$dst, $src2}", [(set GR32:$dst, EFLAGS, (X86smul_flag GR32:$src1, GR32:$src2))], IIC_IMUL32_RR>, - TB; + TB, OpSize32; def IMUL64rr : RI<0xAF, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2), "imul{q}\t{$src2, $dst|$dst, $src2}", @@ -179,14 +180,14 @@ def IMUL16rm : I<0xAF, MRMSrcMem, (outs GR16:$dst), [(set GR16:$dst, EFLAGS, (X86smul_flag GR16:$src1, (load addr:$src2)))], IIC_IMUL16_RM>, - TB, OpSize; + TB, OpSize16; def IMUL32rm : I<0xAF, MRMSrcMem, (outs GR32:$dst), (ins GR32:$src1, i32mem:$src2), "imul{l}\t{$src2, $dst|$dst, $src2}", [(set GR32:$dst, EFLAGS, (X86smul_flag GR32:$src1, (load addr:$src2)))], IIC_IMUL32_RM>, - TB; + TB, OpSize32; def IMUL64rm : RI<0xAF, MRMSrcMem, (outs GR64:$dst), (ins GR64:$src1, i64mem:$src2), "imul{q}\t{$src2, $dst|$dst, $src2}", @@ -208,30 +209,29 @@ def IMUL16rri : Ii16<0x69, MRMSrcReg, // GR16 = GR16*I16 "imul{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR16:$dst, EFLAGS, (X86smul_flag GR16:$src1, imm:$src2))], - IIC_IMUL16_RRI>, OpSize; + IIC_IMUL16_RRI>, OpSize16; def IMUL16rri8 : Ii8<0x6B, MRMSrcReg, // GR16 = GR16*I8 (outs GR16:$dst), (ins GR16:$src1, i16i8imm:$src2), "imul{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR16:$dst, EFLAGS, (X86smul_flag GR16:$src1, i16immSExt8:$src2))], - IIC_IMUL16_RRI>, - OpSize; + IIC_IMUL16_RRI>, OpSize16; def IMUL32rri : Ii32<0x69, MRMSrcReg, // GR32 = GR32*I32 (outs GR32:$dst), (ins GR32:$src1, i32imm:$src2), "imul{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32:$dst, EFLAGS, (X86smul_flag GR32:$src1, imm:$src2))], - IIC_IMUL32_RRI>; + IIC_IMUL32_RRI>, OpSize32; def IMUL32rri8 : Ii8<0x6B, MRMSrcReg, // GR32 = GR32*I8 (outs GR32:$dst), (ins GR32:$src1, i32i8imm:$src2), "imul{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32:$dst, EFLAGS, (X86smul_flag GR32:$src1, i32immSExt8:$src2))], - IIC_IMUL32_RRI>; -def IMUL64rri32 : RIi32<0x69, MRMSrcReg, // GR64 = GR64*I32 - (outs GR64:$dst), (ins GR64:$src1, i64i32imm:$src2), - "imul{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set GR64:$dst, EFLAGS, + IIC_IMUL32_RRI>, OpSize32; +def IMUL64rri32 : RIi32S<0x69, MRMSrcReg, // GR64 = GR64*I32 + (outs GR64:$dst), (ins GR64:$src1, i64i32imm:$src2), + "imul{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + [(set GR64:$dst, EFLAGS, (X86smul_flag GR64:$src1, i64immSExt32:$src2))], IIC_IMUL64_RRI>; def IMUL64rri8 : RIi8<0x6B, MRMSrcReg, // GR64 = GR64*I8 @@ -250,31 +250,31 @@ def IMUL16rmi : Ii16<0x69, MRMSrcMem, // GR16 = [mem16]*I16 [(set GR16:$dst, EFLAGS, (X86smul_flag (load addr:$src1), imm:$src2))], IIC_IMUL16_RMI>, - OpSize; + OpSize16; def IMUL16rmi8 : Ii8<0x6B, MRMSrcMem, // GR16 = [mem16]*I8 (outs GR16:$dst), (ins i16mem:$src1, i16i8imm :$src2), "imul{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR16:$dst, EFLAGS, (X86smul_flag (load addr:$src1), i16immSExt8:$src2))], IIC_IMUL16_RMI>, - OpSize; + OpSize16; def IMUL32rmi : Ii32<0x69, MRMSrcMem, // GR32 = [mem32]*I32 (outs GR32:$dst), (ins i32mem:$src1, i32imm:$src2), "imul{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32:$dst, EFLAGS, (X86smul_flag (load addr:$src1), imm:$src2))], - IIC_IMUL32_RMI>; + IIC_IMUL32_RMI>, OpSize32; def IMUL32rmi8 : Ii8<0x6B, MRMSrcMem, // GR32 = [mem32]*I8 (outs GR32:$dst), (ins i32mem:$src1, i32i8imm: $src2), "imul{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32:$dst, EFLAGS, (X86smul_flag (load addr:$src1), i32immSExt8:$src2))], - IIC_IMUL32_RMI>; -def IMUL64rmi32 : RIi32<0x69, MRMSrcMem, // GR64 = [mem64]*I32 - (outs GR64:$dst), (ins i64mem:$src1, i64i32imm:$src2), - "imul{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set GR64:$dst, EFLAGS, + IIC_IMUL32_RMI>, OpSize32; +def IMUL64rmi32 : RIi32S<0x69, MRMSrcMem, // GR64 = [mem64]*I32 + (outs GR64:$dst), (ins i64mem:$src1, i64i32imm:$src2), + "imul{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}", + [(set GR64:$dst, EFLAGS, (X86smul_flag (load addr:$src1), i64immSExt32:$src2))], IIC_IMUL64_RMI>; @@ -299,10 +299,10 @@ def DIV8r : I<0xF6, MRM6r, (outs), (ins GR8:$src), // AX/r8 = AL,AH "div{b}\t$src", [], IIC_DIV8_REG>; let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in def DIV16r : I<0xF7, MRM6r, (outs), (ins GR16:$src), // DX:AX/r16 = AX,DX - "div{w}\t$src", [], IIC_DIV16>, OpSize; + "div{w}\t$src", [], IIC_DIV16>, OpSize16; let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in def DIV32r : I<0xF7, MRM6r, (outs), (ins GR32:$src), // EDX:EAX/r32 = EAX,EDX - "div{l}\t$src", [], IIC_DIV32>; + "div{l}\t$src", [], IIC_DIV32>, OpSize32; // RDX:RAX/r64 = RAX,RDX let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in def DIV64r : RI<0xF7, MRM6r, (outs), (ins GR64:$src), @@ -316,12 +316,12 @@ def DIV8m : I<0xF6, MRM6m, (outs), (ins i8mem:$src), // AX/[mem8] = AL,AH SchedLoadReg<WriteIDivLd>; let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in def DIV16m : I<0xF7, MRM6m, (outs), (ins i16mem:$src), // DX:AX/[mem16] = AX,DX - "div{w}\t$src", [], IIC_DIV16>, OpSize, + "div{w}\t$src", [], IIC_DIV16>, OpSize16, SchedLoadReg<WriteIDivLd>; let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in // EDX:EAX/[mem32] = EAX,EDX def DIV32m : I<0xF7, MRM6m, (outs), (ins i32mem:$src), "div{l}\t$src", [], IIC_DIV32>, - SchedLoadReg<WriteIDivLd>; + SchedLoadReg<WriteIDivLd>, OpSize32; // RDX:RAX/[mem64] = RAX,RDX let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in def DIV64m : RI<0xF7, MRM6m, (outs), (ins i64mem:$src), @@ -336,10 +336,10 @@ def IDIV8r : I<0xF6, MRM7r, (outs), (ins GR8:$src), // AX/r8 = AL,AH "idiv{b}\t$src", [], IIC_IDIV8>; let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in def IDIV16r: I<0xF7, MRM7r, (outs), (ins GR16:$src), // DX:AX/r16 = AX,DX - "idiv{w}\t$src", [], IIC_IDIV16>, OpSize; + "idiv{w}\t$src", [], IIC_IDIV16>, OpSize16; let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in def IDIV32r: I<0xF7, MRM7r, (outs), (ins GR32:$src), // EDX:EAX/r32 = EAX,EDX - "idiv{l}\t$src", [], IIC_IDIV32>; + "idiv{l}\t$src", [], IIC_IDIV32>, OpSize32; // RDX:RAX/r64 = RAX,RDX let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in def IDIV64r: RI<0xF7, MRM7r, (outs), (ins GR64:$src), @@ -353,11 +353,11 @@ def IDIV8m : I<0xF6, MRM7m, (outs), (ins i8mem:$src), // AX/[mem8] = AL,AH SchedLoadReg<WriteIDivLd>; let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in def IDIV16m: I<0xF7, MRM7m, (outs), (ins i16mem:$src), // DX:AX/[mem16] = AX,DX - "idiv{w}\t$src", [], IIC_IDIV16>, OpSize, + "idiv{w}\t$src", [], IIC_IDIV16>, OpSize16, SchedLoadReg<WriteIDivLd>; let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in // EDX:EAX/[mem32] = EAX,EDX def IDIV32m: I<0xF7, MRM7m, (outs), (ins i32mem:$src), - "idiv{l}\t$src", [], IIC_IDIV32>, + "idiv{l}\t$src", [], IIC_IDIV32>, OpSize32, SchedLoadReg<WriteIDivLd>; let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in // RDX:RAX/[mem64] = RAX,RDX def IDIV64m: RI<0xF7, MRM7m, (outs), (ins i64mem:$src), @@ -381,11 +381,11 @@ def NEG8r : I<0xF6, MRM3r, (outs GR8 :$dst), (ins GR8 :$src1), def NEG16r : I<0xF7, MRM3r, (outs GR16:$dst), (ins GR16:$src1), "neg{w}\t$dst", [(set GR16:$dst, (ineg GR16:$src1)), - (implicit EFLAGS)], IIC_UNARY_REG>, OpSize; + (implicit EFLAGS)], IIC_UNARY_REG>, OpSize16; def NEG32r : I<0xF7, MRM3r, (outs GR32:$dst), (ins GR32:$src1), "neg{l}\t$dst", [(set GR32:$dst, (ineg GR32:$src1)), - (implicit EFLAGS)], IIC_UNARY_REG>; + (implicit EFLAGS)], IIC_UNARY_REG>, OpSize32; def NEG64r : RI<0xF7, MRM3r, (outs GR64:$dst), (ins GR64:$src1), "neg{q}\t$dst", [(set GR64:$dst, (ineg GR64:$src1)), (implicit EFLAGS)], IIC_UNARY_REG>; @@ -400,11 +400,11 @@ def NEG8m : I<0xF6, MRM3m, (outs), (ins i8mem :$dst), def NEG16m : I<0xF7, MRM3m, (outs), (ins i16mem:$dst), "neg{w}\t$dst", [(store (ineg (loadi16 addr:$dst)), addr:$dst), - (implicit EFLAGS)], IIC_UNARY_MEM>, OpSize; + (implicit EFLAGS)], IIC_UNARY_MEM>, OpSize16; def NEG32m : I<0xF7, MRM3m, (outs), (ins i32mem:$dst), "neg{l}\t$dst", [(store (ineg (loadi32 addr:$dst)), addr:$dst), - (implicit EFLAGS)], IIC_UNARY_MEM>; + (implicit EFLAGS)], IIC_UNARY_MEM>, OpSize32; def NEG64m : RI<0xF7, MRM3m, (outs), (ins i64mem:$dst), "neg{q}\t$dst", [(store (ineg (loadi64 addr:$dst)), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>; @@ -422,10 +422,10 @@ def NOT8r : I<0xF6, MRM2r, (outs GR8 :$dst), (ins GR8 :$src1), [(set GR8:$dst, (not GR8:$src1))], IIC_UNARY_REG>; def NOT16r : I<0xF7, MRM2r, (outs GR16:$dst), (ins GR16:$src1), "not{w}\t$dst", - [(set GR16:$dst, (not GR16:$src1))], IIC_UNARY_REG>, OpSize; + [(set GR16:$dst, (not GR16:$src1))], IIC_UNARY_REG>, OpSize16; def NOT32r : I<0xF7, MRM2r, (outs GR32:$dst), (ins GR32:$src1), "not{l}\t$dst", - [(set GR32:$dst, (not GR32:$src1))], IIC_UNARY_REG>; + [(set GR32:$dst, (not GR32:$src1))], IIC_UNARY_REG>, OpSize32; def NOT64r : RI<0xF7, MRM2r, (outs GR64:$dst), (ins GR64:$src1), "not{q}\t$dst", [(set GR64:$dst, (not GR64:$src1))], IIC_UNARY_REG>; } @@ -438,10 +438,11 @@ def NOT8m : I<0xF6, MRM2m, (outs), (ins i8mem :$dst), def NOT16m : I<0xF7, MRM2m, (outs), (ins i16mem:$dst), "not{w}\t$dst", [(store (not (loadi16 addr:$dst)), addr:$dst)], IIC_UNARY_MEM>, - OpSize; + OpSize16; def NOT32m : I<0xF7, MRM2m, (outs), (ins i32mem:$dst), "not{l}\t$dst", - [(store (not (loadi32 addr:$dst)), addr:$dst)], IIC_UNARY_MEM>; + [(store (not (loadi32 addr:$dst)), addr:$dst)], IIC_UNARY_MEM>, + OpSize32; def NOT64m : RI<0xF7, MRM2m, (outs), (ins i64mem:$dst), "not{q}\t$dst", [(store (not (loadi64 addr:$dst)), addr:$dst)], IIC_UNARY_MEM>; } // SchedRW @@ -460,12 +461,12 @@ let isConvertibleToThreeAddress = 1, CodeSize = 1 in { // Can xform into LEA. def INC16r : I<0x40, AddRegFrm, (outs GR16:$dst), (ins GR16:$src1), "inc{w}\t$dst", [(set GR16:$dst, EFLAGS, (X86inc_flag GR16:$src1))], IIC_UNARY_REG>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def INC32r : I<0x40, AddRegFrm, (outs GR32:$dst), (ins GR32:$src1), "inc{l}\t$dst", [(set GR32:$dst, EFLAGS, (X86inc_flag GR32:$src1))], IIC_UNARY_REG>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; def INC64r : RI<0xFF, MRM0r, (outs GR64:$dst), (ins GR64:$src1), "inc{q}\t$dst", [(set GR64:$dst, EFLAGS, (X86inc_flag GR64:$src1))], IIC_UNARY_REG>; @@ -479,38 +480,39 @@ def INC64_16r : I<0xFF, MRM0r, (outs GR16:$dst), (ins GR16:$src1), "inc{w}\t$dst", [(set GR16:$dst, EFLAGS, (X86inc_flag GR16:$src1))], IIC_UNARY_REG>, - OpSize, Requires<[In64BitMode]>; + OpSize16, Requires<[In64BitMode]>; def INC64_32r : I<0xFF, MRM0r, (outs GR32:$dst), (ins GR32:$src1), "inc{l}\t$dst", [(set GR32:$dst, EFLAGS, (X86inc_flag GR32:$src1))], IIC_UNARY_REG>, - Requires<[In64BitMode]>; + OpSize32, Requires<[In64BitMode]>; def DEC64_16r : I<0xFF, MRM1r, (outs GR16:$dst), (ins GR16:$src1), "dec{w}\t$dst", [(set GR16:$dst, EFLAGS, (X86dec_flag GR16:$src1))], IIC_UNARY_REG>, - OpSize, Requires<[In64BitMode]>; + OpSize16, Requires<[In64BitMode]>; def DEC64_32r : I<0xFF, MRM1r, (outs GR32:$dst), (ins GR32:$src1), "dec{l}\t$dst", [(set GR32:$dst, EFLAGS, (X86dec_flag GR32:$src1))], IIC_UNARY_REG>, - Requires<[In64BitMode]>; + OpSize32, Requires<[In64BitMode]>; } // isConvertibleToThreeAddress = 1, CodeSize = 2 -let isCodeGenOnly = 1, CodeSize = 2 in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, + CodeSize = 2 in { def INC32_16r : I<0xFF, MRM0r, (outs GR16:$dst), (ins GR16:$src1), "inc{w}\t$dst", [], IIC_UNARY_REG>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def INC32_32r : I<0xFF, MRM0r, (outs GR32:$dst), (ins GR32:$src1), "inc{l}\t$dst", [], IIC_UNARY_REG>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; def DEC32_16r : I<0xFF, MRM1r, (outs GR16:$dst), (ins GR16:$src1), "dec{w}\t$dst", [], IIC_UNARY_REG>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def DEC32_32r : I<0xFF, MRM1r, (outs GR32:$dst), (ins GR32:$src1), "dec{l}\t$dst", [], IIC_UNARY_REG>, - Requires<[In32BitMode]>; -} // isCodeGenOnly = 1, CodeSize = 2 + OpSize32, Requires<[Not64BitMode]>; +} // isCodeGenOnly = 1, ForceDisassemble = 1, HasSideEffects = 0, CodeSize = 2 } // Constraints = "$src1 = $dst", SchedRW @@ -521,11 +523,11 @@ let CodeSize = 2, SchedRW = [WriteALULd, WriteRMW] in { def INC16m : I<0xFF, MRM0m, (outs), (ins i16mem:$dst), "inc{w}\t$dst", [(store (add (loadi16 addr:$dst), 1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def INC32m : I<0xFF, MRM0m, (outs), (ins i32mem:$dst), "inc{l}\t$dst", [(store (add (loadi32 addr:$dst), 1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; def INC64m : RI<0xFF, MRM0m, (outs), (ins i64mem:$dst), "inc{q}\t$dst", [(store (add (loadi64 addr:$dst), 1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>; @@ -536,19 +538,19 @@ let CodeSize = 2, SchedRW = [WriteALULd, WriteRMW] in { def INC64_16m : I<0xFF, MRM0m, (outs), (ins i16mem:$dst), "inc{w}\t$dst", [(store (add (loadi16 addr:$dst), 1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - OpSize, Requires<[In64BitMode]>; + OpSize16, Requires<[In64BitMode]>; def INC64_32m : I<0xFF, MRM0m, (outs), (ins i32mem:$dst), "inc{l}\t$dst", [(store (add (loadi32 addr:$dst), 1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - Requires<[In64BitMode]>; + OpSize32, Requires<[In64BitMode]>; def DEC64_16m : I<0xFF, MRM1m, (outs), (ins i16mem:$dst), "dec{w}\t$dst", [(store (add (loadi16 addr:$dst), -1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - OpSize, Requires<[In64BitMode]>; + OpSize16, Requires<[In64BitMode]>; def DEC64_32m : I<0xFF, MRM1m, (outs), (ins i32mem:$dst), "dec{l}\t$dst", [(store (add (loadi32 addr:$dst), -1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - Requires<[In64BitMode]>; + OpSize32, Requires<[In64BitMode]>; } // CodeSize = 2, SchedRW let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in { @@ -562,12 +564,12 @@ def DEC16r : I<0x48, AddRegFrm, (outs GR16:$dst), (ins GR16:$src1), "dec{w}\t$dst", [(set GR16:$dst, EFLAGS, (X86dec_flag GR16:$src1))], IIC_UNARY_REG>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def DEC32r : I<0x48, AddRegFrm, (outs GR32:$dst), (ins GR32:$src1), "dec{l}\t$dst", [(set GR32:$dst, EFLAGS, (X86dec_flag GR32:$src1))], IIC_UNARY_REG>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; def DEC64r : RI<0xFF, MRM1r, (outs GR64:$dst), (ins GR64:$src1), "dec{q}\t$dst", [(set GR64:$dst, EFLAGS, (X86dec_flag GR64:$src1))], IIC_UNARY_REG>; @@ -582,11 +584,11 @@ let CodeSize = 2, SchedRW = [WriteALULd, WriteRMW] in { def DEC16m : I<0xFF, MRM1m, (outs), (ins i16mem:$dst), "dec{w}\t$dst", [(store (add (loadi16 addr:$dst), -1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def DEC32m : I<0xFF, MRM1m, (outs), (ins i32mem:$dst), "dec{l}\t$dst", [(store (add (loadi32 addr:$dst), -1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", [(store (add (loadi64 addr:$dst), -1), addr:$dst), (implicit EFLAGS)], IIC_UNARY_MEM>; @@ -600,7 +602,8 @@ class X86TypeInfo<ValueType vt, string instrsuffix, RegisterClass regclass, PatFrag loadnode, X86MemOperand memoperand, ImmType immkind, Operand immoperand, SDPatternOperator immoperator, Operand imm8operand, SDPatternOperator imm8operator, - bit hasOddOpcode, bit hasOpSizePrefix, bit hasREX_WPrefix> { + bit hasOddOpcode, OperandSize opSize, + bit hasREX_WPrefix> { /// VT - This is the value type itself. ValueType VT = vt; @@ -650,9 +653,10 @@ class X86TypeInfo<ValueType vt, string instrsuffix, RegisterClass regclass, /// other datatypes are odd. bit HasOddOpcode = hasOddOpcode; - /// HasOpSizePrefix - This bit is set to true if the instruction should have - /// the 0x66 operand size prefix. This is set for i16 types. - bit HasOpSizePrefix = hasOpSizePrefix; + /// OpSize - Selects whether the instruction needs a 0x66 prefix based on + /// 16-bit vs 32-bit mode. i8/i64 set this to OpSizeFixed. i16 sets this + /// to Opsize16. i32 sets this to OpSize32. + OperandSize OpSize = opSize; /// HasREX_WPrefix - This bit is set to true if the instruction should have /// the 0x40 REX prefix. This is set for i64 types. @@ -664,16 +668,16 @@ def invalid_node : SDNode<"<<invalid_node>>", SDTIntLeaf,[],"<<invalid_node>>">; def Xi8 : X86TypeInfo<i8 , "b", GR8 , loadi8 , i8mem , Imm8 , i8imm , imm, i8imm , invalid_node, - 0, 0, 0>; + 0, OpSizeFixed, 0>; def Xi16 : X86TypeInfo<i16, "w", GR16, loadi16, i16mem, Imm16, i16imm, imm, i16i8imm, i16immSExt8, - 1, 1, 0>; + 1, OpSize16, 0>; def Xi32 : X86TypeInfo<i32, "l", GR32, loadi32, i32mem, Imm32, i32imm, imm, i32i8imm, i32immSExt8, - 1, 0, 0>; + 1, OpSize32, 0>; def Xi64 : X86TypeInfo<i64, "q", GR64, loadi64, i64mem, - Imm32, i64i32imm, i64immSExt32, i64i8imm, i64immSExt8, - 1, 0, 1>; + Imm32S, i64i32imm, i64immSExt32, i64i8imm, i64immSExt8, + 1, OpSizeFixed, 1>; /// ITy - This instruction base class takes the type info for the instruction. /// Using this, it: @@ -693,7 +697,7 @@ class ITy<bits<8> opcode, Format f, X86TypeInfo typeinfo, dag outs, dag ins, itin> { // Infer instruction prefixes from type info. - let hasOpSizePrefix = typeinfo.HasOpSizePrefix; + let OpSize = typeinfo.OpSize; let hasREX_WPrefix = typeinfo.HasREX_WPrefix; } @@ -752,6 +756,7 @@ class BinOpRR_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, Sched<[WriteALU]> { // The disassembler should know about this, but not the asmparser. let isCodeGenOnly = 1; + let ForceDisassemble = 1; let hasSideEffects = 0; } @@ -767,6 +772,7 @@ class BinOpRR_F_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo> Sched<[WriteALU]> { // The disassembler should know about this, but not the asmparser. let isCodeGenOnly = 1; + let ForceDisassemble = 1; let hasSideEffects = 0; } @@ -1305,8 +1311,8 @@ multiclass bmi_andn<string mnemonic, RegisterClass RC, X86MemOperand x86memop, } let Predicates = [HasBMI], Defs = [EFLAGS] in { - defm ANDN32 : bmi_andn<"andn{l}", GR32, i32mem, loadi32>, T8, VEX_4V; - defm ANDN64 : bmi_andn<"andn{q}", GR64, i64mem, loadi64>, T8, VEX_4V, VEX_W; + defm ANDN32 : bmi_andn<"andn{l}", GR32, i32mem, loadi32>, T8PS, VEX_4V; + defm ANDN64 : bmi_andn<"andn{q}", GR64, i64mem, loadi64>, T8PS, VEX_4V, VEX_W; } let Predicates = [HasBMI] in { @@ -1351,21 +1357,21 @@ let hasSideEffects = 0, Predicates = [HasADX], Defs = [EFLAGS] in { let SchedRW = [WriteALU] in { def ADCX32rr : I<0xF6, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), "adcx{l}\t{$src, $dst|$dst, $src}", - [], IIC_BIN_NONMEM>, T8, OpSize; + [], IIC_BIN_NONMEM>, T8PD; - def ADCX64rr : I<0xF6, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), + def ADCX64rr : RI<0xF6, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "adcx{q}\t{$src, $dst|$dst, $src}", - [], IIC_BIN_NONMEM>, T8, OpSize, REX_W, Requires<[In64BitMode]>; + [], IIC_BIN_NONMEM>, T8PD, Requires<[In64BitMode]>; } // SchedRW let mayLoad = 1, SchedRW = [WriteALULd] in { def ADCX32rm : I<0xF6, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "adcx{l}\t{$src, $dst|$dst, $src}", - [], IIC_BIN_MEM>, T8, OpSize; + [], IIC_BIN_MEM>, T8PD; - def ADCX64rm : I<0xF6, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), + def ADCX64rm : RI<0xF6, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "adcx{q}\t{$src, $dst|$dst, $src}", - [], IIC_BIN_MEM>, T8, OpSize, REX_W, Requires<[In64BitMode]>; + [], IIC_BIN_MEM>, T8PD, Requires<[In64BitMode]>; } } @@ -1378,9 +1384,9 @@ let hasSideEffects = 0, Predicates = [HasADX], Defs = [EFLAGS] in { "adox{l}\t{$src, $dst|$dst, $src}", [], IIC_BIN_NONMEM>, T8XS; - def ADOX64rr : I<0xF6, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), + def ADOX64rr : RI<0xF6, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "adox{q}\t{$src, $dst|$dst, $src}", - [], IIC_BIN_NONMEM>, T8XS, REX_W, Requires<[In64BitMode]>; + [], IIC_BIN_NONMEM>, T8XS, Requires<[In64BitMode]>; } // SchedRW let mayLoad = 1, SchedRW = [WriteALULd] in { @@ -1388,8 +1394,8 @@ let hasSideEffects = 0, Predicates = [HasADX], Defs = [EFLAGS] in { "adox{l}\t{$src, $dst|$dst, $src}", [], IIC_BIN_MEM>, T8XS; - def ADOX64rm : I<0xF6, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), + def ADOX64rm : RI<0xF6, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "adox{q}\t{$src, $dst|$dst, $src}", - [], IIC_BIN_MEM>, T8XS, REX_W, Requires<[In64BitMode]>; + [], IIC_BIN_MEM>, T8XS, Requires<[In64BitMode]>; } } diff --git a/lib/Target/X86/X86InstrCMovSetCC.td b/lib/Target/X86/X86InstrCMovSetCC.td index a967a4d..315f213 100644 --- a/lib/Target/X86/X86InstrCMovSetCC.td +++ b/lib/Target/X86/X86InstrCMovSetCC.td @@ -22,13 +22,13 @@ multiclass CMOV<bits<8> opc, string Mnemonic, PatLeaf CondNode> { !strconcat(Mnemonic, "{w}\t{$src2, $dst|$dst, $src2}"), [(set GR16:$dst, (X86cmov GR16:$src1, GR16:$src2, CondNode, EFLAGS))], - IIC_CMOV16_RR>,TB,OpSize; + IIC_CMOV16_RR>, TB, OpSize16; def NAME#32rr : I<opc, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2), !strconcat(Mnemonic, "{l}\t{$src2, $dst|$dst, $src2}"), [(set GR32:$dst, (X86cmov GR32:$src1, GR32:$src2, CondNode, EFLAGS))], - IIC_CMOV32_RR>, TB; + IIC_CMOV32_RR>, TB, OpSize32; def NAME#64rr :RI<opc, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2), !strconcat(Mnemonic, "{q}\t{$src2, $dst|$dst, $src2}"), @@ -44,12 +44,13 @@ multiclass CMOV<bits<8> opc, string Mnemonic, PatLeaf CondNode> { !strconcat(Mnemonic, "{w}\t{$src2, $dst|$dst, $src2}"), [(set GR16:$dst, (X86cmov GR16:$src1, (loadi16 addr:$src2), CondNode, EFLAGS))], IIC_CMOV16_RM>, - TB, OpSize; + TB, OpSize16; def NAME#32rm : I<opc, MRMSrcMem, (outs GR32:$dst), (ins GR32:$src1, i32mem:$src2), !strconcat(Mnemonic, "{l}\t{$src2, $dst|$dst, $src2}"), [(set GR32:$dst, (X86cmov GR32:$src1, (loadi32 addr:$src2), - CondNode, EFLAGS))], IIC_CMOV32_RM>, TB; + CondNode, EFLAGS))], IIC_CMOV32_RM>, + TB, OpSize32; def NAME#64rm :RI<opc, MRMSrcMem, (outs GR64:$dst), (ins GR64:$src1, i64mem:$src2), !strconcat(Mnemonic, "{q}\t{$src2, $dst|$dst, $src2}"), @@ -81,11 +82,11 @@ defm CMOVG : CMOV<0x4F, "cmovg" , X86_COND_G>; // SetCC instructions. multiclass SETCC<bits<8> opc, string Mnemonic, PatLeaf OpNode> { let Uses = [EFLAGS] in { - def r : I<opc, MRM0r, (outs GR8:$dst), (ins), + def r : I<opc, MRMXr, (outs GR8:$dst), (ins), !strconcat(Mnemonic, "\t$dst"), [(set GR8:$dst, (X86setcc OpNode, EFLAGS))], IIC_SET_R>, TB, Sched<[WriteALU]>; - def m : I<opc, MRM0m, (outs), (ins i8mem:$dst), + def m : I<opc, MRMXm, (outs), (ins i8mem:$dst), !strconcat(Mnemonic, "\t$dst"), [(store (X86setcc OpNode, EFLAGS), addr:$dst)], IIC_SET_M>, TB, Sched<[WriteALU, WriteStore]>; diff --git a/lib/Target/X86/X86InstrCompiler.td b/lib/Target/X86/X86InstrCompiler.td index 7d10b67..401849f 100644 --- a/lib/Target/X86/X86InstrCompiler.td +++ b/lib/Target/X86/X86InstrCompiler.td @@ -46,11 +46,11 @@ let Defs = [ESP, EFLAGS], Uses = [ESP] in { def ADJCALLSTACKDOWN32 : I<0, Pseudo, (outs), (ins i32imm:$amt), "#ADJCALLSTACKDOWN", [(X86callseq_start timm:$amt)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def ADJCALLSTACKUP32 : I<0, Pseudo, (outs), (ins i32imm:$amt1, i32imm:$amt2), "#ADJCALLSTACKUP", [(X86callseq_end timm:$amt1, timm:$amt2)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; } // ADJCALLSTACKDOWN/UP implicitly use/def RSP because they may be expanded into @@ -72,7 +72,7 @@ def ADJCALLSTACKUP64 : I<0, Pseudo, (outs), (ins i32imm:$amt1, i32imm:$amt2), // x86-64 va_start lowering magic. -let usesCustomInserter = 1 in { +let usesCustomInserter = 1, Defs = [EFLAGS] in { def VASTART_SAVE_XMM_REGS : I<0, Pseudo, (outs), (ins GR8:$al, @@ -81,7 +81,8 @@ def VASTART_SAVE_XMM_REGS : I<0, Pseudo, "#VASTART_SAVE_XMM_REGS $al, $regsavefi, $offset", [(X86vastart_save_xmm_regs GR8:$al, imm:$regsavefi, - imm:$offset)]>; + imm:$offset), + (implicit EFLAGS)]>; // The VAARG_64 pseudo-instruction takes the address of the va_list, // and places the address of the next argument into a register. @@ -117,7 +118,7 @@ def SEG_ALLOCA_32 : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$size), "# variable sized alloca for segmented stacks", [(set GR32:$dst, (X86SegAlloca GR32:$size))]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; let Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in def SEG_ALLOCA_64 : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$size), @@ -139,12 +140,12 @@ let Defs = [EAX, EDX, ECX, EFLAGS], FPForm = SpecialFP in { def WIN_FTOL_32 : I<0, Pseudo, (outs), (ins RFP32:$src), "# win32 fptoui", [(X86WinFTOL RFP32:$src)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def WIN_FTOL_64 : I<0, Pseudo, (outs), (ins RFP64:$src), "# win32 fptoui", [(X86WinFTOL RFP64:$src)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; } //===----------------------------------------------------------------------===// @@ -172,7 +173,7 @@ let hasSideEffects = 1, isBarrier = 1, isCodeGenOnly = 1, def EH_SjLj_SetJmp32 : I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$buf), "#EH_SJLJ_SETJMP32", [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def EH_SjLj_SetJmp64 : I<0, Pseudo, (outs GR32:$dst), (ins i64mem:$buf), "#EH_SJLJ_SETJMP64", [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>, @@ -181,7 +182,7 @@ let hasSideEffects = 1, isBarrier = 1, isCodeGenOnly = 1, def EH_SjLj_LongJmp32 : I<0, Pseudo, (outs), (ins i32mem:$buf), "#EH_SJLJ_LONGJMP32", [(X86eh_sjlj_longjmp addr:$buf)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def EH_SjLj_LongJmp64 : I<0, Pseudo, (outs), (ins i64mem:$buf), "#EH_SJLJ_LONGJMP64", [(X86eh_sjlj_longjmp addr:$buf)]>, @@ -219,10 +220,9 @@ def MORESTACK_RET_RESTORE_R10 : I<0, Pseudo, (outs), (ins), // Alias instruction mapping movr0 to xor. // FIXME: remove when we can teach regalloc that xor reg, reg is ok. -// FIXME: Set encoding to pseudo. let Defs = [EFLAGS], isReMaterializable = 1, isAsCheapAsAMove = 1, - isCodeGenOnly = 1 in -def MOV32r0 : I<0x31, MRMInitReg, (outs GR32:$dst), (ins), "", + isPseudo = 1 in +def MOV32r0 : I<0, Pseudo, (outs GR32:$dst), (ins), "", [(set GR32:$dst, 0)], IIC_ALU_NONMEM>, Sched<[WriteZero]>; // Other widths can also make use of the 32-bit xor, which may have a smaller @@ -318,13 +318,13 @@ let SchedRW = [WriteMicrocoded] in { let Defs = [ECX,EDI,ESI], Uses = [ECX,EDI,ESI], isCodeGenOnly = 1 in { def REP_MOVSB_32 : I<0xA4, RawFrm, (outs), (ins), "{rep;movsb|rep movsb}", [(X86rep_movs i8)], IIC_REP_MOVS>, REP, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def REP_MOVSW_32 : I<0xA5, RawFrm, (outs), (ins), "{rep;movsw|rep movsw}", - [(X86rep_movs i16)], IIC_REP_MOVS>, REP, OpSize, - Requires<[In32BitMode]>; + [(X86rep_movs i16)], IIC_REP_MOVS>, REP, OpSize16, + Requires<[Not64BitMode]>; def REP_MOVSD_32 : I<0xA5, RawFrm, (outs), (ins), "{rep;movsl|rep movsd}", - [(X86rep_movs i32)], IIC_REP_MOVS>, REP, - Requires<[In32BitMode]>; + [(X86rep_movs i32)], IIC_REP_MOVS>, REP, OpSize32, + Requires<[Not64BitMode]>; } let Defs = [RCX,RDI,RSI], Uses = [RCX,RDI,RSI], isCodeGenOnly = 1 in { @@ -332,10 +332,10 @@ def REP_MOVSB_64 : I<0xA4, RawFrm, (outs), (ins), "{rep;movsb|rep movsb}", [(X86rep_movs i8)], IIC_REP_MOVS>, REP, Requires<[In64BitMode]>; def REP_MOVSW_64 : I<0xA5, RawFrm, (outs), (ins), "{rep;movsw|rep movsw}", - [(X86rep_movs i16)], IIC_REP_MOVS>, REP, OpSize, + [(X86rep_movs i16)], IIC_REP_MOVS>, REP, OpSize16, Requires<[In64BitMode]>; def REP_MOVSD_64 : I<0xA5, RawFrm, (outs), (ins), "{rep;movsl|rep movsd}", - [(X86rep_movs i32)], IIC_REP_MOVS>, REP, + [(X86rep_movs i32)], IIC_REP_MOVS>, REP, OpSize32, Requires<[In64BitMode]>; def REP_MOVSQ_64 : RI<0xA5, RawFrm, (outs), (ins), "{rep;movsq|rep movsq}", [(X86rep_movs i64)], IIC_REP_MOVS>, REP, @@ -347,15 +347,15 @@ let Defs = [ECX,EDI], isCodeGenOnly = 1 in { let Uses = [AL,ECX,EDI] in def REP_STOSB_32 : I<0xAA, RawFrm, (outs), (ins), "{rep;stosb|rep stosb}", [(X86rep_stos i8)], IIC_REP_STOS>, REP, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; let Uses = [AX,ECX,EDI] in def REP_STOSW_32 : I<0xAB, RawFrm, (outs), (ins), "{rep;stosw|rep stosw}", - [(X86rep_stos i16)], IIC_REP_STOS>, REP, OpSize, - Requires<[In32BitMode]>; + [(X86rep_stos i16)], IIC_REP_STOS>, REP, OpSize16, + Requires<[Not64BitMode]>; let Uses = [EAX,ECX,EDI] in def REP_STOSD_32 : I<0xAB, RawFrm, (outs), (ins), "{rep;stosl|rep stosd}", - [(X86rep_stos i32)], IIC_REP_STOS>, REP, - Requires<[In32BitMode]>; + [(X86rep_stos i32)], IIC_REP_STOS>, REP, OpSize32, + Requires<[Not64BitMode]>; } let Defs = [RCX,RDI], isCodeGenOnly = 1 in { @@ -365,11 +365,11 @@ let Defs = [RCX,RDI], isCodeGenOnly = 1 in { Requires<[In64BitMode]>; let Uses = [AX,RCX,RDI] in def REP_STOSW_64 : I<0xAB, RawFrm, (outs), (ins), "{rep;stosw|rep stosw}", - [(X86rep_stos i16)], IIC_REP_STOS>, REP, OpSize, + [(X86rep_stos i16)], IIC_REP_STOS>, REP, OpSize16, Requires<[In64BitMode]>; let Uses = [RAX,RCX,RDI] in def REP_STOSD_64 : I<0xAB, RawFrm, (outs), (ins), "{rep;stosl|rep stosd}", - [(X86rep_stos i32)], IIC_REP_STOS>, REP, + [(X86rep_stos i32)], IIC_REP_STOS>, REP, OpSize32, Requires<[In64BitMode]>; let Uses = [RAX,RCX,RDI] in @@ -395,11 +395,11 @@ let Defs = [EAX, ECX, EDX, FP0, FP1, FP2, FP3, FP4, FP5, FP6, ST0, def TLS_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), "# TLS_addr32", [(X86tlsaddr tls32addr:$sym)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def TLS_base_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), "# TLS_base_addr32", [(X86tlsbaseaddr tls32baseaddr:$sym)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; } // All calls clobber the non-callee saved registers. RSP is marked as @@ -431,7 +431,7 @@ let Defs = [EAX, ECX, EFLAGS], def TLSCall_32 : I<0, Pseudo, (outs), (ins i32mem:$sym), "# TLSCall_32", [(X86TLSCall addr:$sym)]>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // For x86_64, the address of the thunk is passed in %rdi, on return // the address of the variable is in %rax. All other registers are preserved. @@ -590,7 +590,7 @@ defm ATOMSWAP : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMSWAP">; let isCodeGenOnly = 1, Defs = [EFLAGS] in def OR32mrLocked : I<0x09, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$zero), "or{l}\t{$zero, $dst|$dst, $zero}", - [], IIC_ALU_MEM>, Requires<[In32BitMode]>, LOCK, + [], IIC_ALU_MEM>, Requires<[Not64BitMode]>, LOCK, Sched<[WriteALULd, WriteRMW]>; let hasSideEffects = 1 in @@ -618,13 +618,13 @@ def NAME#16mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2), !strconcat(mnemonic, "{w}\t", "{$src2, $dst|$dst, $src2}"), - [], IIC_ALU_NONMEM>, OpSize, LOCK; + [], IIC_ALU_NONMEM>, OpSize16, LOCK; def NAME#32mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2), !strconcat(mnemonic, "{l}\t", "{$src2, $dst|$dst, $src2}"), - [], IIC_ALU_NONMEM>, LOCK; + [], IIC_ALU_NONMEM>, OpSize32, LOCK; def NAME#64mr : RI<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src2), @@ -644,14 +644,14 @@ def NAME#16mi : Ii16<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, ImmMod, (outs), (ins i16mem :$dst, i16imm :$src2), !strconcat(mnemonic, "{w}\t", "{$src2, $dst|$dst, $src2}"), - [], IIC_ALU_MEM>, OpSize, LOCK; + [], IIC_ALU_MEM>, OpSize16, LOCK; def NAME#32mi : Ii32<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, ImmMod, (outs), (ins i32mem :$dst, i32imm :$src2), !strconcat(mnemonic, "{l}\t", "{$src2, $dst|$dst, $src2}"), - [], IIC_ALU_MEM>, LOCK; + [], IIC_ALU_MEM>, OpSize32, LOCK; def NAME#64mi32 : RIi32<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, @@ -665,13 +665,13 @@ def NAME#16mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, ImmMod, (outs), (ins i16mem :$dst, i16i8imm :$src2), !strconcat(mnemonic, "{w}\t", "{$src2, $dst|$dst, $src2}"), - [], IIC_ALU_MEM>, OpSize, LOCK; + [], IIC_ALU_MEM>, OpSize16, LOCK; def NAME#32mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, ImmMod, (outs), (ins i32mem :$dst, i32i8imm :$src2), !strconcat(mnemonic, "{l}\t", "{$src2, $dst|$dst, $src2}"), - [], IIC_ALU_MEM>, LOCK; + [], IIC_ALU_MEM>, OpSize32, LOCK; def NAME#64mi8 : RIi8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, ImmMod, (outs), (ins i64mem :$dst, i64i8imm :$src2), @@ -700,10 +700,10 @@ def NAME#8m : I<Opc8, Form, (outs), (ins i8mem :$dst), [], IIC_UNARY_MEM>, LOCK; def NAME#16m : I<Opc, Form, (outs), (ins i16mem:$dst), !strconcat(mnemonic, "{w}\t$dst"), - [], IIC_UNARY_MEM>, OpSize, LOCK; + [], IIC_UNARY_MEM>, OpSize16, LOCK; def NAME#32m : I<Opc, Form, (outs), (ins i32mem:$dst), !strconcat(mnemonic, "{l}\t$dst"), - [], IIC_UNARY_MEM>, LOCK; + [], IIC_UNARY_MEM>, OpSize32, LOCK; def NAME#64m : RI<Opc, Form, (outs), (ins i64mem:$dst), !strconcat(mnemonic, "{q}\t$dst"), [], IIC_UNARY_MEM>, LOCK; @@ -735,11 +735,11 @@ let isCodeGenOnly = 1, SchedRW = [WriteALULd, WriteRMW] in { let Defs = [AX, EFLAGS], Uses = [AX] in def NAME#16 : I<Opc, Form, (outs), (ins i16mem:$ptr, GR16:$swap), !strconcat(mnemonic, "{w}\t{$swap, $ptr|$ptr, $swap}"), - [(frag addr:$ptr, GR16:$swap, 2)], itin>, TB, OpSize, LOCK; + [(frag addr:$ptr, GR16:$swap, 2)], itin>, TB, OpSize16, LOCK; let Defs = [EAX, EFLAGS], Uses = [EAX] in def NAME#32 : I<Opc, Form, (outs), (ins i32mem:$ptr, GR32:$swap), !strconcat(mnemonic, "{l}\t{$swap, $ptr|$ptr, $swap}"), - [(frag addr:$ptr, GR32:$swap, 4)], itin>, TB, LOCK; + [(frag addr:$ptr, GR32:$swap, 4)], itin>, TB, OpSize32, LOCK; let Defs = [RAX, EFLAGS], Uses = [RAX] in def NAME#64 : RI<Opc, Form, (outs), (ins i64mem:$ptr, GR64:$swap), !strconcat(mnemonic, "{q}\t{$swap, $ptr|$ptr, $swap}"), @@ -782,14 +782,14 @@ multiclass ATOMIC_LOAD_BINOP<bits<8> opc8, bits<8> opc, string mnemonic, [(set GR16:$dst, (!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val))], - itin>, OpSize; + itin>, OpSize16; def NAME#32 : I<opc, MRMSrcMem, (outs GR32:$dst), (ins GR32:$val, i32mem:$ptr), !strconcat(mnemonic, "{l}\t{$val, $ptr|$ptr, $val}"), [(set GR32:$dst, (!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val))], - itin>; + itin>, OpSize32; def NAME#64 : RI<opc, MRMSrcMem, (outs GR64:$dst), (ins GR64:$val, i64mem:$ptr), !strconcat(mnemonic, "{q}\t{$val, $ptr|$ptr, $val}"), @@ -1020,22 +1020,22 @@ def X86tcret_6regs : PatFrag<(ops node:$ptr, node:$off), def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off), (TCRETURNri ptr_rc_tailcall:$dst, imm:$off)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // FIXME: This is disabled for 32-bit PIC mode because the global base // register which is part of the address mode may be assigned a // callee-saved register. def : Pat<(X86tcret (load addr:$dst), imm:$off), (TCRETURNmi addr:$dst, imm:$off)>, - Requires<[In32BitMode, IsNotPIC]>; + Requires<[Not64BitMode, IsNotPIC]>; def : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off), (TCRETURNdi texternalsym:$dst, imm:$off)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(X86tcret (i32 texternalsym:$dst), imm:$off), (TCRETURNdi texternalsym:$dst, imm:$off)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off), (TCRETURNri64 ptr_rc_tailcall:$dst, imm:$off)>, @@ -1304,13 +1304,13 @@ def : Pat<(and GR32:$src1, 0xff), (MOVZX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src1, GR32_ABCD)), sub_8bit))>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // r & (2^8-1) ==> movz def : Pat<(and GR16:$src1, 0xff), (EXTRACT_SUBREG (MOVZX32rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src1, GR16_ABCD)), sub_8bit)), sub_16bit)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // r & (2^32-1) ==> movz def : Pat<(and GR64:$src, 0x00000000FFFFFFFF), @@ -1345,13 +1345,13 @@ def : Pat<(sext_inreg GR32:$src, i8), (MOVSX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), sub_8bit))>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(sext_inreg GR16:$src, i8), (EXTRACT_SUBREG (i32 (MOVSX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), sub_8bit))), sub_16bit)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(sext_inreg GR64:$src, i32), (MOVSX64rr32 (EXTRACT_SUBREG GR64:$src, sub_32bit))>; @@ -1383,11 +1383,11 @@ def : Pat<(i16 (trunc GR32:$src)), def : Pat<(i8 (trunc GR32:$src)), (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), sub_8bit)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(i8 (trunc GR16:$src)), (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), sub_8bit)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(i32 (trunc GR64:$src)), (EXTRACT_SUBREG GR64:$src, sub_32bit)>; def : Pat<(i16 (trunc GR64:$src)), @@ -1405,38 +1405,38 @@ def : Pat<(i8 (trunc GR16:$src)), def : Pat<(i8 (trunc (srl_su GR16:$src, (i8 8)))), (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), sub_8bit_hi)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(i8 (trunc (srl_su GR32:$src, (i8 8)))), (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), sub_8bit_hi)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(srl GR16:$src, (i8 8)), (EXTRACT_SUBREG (MOVZX32rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), sub_8bit_hi)), sub_16bit)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(i32 (zext (srl_su GR16:$src, (i8 8)))), (MOVZX32rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), sub_8bit_hi))>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(i32 (anyext (srl_su GR16:$src, (i8 8)))), (MOVZX32rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), sub_8bit_hi))>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(and (srl_su GR32:$src, (i8 8)), (i32 255)), (MOVZX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), sub_8bit_hi))>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : Pat<(srl (and_su GR32:$src, 0xff00), (i8 8)), (MOVZX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), sub_8bit_hi))>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // h-register tricks. // For now, be conservative on x86-64 and use an h-register extract only if the @@ -1530,62 +1530,34 @@ def : Pat<(shl GR64:$src1, (i8 1)), (ADD64rr GR64:$src1, GR64:$src1)>; def immShift32 : ImmLeaf<i8, [{ return CountTrailingOnes_32(Imm) >= 5; }]>; def immShift64 : ImmLeaf<i8, [{ return CountTrailingOnes_32(Imm) >= 6; }]>; -// (shl x (and y, 31)) ==> (shl x, y) -def : Pat<(shl GR8:$src1, (and CL, immShift32)), - (SHL8rCL GR8:$src1)>; -def : Pat<(shl GR16:$src1, (and CL, immShift32)), - (SHL16rCL GR16:$src1)>; -def : Pat<(shl GR32:$src1, (and CL, immShift32)), - (SHL32rCL GR32:$src1)>; -def : Pat<(store (shl (loadi8 addr:$dst), (and CL, immShift32)), addr:$dst), - (SHL8mCL addr:$dst)>; -def : Pat<(store (shl (loadi16 addr:$dst), (and CL, immShift32)), addr:$dst), - (SHL16mCL addr:$dst)>; -def : Pat<(store (shl (loadi32 addr:$dst), (and CL, immShift32)), addr:$dst), - (SHL32mCL addr:$dst)>; - -def : Pat<(srl GR8:$src1, (and CL, immShift32)), - (SHR8rCL GR8:$src1)>; -def : Pat<(srl GR16:$src1, (and CL, immShift32)), - (SHR16rCL GR16:$src1)>; -def : Pat<(srl GR32:$src1, (and CL, immShift32)), - (SHR32rCL GR32:$src1)>; -def : Pat<(store (srl (loadi8 addr:$dst), (and CL, immShift32)), addr:$dst), - (SHR8mCL addr:$dst)>; -def : Pat<(store (srl (loadi16 addr:$dst), (and CL, immShift32)), addr:$dst), - (SHR16mCL addr:$dst)>; -def : Pat<(store (srl (loadi32 addr:$dst), (and CL, immShift32)), addr:$dst), - (SHR32mCL addr:$dst)>; - -def : Pat<(sra GR8:$src1, (and CL, immShift32)), - (SAR8rCL GR8:$src1)>; -def : Pat<(sra GR16:$src1, (and CL, immShift32)), - (SAR16rCL GR16:$src1)>; -def : Pat<(sra GR32:$src1, (and CL, immShift32)), - (SAR32rCL GR32:$src1)>; -def : Pat<(store (sra (loadi8 addr:$dst), (and CL, immShift32)), addr:$dst), - (SAR8mCL addr:$dst)>; -def : Pat<(store (sra (loadi16 addr:$dst), (and CL, immShift32)), addr:$dst), - (SAR16mCL addr:$dst)>; -def : Pat<(store (sra (loadi32 addr:$dst), (and CL, immShift32)), addr:$dst), - (SAR32mCL addr:$dst)>; - -// (shl x (and y, 63)) ==> (shl x, y) -def : Pat<(shl GR64:$src1, (and CL, immShift64)), - (SHL64rCL GR64:$src1)>; -def : Pat<(store (shl (loadi64 addr:$dst), (and CL, 63)), addr:$dst), - (SHL64mCL addr:$dst)>; - -def : Pat<(srl GR64:$src1, (and CL, immShift64)), - (SHR64rCL GR64:$src1)>; -def : Pat<(store (srl (loadi64 addr:$dst), (and CL, 63)), addr:$dst), - (SHR64mCL addr:$dst)>; - -def : Pat<(sra GR64:$src1, (and CL, immShift64)), - (SAR64rCL GR64:$src1)>; -def : Pat<(store (sra (loadi64 addr:$dst), (and CL, 63)), addr:$dst), - (SAR64mCL addr:$dst)>; +// Shift amount is implicitly masked. +multiclass MaskedShiftAmountPats<SDNode frag, string name> { + // (shift x (and y, 31)) ==> (shift x, y) + def : Pat<(frag GR8:$src1, (and CL, immShift32)), + (!cast<Instruction>(name # "8rCL") GR8:$src1)>; + def : Pat<(frag GR16:$src1, (and CL, immShift32)), + (!cast<Instruction>(name # "16rCL") GR16:$src1)>; + def : Pat<(frag GR32:$src1, (and CL, immShift32)), + (!cast<Instruction>(name # "32rCL") GR32:$src1)>; + def : Pat<(store (frag (loadi8 addr:$dst), (and CL, immShift32)), addr:$dst), + (!cast<Instruction>(name # "8mCL") addr:$dst)>; + def : Pat<(store (frag (loadi16 addr:$dst), (and CL, immShift32)), addr:$dst), + (!cast<Instruction>(name # "16mCL") addr:$dst)>; + def : Pat<(store (frag (loadi32 addr:$dst), (and CL, immShift32)), addr:$dst), + (!cast<Instruction>(name # "32mCL") addr:$dst)>; + + // (shift x (and y, 63)) ==> (shift x, y) + def : Pat<(frag GR64:$src1, (and CL, immShift64)), + (!cast<Instruction>(name # "64rCL") GR64:$src1)>; + def : Pat<(store (frag (loadi64 addr:$dst), (and CL, 63)), addr:$dst), + (!cast<Instruction>(name # "64mCL") addr:$dst)>; +} +defm : MaskedShiftAmountPats<shl, "SHL">; +defm : MaskedShiftAmountPats<srl, "SHR">; +defm : MaskedShiftAmountPats<sra, "SAR">; +defm : MaskedShiftAmountPats<rotl, "ROL">; +defm : MaskedShiftAmountPats<rotr, "ROR">; // (anyext (setcc_carry)) -> (setcc_carry) def : Pat<(i16 (anyext (i8 (X86setcc_c X86_COND_B, EFLAGS)))), @@ -1725,17 +1697,17 @@ def : Pat<(mul (loadi64 addr:$src1), i64immSExt32:$src2), // Increment reg. def : Pat<(add GR8 :$src, 1), (INC8r GR8 :$src)>; -def : Pat<(add GR16:$src, 1), (INC16r GR16:$src)>, Requires<[In32BitMode]>; +def : Pat<(add GR16:$src, 1), (INC16r GR16:$src)>, Requires<[Not64BitMode]>; def : Pat<(add GR16:$src, 1), (INC64_16r GR16:$src)>, Requires<[In64BitMode]>; -def : Pat<(add GR32:$src, 1), (INC32r GR32:$src)>, Requires<[In32BitMode]>; +def : Pat<(add GR32:$src, 1), (INC32r GR32:$src)>, Requires<[Not64BitMode]>; def : Pat<(add GR32:$src, 1), (INC64_32r GR32:$src)>, Requires<[In64BitMode]>; def : Pat<(add GR64:$src, 1), (INC64r GR64:$src)>; // Decrement reg. def : Pat<(add GR8 :$src, -1), (DEC8r GR8 :$src)>; -def : Pat<(add GR16:$src, -1), (DEC16r GR16:$src)>, Requires<[In32BitMode]>; +def : Pat<(add GR16:$src, -1), (DEC16r GR16:$src)>, Requires<[Not64BitMode]>; def : Pat<(add GR16:$src, -1), (DEC64_16r GR16:$src)>, Requires<[In64BitMode]>; -def : Pat<(add GR32:$src, -1), (DEC32r GR32:$src)>, Requires<[In32BitMode]>; +def : Pat<(add GR32:$src, -1), (DEC32r GR32:$src)>, Requires<[Not64BitMode]>; def : Pat<(add GR32:$src, -1), (DEC64_32r GR32:$src)>, Requires<[In64BitMode]>; def : Pat<(add GR64:$src, -1), (DEC64r GR64:$src)>; @@ -1839,3 +1811,9 @@ def : Pat<(cttz_zero_undef GR64:$src), (BSF64rr GR64:$src)>; def : Pat<(cttz_zero_undef (loadi16 addr:$src)), (BSF16rm addr:$src)>; def : Pat<(cttz_zero_undef (loadi32 addr:$src)), (BSF32rm addr:$src)>; def : Pat<(cttz_zero_undef (loadi64 addr:$src)), (BSF64rm addr:$src)>; + +// When HasMOVBE is enabled it is possible to get a non-legalized +// register-register 16 bit bswap. This maps it to a ROL instruction. +let Predicates = [HasMOVBE] in { + def : Pat<(bswap GR16:$src), (ROL16ri GR16:$src, (i8 8))>; +} diff --git a/lib/Target/X86/X86InstrControl.td b/lib/Target/X86/X86InstrControl.td index e4ccc06..39ad395 100644 --- a/lib/Target/X86/X86InstrControl.td +++ b/lib/Target/X86/X86InstrControl.td @@ -21,42 +21,50 @@ // ST1 arguments when returning values on the x87 stack. let isTerminator = 1, isReturn = 1, isBarrier = 1, hasCtrlDep = 1, FPForm = SpecialFP, SchedRW = [WriteJumpLd] in { - def RET : I <0xC3, RawFrm, (outs), (ins variable_ops), - "ret", - [(X86retflag 0)], IIC_RET>; + def RETL : I <0xC3, RawFrm, (outs), (ins variable_ops), + "ret{l}", [(X86retflag 0)], IIC_RET>, OpSize32, + Requires<[Not64BitMode]>; + def RETQ : I <0xC3, RawFrm, (outs), (ins variable_ops), + "ret{q}", [(X86retflag 0)], IIC_RET>, OpSize32, + Requires<[In64BitMode]>; def RETW : I <0xC3, RawFrm, (outs), (ins), "ret{w}", - [], IIC_RET>, OpSize; - def RETI : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt, variable_ops), - "ret\t$amt", - [(X86retflag timm:$amt)], IIC_RET_IMM>; + [], IIC_RET>, OpSize16; + def RETIL : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt, variable_ops), + "ret{l}\t$amt", + [(X86retflag timm:$amt)], IIC_RET_IMM>, OpSize32, + Requires<[Not64BitMode]>; + def RETIQ : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt, variable_ops), + "ret{q}\t$amt", + [(X86retflag timm:$amt)], IIC_RET_IMM>, OpSize32, + Requires<[In64BitMode]>; def RETIW : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt), "ret{w}\t$amt", - [], IIC_RET_IMM>, OpSize; + [], IIC_RET_IMM>, OpSize16; def LRETL : I <0xCB, RawFrm, (outs), (ins), - "{l}ret{l|f}", [], IIC_RET>; - def LRETW : I <0xCB, RawFrm, (outs), (ins), - "{l}ret{w|f}", [], IIC_RET>, OpSize; + "{l}ret{l|f}", [], IIC_RET>, OpSize32; def LRETQ : RI <0xCB, RawFrm, (outs), (ins), - "{l}ret{q|f}", [], IIC_RET>; - def LRETI : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt), - "{l}ret{l|f}\t$amt", [], IIC_RET>; + "{l}ret{|f}q", [], IIC_RET>, Requires<[In64BitMode]>; + def LRETW : I <0xCB, RawFrm, (outs), (ins), + "{l}ret{w|f}", [], IIC_RET>, OpSize16; + def LRETIL : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt), + "{l}ret{l|f}\t$amt", [], IIC_RET>, OpSize32; + def LRETIQ : RIi16<0xCA, RawFrm, (outs), (ins i16imm:$amt), + "{l}ret{|f}q\t$amt", [], IIC_RET>, Requires<[In64BitMode]>; def LRETIW : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt), - "{l}ret{w|f}\t$amt", [], IIC_RET>, OpSize; + "{l}ret{w|f}\t$amt", [], IIC_RET>, OpSize16; } // Unconditional branches. let isBarrier = 1, isBranch = 1, isTerminator = 1, SchedRW = [WriteJump] in { def JMP_4 : Ii32PCRel<0xE9, RawFrm, (outs), (ins brtarget:$dst), - "jmp\t$dst", [(br bb:$dst)], IIC_JMP_REL>; + "jmp\t$dst", [(br bb:$dst)], IIC_JMP_REL>, OpSize32; + def JMP_2 : Ii16PCRel<0xE9, RawFrm, (outs), (ins brtarget:$dst), + "jmp\t$dst", [(br bb:$dst)], IIC_JMP_REL>, OpSize16, + Requires<[In16BitMode]>; let hasSideEffects = 0 in def JMP_1 : Ii8PCRel<0xEB, RawFrm, (outs), (ins brtarget8:$dst), "jmp\t$dst", [], IIC_JMP_REL>; - // FIXME : Intel syntax for JMP64pcrel32 such that it is not ambiguious - // with JMP_1. - let hasSideEffects = 0 in - def JMP64pcrel32 : I<0xE9, RawFrm, (outs), (ins brtarget:$dst), - "jmpq\t$dst", [], IIC_JMP_REL>; } // Conditional Branches. @@ -65,8 +73,12 @@ let isBranch = 1, isTerminator = 1, Uses = [EFLAGS], SchedRW = [WriteJump] in { let hasSideEffects = 0 in def _1 : Ii8PCRel <opc1, RawFrm, (outs), (ins brtarget8:$dst), asm, [], IIC_Jcc>; + def _2 : Ii16PCRel<opc4, RawFrm, (outs), (ins brtarget:$dst), asm, + [(X86brcond bb:$dst, Cond, EFLAGS)], IIC_Jcc>, OpSize16, + TB, Requires<[In16BitMode]>; def _4 : Ii32PCRel<opc4, RawFrm, (outs), (ins brtarget:$dst), asm, - [(X86brcond bb:$dst, Cond, EFLAGS)], IIC_Jcc>, TB; + [(X86brcond bb:$dst, Cond, EFLAGS)], IIC_Jcc>, TB, + OpSize32; } } @@ -94,10 +106,10 @@ let isBranch = 1, isTerminator = 1, hasSideEffects = 0, SchedRW = [WriteJump] in // jecxz. let Uses = [CX] in def JCXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst), - "jcxz\t$dst", [], IIC_JCXZ>, AdSize, Requires<[In32BitMode]>; + "jcxz\t$dst", [], IIC_JCXZ>, AdSize, Requires<[Not64BitMode]>; let Uses = [ECX] in def JECXZ_32 : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst), - "jecxz\t$dst", [], IIC_JCXZ>, Requires<[In32BitMode]>; + "jecxz\t$dst", [], IIC_JCXZ>, Requires<[Not64BitMode]>; // J*CXZ instruction: 64-bit versions of this instruction for the asmparser. // In 64-bit mode, the address size prefix is jecxz and the unprefixed version @@ -112,12 +124,19 @@ let isBranch = 1, isTerminator = 1, hasSideEffects = 0, SchedRW = [WriteJump] in // Indirect branches let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in { + def JMP16r : I<0xFF, MRM4r, (outs), (ins GR16:$dst), "jmp{w}\t{*}$dst", + [(brind GR16:$dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>, + OpSize16, Sched<[WriteJump]>; + def JMP16m : I<0xFF, MRM4m, (outs), (ins i16mem:$dst), "jmp{w}\t{*}$dst", + [(brind (loadi16 addr:$dst))], IIC_JMP_MEM>, + Requires<[Not64BitMode]>, OpSize16, Sched<[WriteJumpLd]>; + def JMP32r : I<0xFF, MRM4r, (outs), (ins GR32:$dst), "jmp{l}\t{*}$dst", - [(brind GR32:$dst)], IIC_JMP_REG>, Requires<[In32BitMode]>, - Sched<[WriteJump]>; + [(brind GR32:$dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>, + OpSize32, Sched<[WriteJump]>; def JMP32m : I<0xFF, MRM4m, (outs), (ins i32mem:$dst), "jmp{l}\t{*}$dst", [(brind (loadi32 addr:$dst))], IIC_JMP_MEM>, - Requires<[In32BitMode]>, Sched<[WriteJumpLd]>; + Requires<[Not64BitMode]>, OpSize32, Sched<[WriteJumpLd]>; def JMP64r : I<0xFF, MRM4r, (outs), (ins GR64:$dst), "jmp{q}\t{*}$dst", [(brind GR64:$dst)], IIC_JMP_REG>, Requires<[In64BitMode]>, @@ -129,20 +148,20 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in { def FARJMP16i : Iseg16<0xEA, RawFrmImm16, (outs), (ins i16imm:$off, i16imm:$seg), "ljmp{w}\t{$seg, $off|$off, $seg}", [], - IIC_JMP_FAR_PTR>, OpSize, Sched<[WriteJump]>; + IIC_JMP_FAR_PTR>, OpSize16, Sched<[WriteJump]>; def FARJMP32i : Iseg32<0xEA, RawFrmImm16, (outs), (ins i32imm:$off, i16imm:$seg), "ljmp{l}\t{$seg, $off|$off, $seg}", [], - IIC_JMP_FAR_PTR>, Sched<[WriteJump]>; + IIC_JMP_FAR_PTR>, OpSize32, Sched<[WriteJump]>; def FARJMP64 : RI<0xFF, MRM5m, (outs), (ins opaque80mem:$dst), "ljmp{q}\t{*}$dst", [], IIC_JMP_FAR_MEM>, Sched<[WriteJump]>; def FARJMP16m : I<0xFF, MRM5m, (outs), (ins opaque32mem:$dst), - "ljmp{w}\t{*}$dst", [], IIC_JMP_FAR_MEM>, OpSize, + "ljmp{w}\t{*}$dst", [], IIC_JMP_FAR_MEM>, OpSize16, Sched<[WriteJumpLd]>; def FARJMP32m : I<0xFF, MRM5m, (outs), (ins opaque48mem:$dst), - "ljmp{l}\t{*}$dst", [], IIC_JMP_FAR_MEM>, + "ljmp{l}\t{*}$dst", [], IIC_JMP_FAR_MEM>, OpSize32, Sched<[WriteJumpLd]>; } @@ -165,38 +184,44 @@ let isCall = 1 in let Uses = [ESP] in { def CALLpcrel32 : Ii32PCRel<0xE8, RawFrm, (outs), (ins i32imm_pcrel:$dst), - "call{l}\t$dst", [], IIC_CALL_RI>, - Requires<[In32BitMode]>, Sched<[WriteJump]>; + "call{l}\t$dst", [], IIC_CALL_RI>, OpSize32, + Requires<[Not64BitMode]>, Sched<[WriteJump]>; + def CALLpcrel16 : Ii16PCRel<0xE8, RawFrm, + (outs), (ins i16imm_pcrel:$dst), + "call{w}\t$dst", [], IIC_CALL_RI>, OpSize16, + Sched<[WriteJump]>; + def CALL16r : I<0xFF, MRM2r, (outs), (ins GR16:$dst), + "call{w}\t{*}$dst", [(X86call GR16:$dst)], IIC_CALL_RI>, + OpSize16, Requires<[Not64BitMode]>, Sched<[WriteJump]>; + def CALL16m : I<0xFF, MRM2m, (outs), (ins i16mem:$dst), + "call{w}\t{*}$dst", [(X86call (loadi16 addr:$dst))], + IIC_CALL_MEM>, OpSize16, + Requires<[Not64BitMode,FavorMemIndirectCall]>, + Sched<[WriteJumpLd]>; def CALL32r : I<0xFF, MRM2r, (outs), (ins GR32:$dst), "call{l}\t{*}$dst", [(X86call GR32:$dst)], IIC_CALL_RI>, - Requires<[In32BitMode]>, Sched<[WriteJump]>; + OpSize32, Requires<[Not64BitMode]>, Sched<[WriteJump]>; def CALL32m : I<0xFF, MRM2m, (outs), (ins i32mem:$dst), "call{l}\t{*}$dst", [(X86call (loadi32 addr:$dst))], - IIC_CALL_MEM>, - Requires<[In32BitMode,FavorMemIndirectCall]>, + IIC_CALL_MEM>, OpSize32, + Requires<[Not64BitMode,FavorMemIndirectCall]>, Sched<[WriteJumpLd]>; def FARCALL16i : Iseg16<0x9A, RawFrmImm16, (outs), (ins i16imm:$off, i16imm:$seg), "lcall{w}\t{$seg, $off|$off, $seg}", [], - IIC_CALL_FAR_PTR>, OpSize, Sched<[WriteJump]>; + IIC_CALL_FAR_PTR>, OpSize16, Sched<[WriteJump]>; def FARCALL32i : Iseg32<0x9A, RawFrmImm16, (outs), (ins i32imm:$off, i16imm:$seg), "lcall{l}\t{$seg, $off|$off, $seg}", [], - IIC_CALL_FAR_PTR>, Sched<[WriteJump]>; + IIC_CALL_FAR_PTR>, OpSize32, Sched<[WriteJump]>; def FARCALL16m : I<0xFF, MRM3m, (outs), (ins opaque32mem:$dst), - "lcall{w}\t{*}$dst", [], IIC_CALL_FAR_MEM>, OpSize, + "lcall{w}\t{*}$dst", [], IIC_CALL_FAR_MEM>, OpSize16, Sched<[WriteJumpLd]>; def FARCALL32m : I<0xFF, MRM3m, (outs), (ins opaque48mem:$dst), - "lcall{l}\t{*}$dst", [], IIC_CALL_FAR_MEM>, + "lcall{l}\t{*}$dst", [], IIC_CALL_FAR_MEM>, OpSize32, Sched<[WriteJumpLd]>; - - // callw for 16 bit code for the assembler. - let isAsmParserOnly = 1 in - def CALLpcrel16 : Ii16PCRel<0xE8, RawFrm, - (outs), (ins i16imm_pcrel:$dst), - "callw\t$dst", []>, OpSize; } @@ -240,7 +265,7 @@ let isCall = 1, Uses = [RSP], SchedRW = [WriteJump] in { // the 32-bit pcrel field that we have. def CALL64pcrel32 : Ii32PCRel<0xE8, RawFrm, (outs), (ins i64i32imm_pcrel:$dst), - "call{q}\t$dst", [], IIC_CALL_RI>, + "call{q}\t$dst", [], IIC_CALL_RI>, OpSize32, Requires<[In64BitMode]>; def CALL64r : I<0xFF, MRM2r, (outs), (ins GR64:$dst), "call{q}\t{*}$dst", [(X86call GR64:$dst)], diff --git a/lib/Target/X86/X86InstrExtension.td b/lib/Target/X86/X86InstrExtension.td index 4090550..6be6a1f 100644 --- a/lib/Target/X86/X86InstrExtension.td +++ b/lib/Target/X86/X86InstrExtension.td @@ -14,17 +14,17 @@ let neverHasSideEffects = 1 in { let Defs = [AX], Uses = [AL] in def CBW : I<0x98, RawFrm, (outs), (ins), - "{cbtw|cbw}", [], IIC_CBW>, OpSize; // AX = signext(AL) + "{cbtw|cbw}", [], IIC_CBW>, OpSize16; // AX = signext(AL) let Defs = [EAX], Uses = [AX] in def CWDE : I<0x98, RawFrm, (outs), (ins), - "{cwtl|cwde}", [], IIC_CBW>; // EAX = signext(AX) + "{cwtl|cwde}", [], IIC_CBW>, OpSize32; // EAX = signext(AX) let Defs = [AX,DX], Uses = [AX] in def CWD : I<0x99, RawFrm, (outs), (ins), - "{cwtd|cwd}", [], IIC_CBW>, OpSize; // DX:AX = signext(AX) + "{cwtd|cwd}", [], IIC_CBW>, OpSize16; // DX:AX = signext(AX) let Defs = [EAX,EDX], Uses = [EAX] in def CDQ : I<0x99, RawFrm, (outs), (ins), - "{cltd|cdq}", [], IIC_CBW>; // EDX:EAX = signext(EAX) + "{cltd|cdq}", [], IIC_CBW>, OpSize32; // EDX:EAX = signext(EAX) let Defs = [RAX], Uses = [EAX] in @@ -42,54 +42,54 @@ let neverHasSideEffects = 1 in { let neverHasSideEffects = 1 in { def MOVSX16rr8 : I<0xBE, MRMSrcReg, (outs GR16:$dst), (ins GR8:$src), "movs{bw|x}\t{$src, $dst|$dst, $src}", [], IIC_MOVSX_R16_R8>, - TB, OpSize, Sched<[WriteALU]>; + TB, OpSize16, Sched<[WriteALU]>; let mayLoad = 1 in def MOVSX16rm8 : I<0xBE, MRMSrcMem, (outs GR16:$dst), (ins i8mem:$src), "movs{bw|x}\t{$src, $dst|$dst, $src}", [], IIC_MOVSX_R16_M8>, - TB, OpSize, Sched<[WriteALULd]>; + TB, OpSize16, Sched<[WriteALULd]>; } // neverHasSideEffects = 1 def MOVSX32rr8 : I<0xBE, MRMSrcReg, (outs GR32:$dst), (ins GR8:$src), "movs{bl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (sext GR8:$src))], IIC_MOVSX>, TB, - Sched<[WriteALU]>; + OpSize32, Sched<[WriteALU]>; def MOVSX32rm8 : I<0xBE, MRMSrcMem, (outs GR32:$dst), (ins i8mem :$src), "movs{bl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (sextloadi32i8 addr:$src))], IIC_MOVSX>, TB, - Sched<[WriteALULd]>; + OpSize32, Sched<[WriteALULd]>; def MOVSX32rr16: I<0xBF, MRMSrcReg, (outs GR32:$dst), (ins GR16:$src), "movs{wl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (sext GR16:$src))], IIC_MOVSX>, TB, - Sched<[WriteALU]>; + OpSize32, Sched<[WriteALU]>; def MOVSX32rm16: I<0xBF, MRMSrcMem, (outs GR32:$dst), (ins i16mem:$src), "movs{wl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (sextloadi32i16 addr:$src))], IIC_MOVSX>, - TB, Sched<[WriteALULd]>; + OpSize32, TB, Sched<[WriteALULd]>; let neverHasSideEffects = 1 in { def MOVZX16rr8 : I<0xB6, MRMSrcReg, (outs GR16:$dst), (ins GR8:$src), "movz{bw|x}\t{$src, $dst|$dst, $src}", [], IIC_MOVZX_R16_R8>, - TB, OpSize, Sched<[WriteALU]>; + TB, OpSize16, Sched<[WriteALU]>; let mayLoad = 1 in def MOVZX16rm8 : I<0xB6, MRMSrcMem, (outs GR16:$dst), (ins i8mem:$src), "movz{bw|x}\t{$src, $dst|$dst, $src}", [], IIC_MOVZX_R16_M8>, - TB, OpSize, Sched<[WriteALULd]>; + TB, OpSize16, Sched<[WriteALULd]>; } // neverHasSideEffects = 1 def MOVZX32rr8 : I<0xB6, MRMSrcReg, (outs GR32:$dst), (ins GR8 :$src), "movz{bl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (zext GR8:$src))], IIC_MOVZX>, TB, - Sched<[WriteALU]>; + OpSize32, Sched<[WriteALU]>; def MOVZX32rm8 : I<0xB6, MRMSrcMem, (outs GR32:$dst), (ins i8mem :$src), "movz{bl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (zextloadi32i8 addr:$src))], IIC_MOVZX>, TB, - Sched<[WriteALULd]>; + OpSize32, Sched<[WriteALULd]>; def MOVZX32rr16: I<0xB7, MRMSrcReg, (outs GR32:$dst), (ins GR16:$src), "movz{wl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (zext GR16:$src))], IIC_MOVZX>, TB, - Sched<[WriteALU]>; + OpSize32, Sched<[WriteALU]>; def MOVZX32rm16: I<0xB7, MRMSrcMem, (outs GR32:$dst), (ins i16mem:$src), "movz{wl|x}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (zextloadi32i16 addr:$src))], IIC_MOVZX>, - TB, Sched<[WriteALULd]>; + TB, OpSize32, Sched<[WriteALULd]>; // These are the same as the regular MOVZX32rr8 and MOVZX32rm8 // except that they use GR32_NOREX for the output operand register class diff --git a/lib/Target/X86/X86InstrFMA.td b/lib/Target/X86/X86InstrFMA.td index 69cd5a5..df6c9da 100644 --- a/lib/Target/X86/X86InstrFMA.td +++ b/lib/Target/X86/X86InstrFMA.td @@ -20,7 +20,7 @@ multiclass fma3p_rm<bits<8> opc, string OpcodeStr, PatFrag MemFrag128, PatFrag MemFrag256, ValueType OpVT128, ValueType OpVT256, SDPatternOperator Op = null_frag> { - let isCommutable = 1 in + let usesCustomInserter = 1 in def r : FMA3<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, VR128:$src3), !strconcat(OpcodeStr, @@ -36,7 +36,7 @@ multiclass fma3p_rm<bits<8> opc, string OpcodeStr, [(set VR128:$dst, (OpVT128 (Op VR128:$src2, VR128:$src1, (MemFrag128 addr:$src3))))]>; - let isCommutable = 1 in + let usesCustomInserter = 1 in def rY : FMA3<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2, VR256:$src3), !strconcat(OpcodeStr, @@ -59,6 +59,7 @@ multiclass fma3p_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231, string OpcodeStr, string PackTy, PatFrag MemFrag128, PatFrag MemFrag256, SDNode Op, ValueType OpTy128, ValueType OpTy256> { + let isCommutable = 1 in defm r213 : fma3p_rm<opc213, !strconcat(OpcodeStr, "213", PackTy), MemFrag128, MemFrag256, OpTy128, OpTy256, Op>; @@ -66,6 +67,7 @@ let neverHasSideEffects = 1 in { defm r132 : fma3p_rm<opc132, !strconcat(OpcodeStr, "132", PackTy), MemFrag128, MemFrag256, OpTy128, OpTy256>; + let isCommutable = 1 in defm r231 : fma3p_rm<opc231, !strconcat(OpcodeStr, "231", PackTy), MemFrag128, MemFrag256, OpTy128, OpTy256>; @@ -118,13 +120,14 @@ let Constraints = "$src1 = $dst" in { multiclass fma3s_rm<bits<8> opc, string OpcodeStr, X86MemOperand x86memop, RegisterClass RC, ValueType OpVT, PatFrag mem_frag, SDPatternOperator OpNode = null_frag> { - let isCommutable = 1 in + let usesCustomInserter = 1 in def r : FMA3<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, RC:$src3), !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpVT (OpNode RC:$src2, RC:$src1, RC:$src3)))]>; + let mayLoad = 1 in def m : FMA3<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, RC:$src2, x86memop:$src3), @@ -134,51 +137,49 @@ multiclass fma3s_rm<bits<8> opc, string OpcodeStr, X86MemOperand x86memop, (OpVT (OpNode RC:$src2, RC:$src1, (mem_frag addr:$src3))))]>; } - -multiclass fma3s_rm_int<bits<8> opc, string OpcodeStr, Operand memop, - ComplexPattern mem_cpat, Intrinsic IntId, - RegisterClass RC> { - let isCommutable = 1 in - def r_Int : FMA3<opc, MRMSrcReg, (outs VR128:$dst), - (ins VR128:$src1, VR128:$src2, VR128:$src3), - !strconcat(OpcodeStr, - "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), - [(set VR128:$dst, (IntId VR128:$src2, VR128:$src1, - VR128:$src3))]>; - def m_Int : FMA3<opc, MRMSrcMem, (outs VR128:$dst), - (ins VR128:$src1, VR128:$src2, memop:$src3), - !strconcat(OpcodeStr, - "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), - [(set VR128:$dst, - (IntId VR128:$src2, VR128:$src1, mem_cpat:$src3))]>; -} } // Constraints = "$src1 = $dst" multiclass fma3s_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231, - string OpStr, string PackTy, Intrinsic Int, + string OpStr, string PackTy, string PT2, Intrinsic Int, SDNode OpNode, RegisterClass RC, ValueType OpVT, X86MemOperand x86memop, Operand memop, PatFrag mem_frag, ComplexPattern mem_cpat> { let neverHasSideEffects = 1 in { defm r132 : fma3s_rm<opc132, !strconcat(OpStr, "132", PackTy), x86memop, RC, OpVT, mem_frag>; + let isCommutable = 1 in defm r231 : fma3s_rm<opc231, !strconcat(OpStr, "231", PackTy), x86memop, RC, OpVT, mem_frag>; } +let isCommutable = 1 in defm r213 : fma3s_rm<opc213, !strconcat(OpStr, "213", PackTy), - x86memop, RC, OpVT, mem_frag, OpNode>, - fma3s_rm_int<opc213, !strconcat(OpStr, "213", PackTy), - memop, mem_cpat, Int, RC>; + x86memop, RC, OpVT, mem_frag, OpNode>; } multiclass fma3s<bits<8> opc132, bits<8> opc213, bits<8> opc231, string OpStr, Intrinsic IntF32, Intrinsic IntF64, SDNode OpNode> { - defm SS : fma3s_forms<opc132, opc213, opc231, OpStr, "ss", IntF32, OpNode, + defm SS : fma3s_forms<opc132, opc213, opc231, OpStr, "ss", "SS", IntF32, OpNode, FR32, f32, f32mem, ssmem, loadf32, sse_load_f32>; - defm SD : fma3s_forms<opc132, opc213, opc231, OpStr, "sd", IntF64, OpNode, + defm SD : fma3s_forms<opc132, opc213, opc231, OpStr, "sd", "PD", IntF64, OpNode, FR64, f64, f64mem, sdmem, loadf64, sse_load_f64>, VEX_W; + + def : Pat<(IntF32 VR128:$src1, VR128:$src2, VR128:$src3), + (COPY_TO_REGCLASS + (!cast<Instruction>(NAME#"SSr213r") + (COPY_TO_REGCLASS $src2, FR32), + (COPY_TO_REGCLASS $src1, FR32), + (COPY_TO_REGCLASS $src3, FR32)), + VR128)>; + + def : Pat<(IntF64 VR128:$src1, VR128:$src2, VR128:$src3), + (COPY_TO_REGCLASS + (!cast<Instruction>(NAME#"SDr213r") + (COPY_TO_REGCLASS $src2, FR64), + (COPY_TO_REGCLASS $src1, FR64), + (COPY_TO_REGCLASS $src3, FR64)), + VR128)>; } defm VFMADD : fma3s<0x99, 0xA9, 0xB9, "vfmadd", int_x86_fma_vfmadd_ss, @@ -220,7 +221,7 @@ multiclass fma4s<bits<8> opc, string OpcodeStr, RegisterClass RC, [(set RC:$dst, (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3))]>, VEX_LIG; // For disassembler -let isCodeGenOnly = 1, hasSideEffects = 0 in +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rr_REV : FMA4<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, RC:$src3), !strconcat(OpcodeStr, @@ -230,6 +231,7 @@ let isCodeGenOnly = 1, hasSideEffects = 0 in multiclass fma4s_int<bits<8> opc, string OpcodeStr, Operand memop, ComplexPattern mem_cpat, Intrinsic Int> { +let isCodeGenOnly = 1 in { let isCommutable = 1 in def rr_Int : FMA4<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, VR128:$src3), @@ -249,6 +251,7 @@ multiclass fma4s_int<bits<8> opc, string OpcodeStr, Operand memop, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, (Int VR128:$src1, mem_cpat:$src2, VR128:$src3))]>, VEX_LIG; +} // isCodeGenOnly = 1 } multiclass fma4p<bits<8> opc, string OpcodeStr, SDNode OpNode, @@ -295,7 +298,7 @@ multiclass fma4p<bits<8> opc, string OpcodeStr, SDNode OpNode, [(set VR256:$dst, (OpNode VR256:$src1, (ld_frag256 addr:$src2), VR256:$src3))]>, VEX_L; // For disassembler -let isCodeGenOnly = 1, hasSideEffects = 0 in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { def rr_REV : FMA4<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, VR128:$src3), !strconcat(OpcodeStr, diff --git a/lib/Target/X86/X86InstrFPStack.td b/lib/Target/X86/X86InstrFPStack.td index 7c37888..4ad7b7e 100644 --- a/lib/Target/X86/X86InstrFPStack.td +++ b/lib/Target/X86/X86InstrFPStack.td @@ -206,74 +206,91 @@ def _FI32m : FPI<0xDA, fp, (outs), (ins i32mem:$src), } let Defs = [FPSW] in { +// FPBinary_rr just defines pseudo-instructions, no need to set a scheduling +// resources. defm ADD : FPBinary_rr<fadd>; defm SUB : FPBinary_rr<fsub>; defm MUL : FPBinary_rr<fmul>; defm DIV : FPBinary_rr<fdiv>; +// Sets the scheduling resources for the actual NAME#_F<size>m defintions. +let SchedRW = [WriteFAddLd] in { defm ADD : FPBinary<fadd, MRM0m, "add">; defm SUB : FPBinary<fsub, MRM4m, "sub">; defm SUBR: FPBinary<fsub ,MRM5m, "subr">; +} +let SchedRW = [WriteFMulLd] in { defm MUL : FPBinary<fmul, MRM1m, "mul">; +} +let SchedRW = [WriteFDivLd] in { defm DIV : FPBinary<fdiv, MRM6m, "div">; defm DIVR: FPBinary<fdiv, MRM7m, "divr">; } +} -class FPST0rInst<bits<8> o, string asm> - : FPI<o, AddRegFrm, (outs), (ins RST:$op), asm>, D8; -class FPrST0Inst<bits<8> o, string asm> - : FPI<o, AddRegFrm, (outs), (ins RST:$op), asm>, DC; -class FPrST0PInst<bits<8> o, string asm> - : FPI<o, AddRegFrm, (outs), (ins RST:$op), asm>, DE; +class FPST0rInst<Format fp, string asm> + : FPI<0xD8, fp, (outs), (ins RST:$op), asm>; +class FPrST0Inst<Format fp, string asm> + : FPI<0xDC, fp, (outs), (ins RST:$op), asm>; +class FPrST0PInst<Format fp, string asm> + : FPI<0xDE, fp, (outs), (ins RST:$op), asm>; // NOTE: GAS and apparently all other AT&T style assemblers have a broken notion // of some of the 'reverse' forms of the fsub and fdiv instructions. As such, // we have to put some 'r's in and take them out of weird places. -def ADD_FST0r : FPST0rInst <0xC0, "fadd\t$op">; -def ADD_FrST0 : FPrST0Inst <0xC0, "fadd\t{%st(0), $op|$op, st(0)}">; -def ADD_FPrST0 : FPrST0PInst<0xC0, "faddp\t$op">; -def SUBR_FST0r : FPST0rInst <0xE8, "fsubr\t$op">; -def SUB_FrST0 : FPrST0Inst <0xE8, "fsub{r}\t{%st(0), $op|$op, st(0)}">; -def SUB_FPrST0 : FPrST0PInst<0xE8, "fsub{r}p\t$op">; -def SUB_FST0r : FPST0rInst <0xE0, "fsub\t$op">; -def SUBR_FrST0 : FPrST0Inst <0xE0, "fsub{|r}\t{%st(0), $op|$op, st(0)}">; -def SUBR_FPrST0 : FPrST0PInst<0xE0, "fsub{|r}p\t$op">; -def MUL_FST0r : FPST0rInst <0xC8, "fmul\t$op">; -def MUL_FrST0 : FPrST0Inst <0xC8, "fmul\t{%st(0), $op|$op, st(0)}">; -def MUL_FPrST0 : FPrST0PInst<0xC8, "fmulp\t$op">; -def DIVR_FST0r : FPST0rInst <0xF8, "fdivr\t$op">; -def DIV_FrST0 : FPrST0Inst <0xF8, "fdiv{r}\t{%st(0), $op|$op, st(0)}">; -def DIV_FPrST0 : FPrST0PInst<0xF8, "fdiv{r}p\t$op">; -def DIV_FST0r : FPST0rInst <0xF0, "fdiv\t$op">; -def DIVR_FrST0 : FPrST0Inst <0xF0, "fdiv{|r}\t{%st(0), $op|$op, st(0)}">; -def DIVR_FPrST0 : FPrST0PInst<0xF0, "fdiv{|r}p\t$op">; - -def COM_FST0r : FPST0rInst <0xD0, "fcom\t$op">; -def COMP_FST0r : FPST0rInst <0xD8, "fcomp\t$op">; +let SchedRW = [WriteFAdd] in { +def ADD_FST0r : FPST0rInst <MRM0r, "fadd\t$op">; +def ADD_FrST0 : FPrST0Inst <MRM0r, "fadd\t{%st(0), $op|$op, st(0)}">; +def ADD_FPrST0 : FPrST0PInst<MRM0r, "faddp\t$op">; +def SUBR_FST0r : FPST0rInst <MRM5r, "fsubr\t$op">; +def SUB_FrST0 : FPrST0Inst <MRM5r, "fsub{r}\t{%st(0), $op|$op, st(0)}">; +def SUB_FPrST0 : FPrST0PInst<MRM5r, "fsub{r}p\t$op">; +def SUB_FST0r : FPST0rInst <MRM4r, "fsub\t$op">; +def SUBR_FrST0 : FPrST0Inst <MRM4r, "fsub{|r}\t{%st(0), $op|$op, st(0)}">; +def SUBR_FPrST0 : FPrST0PInst<MRM4r, "fsub{|r}p\t$op">; +} // SchedRW +let SchedRW = [WriteFMul] in { +def MUL_FST0r : FPST0rInst <MRM1r, "fmul\t$op">; +def MUL_FrST0 : FPrST0Inst <MRM1r, "fmul\t{%st(0), $op|$op, st(0)}">; +def MUL_FPrST0 : FPrST0PInst<MRM1r, "fmulp\t$op">; +} // SchedRW +let SchedRW = [WriteFDiv] in { +def DIVR_FST0r : FPST0rInst <MRM7r, "fdivr\t$op">; +def DIV_FrST0 : FPrST0Inst <MRM7r, "fdiv{r}\t{%st(0), $op|$op, st(0)}">; +def DIV_FPrST0 : FPrST0PInst<MRM7r, "fdiv{r}p\t$op">; +def DIV_FST0r : FPST0rInst <MRM6r, "fdiv\t$op">; +def DIVR_FrST0 : FPrST0Inst <MRM6r, "fdiv{|r}\t{%st(0), $op|$op, st(0)}">; +def DIVR_FPrST0 : FPrST0PInst<MRM6r, "fdiv{|r}p\t$op">; +} // SchedRW + +def COM_FST0r : FPST0rInst <MRM2r, "fcom\t$op">; +def COMP_FST0r : FPST0rInst <MRM3r, "fcomp\t$op">; // Unary operations. -multiclass FPUnary<SDNode OpNode, bits<8> opcode, string asmstring> { +multiclass FPUnary<SDNode OpNode, Format fp, string asmstring> { def _Fp32 : FpIf32<(outs RFP32:$dst), (ins RFP32:$src), OneArgFPRW, [(set RFP32:$dst, (OpNode RFP32:$src))]>; def _Fp64 : FpIf64<(outs RFP64:$dst), (ins RFP64:$src), OneArgFPRW, [(set RFP64:$dst, (OpNode RFP64:$src))]>; def _Fp80 : FpI_<(outs RFP80:$dst), (ins RFP80:$src), OneArgFPRW, [(set RFP80:$dst, (OpNode RFP80:$src))]>; -def _F : FPI<opcode, RawFrm, (outs), (ins), asmstring>, D9; +def _F : FPI<0xD9, fp, (outs), (ins), asmstring>; } let Defs = [FPSW] in { -defm CHS : FPUnary<fneg, 0xE0, "fchs">; -defm ABS : FPUnary<fabs, 0xE1, "fabs">; -defm SQRT: FPUnary<fsqrt,0xFA, "fsqrt">; -defm SIN : FPUnary<fsin, 0xFE, "fsin">; -defm COS : FPUnary<fcos, 0xFF, "fcos">; +defm CHS : FPUnary<fneg, MRM_E0, "fchs">; +defm ABS : FPUnary<fabs, MRM_E1, "fabs">; +let SchedRW = [WriteFSqrt] in { +defm SQRT: FPUnary<fsqrt,MRM_FA, "fsqrt">; +} +defm SIN : FPUnary<fsin, MRM_FE, "fsin">; +defm COS : FPUnary<fcos, MRM_FF, "fcos">; let neverHasSideEffects = 1 in { def TST_Fp32 : FpIf32<(outs), (ins RFP32:$src), OneArgFP, []>; def TST_Fp64 : FpIf64<(outs), (ins RFP64:$src), OneArgFP, []>; def TST_Fp80 : FpI_<(outs), (ins RFP80:$src), OneArgFP, []>; } -def TST_F : FPI<0xE4, RawFrm, (outs), (ins), "ftst">, D9; +def TST_F : FPI<0xD9, MRM_E4, (outs), (ins), "ftst">; } // Defs = [FPSW] // Versions of FP instructions that take a single memory operand. Added for the @@ -336,22 +353,22 @@ defm CMOVNP : FPCMov<X86_COND_NP>; let Predicates = [HasCMov] in { // These are not factored because there's no clean way to pass DA/DB. -def CMOVB_F : FPI<0xC0, AddRegFrm, (outs RST:$op), (ins), - "fcmovb\t{$op, %st(0)|st(0), $op}">, DA; -def CMOVBE_F : FPI<0xD0, AddRegFrm, (outs RST:$op), (ins), - "fcmovbe\t{$op, %st(0)|st(0), $op}">, DA; -def CMOVE_F : FPI<0xC8, AddRegFrm, (outs RST:$op), (ins), - "fcmove\t{$op, %st(0)|st(0), $op}">, DA; -def CMOVP_F : FPI<0xD8, AddRegFrm, (outs RST:$op), (ins), - "fcmovu\t{$op, %st(0)|st(0), $op}">, DA; -def CMOVNB_F : FPI<0xC0, AddRegFrm, (outs RST:$op), (ins), - "fcmovnb\t{$op, %st(0)|st(0), $op}">, DB; -def CMOVNBE_F: FPI<0xD0, AddRegFrm, (outs RST:$op), (ins), - "fcmovnbe\t{$op, %st(0)|st(0), $op}">, DB; -def CMOVNE_F : FPI<0xC8, AddRegFrm, (outs RST:$op), (ins), - "fcmovne\t{$op, %st(0)|st(0), $op}">, DB; -def CMOVNP_F : FPI<0xD8, AddRegFrm, (outs RST:$op), (ins), - "fcmovnu\t{$op, %st(0)|st(0), $op}">, DB; +def CMOVB_F : FPI<0xDA, MRM0r, (outs RST:$op), (ins), + "fcmovb\t{$op, %st(0)|st(0), $op}">; +def CMOVBE_F : FPI<0xDA, MRM2r, (outs RST:$op), (ins), + "fcmovbe\t{$op, %st(0)|st(0), $op}">; +def CMOVE_F : FPI<0xDA, MRM1r, (outs RST:$op), (ins), + "fcmove\t{$op, %st(0)|st(0), $op}">; +def CMOVP_F : FPI<0xDA, MRM3r, (outs RST:$op), (ins), + "fcmovu\t{$op, %st(0)|st(0), $op}">; +def CMOVNB_F : FPI<0xDB, MRM0r, (outs RST:$op), (ins), + "fcmovnb\t{$op, %st(0)|st(0), $op}">; +def CMOVNBE_F: FPI<0xDB, MRM2r, (outs RST:$op), (ins), + "fcmovnbe\t{$op, %st(0)|st(0), $op}">; +def CMOVNE_F : FPI<0xDB, MRM1r, (outs RST:$op), (ins), + "fcmovne\t{$op, %st(0)|st(0), $op}">; +def CMOVNP_F : FPI<0xDB, MRM3r, (outs RST:$op), (ins), + "fcmovnu\t{$op, %st(0)|st(0), $op}">; } // Predicates = [HasCMov] // Floating point loads & stores. @@ -492,14 +509,10 @@ def ISTT_FP64m : FPI<0xDD, MRM1m, (outs), (ins i64mem:$dst), // FP Stack manipulation instructions. let SchedRW = [WriteMove] in { -def LD_Frr : FPI<0xC0, AddRegFrm, (outs), (ins RST:$op), "fld\t$op", - IIC_FLD>, D9; -def ST_Frr : FPI<0xD0, AddRegFrm, (outs), (ins RST:$op), "fst\t$op", - IIC_FST>, DD; -def ST_FPrr : FPI<0xD8, AddRegFrm, (outs), (ins RST:$op), "fstp\t$op", - IIC_FST>, DD; -def XCH_F : FPI<0xC8, AddRegFrm, (outs), (ins RST:$op), "fxch\t$op", - IIC_FXCH>, D9; +def LD_Frr : FPI<0xD9, MRM0r, (outs), (ins RST:$op), "fld\t$op", IIC_FLD>; +def ST_Frr : FPI<0xDD, MRM2r, (outs), (ins RST:$op), "fst\t$op", IIC_FST>; +def ST_FPrr : FPI<0xDD, MRM3r, (outs), (ins RST:$op), "fstp\t$op", IIC_FST>; +def XCH_F : FPI<0xD9, MRM1r, (outs), (ins RST:$op), "fxch\t$op", IIC_FXCH>; } // Floating point constant loads. @@ -519,8 +532,8 @@ def LD_Fp180 : FpI_<(outs RFP80:$dst), (ins), ZeroArgFP, } let SchedRW = [WriteZero] in { -def LD_F0 : FPI<0xEE, RawFrm, (outs), (ins), "fldz", IIC_FLDZ>, D9; -def LD_F1 : FPI<0xE8, RawFrm, (outs), (ins), "fld1", IIC_FIST>, D9; +def LD_F0 : FPI<0xD9, MRM_EE, (outs), (ins), "fldz", IIC_FLDZ>; +def LD_F1 : FPI<0xD9, MRM_E8, (outs), (ins), "fld1", IIC_FIST>; } // Floating point compares. @@ -546,40 +559,35 @@ def UCOM_FpIr80: FpI_<(outs), (ins RFP80:$lhs, RFP80:$rhs), CompareFP, } let Defs = [FPSW], Uses = [ST0] in { -def UCOM_Fr : FPI<0xE0, AddRegFrm, // FPSW = cmp ST(0) with ST(i) - (outs), (ins RST:$reg), - "fucom\t$reg", IIC_FUCOM>, DD; -def UCOM_FPr : FPI<0xE8, AddRegFrm, // FPSW = cmp ST(0) with ST(i), pop - (outs), (ins RST:$reg), - "fucomp\t$reg", IIC_FUCOM>, DD; -def UCOM_FPPr : FPI<0xE9, RawFrm, // cmp ST(0) with ST(1), pop, pop - (outs), (ins), - "fucompp", IIC_FUCOM>, DA; +def UCOM_Fr : FPI<0xDD, MRM4r, // FPSW = cmp ST(0) with ST(i) + (outs), (ins RST:$reg), "fucom\t$reg", IIC_FUCOM>; +def UCOM_FPr : FPI<0xDD, MRM5r, // FPSW = cmp ST(0) with ST(i), pop + (outs), (ins RST:$reg), "fucomp\t$reg", IIC_FUCOM>; +def UCOM_FPPr : FPI<0xDA, MRM_E9, // cmp ST(0) with ST(1), pop, pop + (outs), (ins), "fucompp", IIC_FUCOM>; } let Defs = [EFLAGS, FPSW], Uses = [ST0] in { -def UCOM_FIr : FPI<0xE8, AddRegFrm, // CC = cmp ST(0) with ST(i) - (outs), (ins RST:$reg), - "fucomi\t$reg", IIC_FUCOMI>, DB; -def UCOM_FIPr : FPI<0xE8, AddRegFrm, // CC = cmp ST(0) with ST(i), pop - (outs), (ins RST:$reg), - "fucompi\t$reg", IIC_FUCOMI>, DF; +def UCOM_FIr : FPI<0xDB, MRM5r, // CC = cmp ST(0) with ST(i) + (outs), (ins RST:$reg), "fucomi\t$reg", IIC_FUCOMI>; +def UCOM_FIPr : FPI<0xDF, MRM5r, // CC = cmp ST(0) with ST(i), pop + (outs), (ins RST:$reg), "fucompi\t$reg", IIC_FUCOMI>; } let Defs = [EFLAGS, FPSW] in { -def COM_FIr : FPI<0xF0, AddRegFrm, (outs), (ins RST:$reg), - "fcomi\t$reg", IIC_FCOMI>, DB; -def COM_FIPr : FPI<0xF0, AddRegFrm, (outs), (ins RST:$reg), - "fcompi\t$reg", IIC_FCOMI>, DF; +def COM_FIr : FPI<0xDB, MRM6r, (outs), (ins RST:$reg), + "fcomi\t$reg", IIC_FCOMI>; +def COM_FIPr : FPI<0xDF, MRM6r, (outs), (ins RST:$reg), + "fcompi\t$reg", IIC_FCOMI>; } } // SchedRW // Floating point flag ops. let SchedRW = [WriteALU] in { let Defs = [AX], Uses = [FPSW] in -def FNSTSW16r : I<0xE0, RawFrm, // AX = fp flags +def FNSTSW16r : I<0xDF, MRM_E0, // AX = fp flags (outs), (ins), "fnstsw\t{%ax|ax}", - [(set AX, (X86fp_stsw FPSW))], IIC_FNSTSW>, DF; + [(set AX, (X86fp_stsw FPSW))], IIC_FNSTSW>; def FNSTCW16m : I<0xD9, MRM7m, // [mem16] = X87 control world (outs), (ins i16mem:$dst), "fnstcw\t$dst", @@ -593,50 +601,50 @@ def FLDCW16m : I<0xD9, MRM5m, // X87 control world = [mem16] // FPU control instructions let SchedRW = [WriteMicrocoded] in { let Defs = [FPSW] in -def FNINIT : I<0xE3, RawFrm, (outs), (ins), "fninit", [], IIC_FNINIT>, DB; -def FFREE : FPI<0xC0, AddRegFrm, (outs), (ins RST:$reg), - "ffree\t$reg", IIC_FFREE>, DD; +def FNINIT : I<0xDB, MRM_E3, (outs), (ins), "fninit", [], IIC_FNINIT>; +def FFREE : FPI<0xDD, MRM0r, (outs), (ins RST:$reg), + "ffree\t$reg", IIC_FFREE>; // Clear exceptions let Defs = [FPSW] in -def FNCLEX : I<0xE2, RawFrm, (outs), (ins), "fnclex", [], IIC_FNCLEX>, DB; +def FNCLEX : I<0xDB, MRM_E2, (outs), (ins), "fnclex", [], IIC_FNCLEX>; } // SchedRW // Operandless floating-point instructions for the disassembler. let SchedRW = [WriteMicrocoded] in { def WAIT : I<0x9B, RawFrm, (outs), (ins), "wait", [], IIC_WAIT>; -def FNOP : I<0xD0, RawFrm, (outs), (ins), "fnop", [], IIC_FNOP>, D9; -def FXAM : I<0xE5, RawFrm, (outs), (ins), "fxam", [], IIC_FXAM>, D9; -def FLDL2T : I<0xE9, RawFrm, (outs), (ins), "fldl2t", [], IIC_FLDL>, D9; -def FLDL2E : I<0xEA, RawFrm, (outs), (ins), "fldl2e", [], IIC_FLDL>, D9; -def FLDPI : I<0xEB, RawFrm, (outs), (ins), "fldpi", [], IIC_FLDL>, D9; -def FLDLG2 : I<0xEC, RawFrm, (outs), (ins), "fldlg2", [], IIC_FLDL>, D9; -def FLDLN2 : I<0xED, RawFrm, (outs), (ins), "fldln2", [], IIC_FLDL>, D9; -def F2XM1 : I<0xF0, RawFrm, (outs), (ins), "f2xm1", [], IIC_F2XM1>, D9; -def FYL2X : I<0xF1, RawFrm, (outs), (ins), "fyl2x", [], IIC_FYL2X>, D9; -def FPTAN : I<0xF2, RawFrm, (outs), (ins), "fptan", [], IIC_FPTAN>, D9; -def FPATAN : I<0xF3, RawFrm, (outs), (ins), "fpatan", [], IIC_FPATAN>, D9; -def FXTRACT : I<0xF4, RawFrm, (outs), (ins), "fxtract", [], IIC_FXTRACT>, D9; -def FPREM1 : I<0xF5, RawFrm, (outs), (ins), "fprem1", [], IIC_FPREM1>, D9; -def FDECSTP : I<0xF6, RawFrm, (outs), (ins), "fdecstp", [], IIC_FPSTP>, D9; -def FINCSTP : I<0xF7, RawFrm, (outs), (ins), "fincstp", [], IIC_FPSTP>, D9; -def FPREM : I<0xF8, RawFrm, (outs), (ins), "fprem", [], IIC_FPREM>, D9; -def FYL2XP1 : I<0xF9, RawFrm, (outs), (ins), "fyl2xp1", [], IIC_FYL2XP1>, D9; -def FSINCOS : I<0xFB, RawFrm, (outs), (ins), "fsincos", [], IIC_FSINCOS>, D9; -def FRNDINT : I<0xFC, RawFrm, (outs), (ins), "frndint", [], IIC_FRNDINT>, D9; -def FSCALE : I<0xFD, RawFrm, (outs), (ins), "fscale", [], IIC_FSCALE>, D9; -def FCOMPP : I<0xD9, RawFrm, (outs), (ins), "fcompp", [], IIC_FCOMPP>, DE; +def FNOP : I<0xD9, MRM_D0, (outs), (ins), "fnop", [], IIC_FNOP>; +def FXAM : I<0xD9, MRM_E5, (outs), (ins), "fxam", [], IIC_FXAM>; +def FLDL2T : I<0xD9, MRM_E9, (outs), (ins), "fldl2t", [], IIC_FLDL>; +def FLDL2E : I<0xD9, MRM_EA, (outs), (ins), "fldl2e", [], IIC_FLDL>; +def FLDPI : I<0xD9, MRM_EB, (outs), (ins), "fldpi", [], IIC_FLDL>; +def FLDLG2 : I<0xD9, MRM_EC, (outs), (ins), "fldlg2", [], IIC_FLDL>; +def FLDLN2 : I<0xD9, MRM_ED, (outs), (ins), "fldln2", [], IIC_FLDL>; +def F2XM1 : I<0xD9, MRM_F0, (outs), (ins), "f2xm1", [], IIC_F2XM1>; +def FYL2X : I<0xD9, MRM_F1, (outs), (ins), "fyl2x", [], IIC_FYL2X>; +def FPTAN : I<0xD9, MRM_F2, (outs), (ins), "fptan", [], IIC_FPTAN>; +def FPATAN : I<0xD9, MRM_F3, (outs), (ins), "fpatan", [], IIC_FPATAN>; +def FXTRACT : I<0xD9, MRM_F4, (outs), (ins), "fxtract", [], IIC_FXTRACT>; +def FPREM1 : I<0xD9, MRM_F5, (outs), (ins), "fprem1", [], IIC_FPREM1>; +def FDECSTP : I<0xD9, MRM_F6, (outs), (ins), "fdecstp", [], IIC_FPSTP>; +def FINCSTP : I<0xD9, MRM_F7, (outs), (ins), "fincstp", [], IIC_FPSTP>; +def FPREM : I<0xD9, MRM_F8, (outs), (ins), "fprem", [], IIC_FPREM>; +def FYL2XP1 : I<0xD9, MRM_F9, (outs), (ins), "fyl2xp1", [], IIC_FYL2XP1>; +def FSINCOS : I<0xD9, MRM_FB, (outs), (ins), "fsincos", [], IIC_FSINCOS>; +def FRNDINT : I<0xD9, MRM_FC, (outs), (ins), "frndint", [], IIC_FRNDINT>; +def FSCALE : I<0xD9, MRM_FD, (outs), (ins), "fscale", [], IIC_FSCALE>; +def FCOMPP : I<0xDE, MRM_D9, (outs), (ins), "fcompp", [], IIC_FCOMPP>; def FXSAVE : I<0xAE, MRM0m, (outs opaque512mem:$dst), (ins), "fxsave\t$dst", [], IIC_FXSAVE>, TB; -def FXSAVE64 : I<0xAE, MRM0m, (outs opaque512mem:$dst), (ins), - "fxsaveq\t$dst", [], IIC_FXSAVE>, TB, REX_W, - Requires<[In64BitMode]>; +def FXSAVE64 : RI<0xAE, MRM0m, (outs opaque512mem:$dst), (ins), + "fxsave{q|64}\t$dst", [], IIC_FXSAVE>, TB, + Requires<[In64BitMode]>; def FXRSTOR : I<0xAE, MRM1m, (outs), (ins opaque512mem:$src), "fxrstor\t$src", [], IIC_FXRSTOR>, TB; -def FXRSTOR64 : I<0xAE, MRM1m, (outs), (ins opaque512mem:$src), - "fxrstorq\t$src", [], IIC_FXRSTOR>, TB, REX_W, +def FXRSTOR64 : RI<0xAE, MRM1m, (outs), (ins opaque512mem:$src), + "fxrstor{q|64}\t$src", [], IIC_FXRSTOR>, TB, Requires<[In64BitMode]>; } // SchedRW diff --git a/lib/Target/X86/X86InstrFormats.td b/lib/Target/X86/X86InstrFormats.td index 0fd9011..cc30266 100644 --- a/lib/Target/X86/X86InstrFormats.td +++ b/lib/Target/X86/X86InstrFormats.td @@ -14,54 +14,48 @@ // Format specifies the encoding used by the instruction. This is part of the // ad-hoc solution used to emit machine instruction encodings by our machine // code emitter. -class Format<bits<6> val> { - bits<6> Value = val; +class Format<bits<7> val> { + bits<7> Value = val; } def Pseudo : Format<0>; def RawFrm : Format<1>; def AddRegFrm : Format<2>; def MRMDestReg : Format<3>; def MRMDestMem : Format<4>; def MRMSrcReg : Format<5>; -def MRMSrcMem : Format<6>; +def MRMSrcMem : Format<6>; def RawFrmMemOffs : Format<7>; +def RawFrmSrc : Format<8>; def RawFrmDst : Format<9>; +def RawFrmDstSrc: Format<10>; +def RawFrmImm8 : Format<11>; +def RawFrmImm16 : Format<12>; +def MRMXr : Format<14>; def MRMXm : Format<15>; def MRM0r : Format<16>; def MRM1r : Format<17>; def MRM2r : Format<18>; def MRM3r : Format<19>; def MRM4r : Format<20>; def MRM5r : Format<21>; def MRM6r : Format<22>; def MRM7r : Format<23>; def MRM0m : Format<24>; def MRM1m : Format<25>; def MRM2m : Format<26>; def MRM3m : Format<27>; def MRM4m : Format<28>; def MRM5m : Format<29>; def MRM6m : Format<30>; def MRM7m : Format<31>; -def MRMInitReg : Format<32>; -def MRM_C1 : Format<33>; -def MRM_C2 : Format<34>; -def MRM_C3 : Format<35>; -def MRM_C4 : Format<36>; -def MRM_C8 : Format<37>; -def MRM_C9 : Format<38>; -def MRM_CA : Format<39>; -def MRM_CB : Format<40>; -def MRM_E8 : Format<41>; -def MRM_F0 : Format<42>; -def RawFrmImm8 : Format<43>; -def RawFrmImm16 : Format<44>; -def MRM_F8 : Format<45>; -def MRM_F9 : Format<46>; -def MRM_D0 : Format<47>; -def MRM_D1 : Format<48>; -def MRM_D4 : Format<49>; -def MRM_D5 : Format<50>; -def MRM_D6 : Format<51>; -def MRM_D8 : Format<52>; -def MRM_D9 : Format<53>; -def MRM_DA : Format<54>; -def MRM_DB : Format<55>; -def MRM_DC : Format<56>; -def MRM_DD : Format<57>; -def MRM_DE : Format<58>; -def MRM_DF : Format<59>; +def MRM_C0 : Format<32>; def MRM_C1 : Format<33>; def MRM_C2 : Format<34>; +def MRM_C3 : Format<35>; def MRM_C4 : Format<36>; def MRM_C8 : Format<37>; +def MRM_C9 : Format<38>; def MRM_CA : Format<39>; def MRM_CB : Format<40>; +def MRM_D0 : Format<41>; def MRM_D1 : Format<42>; def MRM_D4 : Format<43>; +def MRM_D5 : Format<44>; def MRM_D6 : Format<45>; def MRM_D8 : Format<46>; +def MRM_D9 : Format<47>; def MRM_DA : Format<48>; def MRM_DB : Format<49>; +def MRM_DC : Format<50>; def MRM_DD : Format<51>; def MRM_DE : Format<52>; +def MRM_DF : Format<53>; def MRM_E0 : Format<54>; def MRM_E1 : Format<55>; +def MRM_E2 : Format<56>; def MRM_E3 : Format<57>; def MRM_E4 : Format<58>; +def MRM_E5 : Format<59>; def MRM_E8 : Format<60>; def MRM_E9 : Format<61>; +def MRM_EA : Format<62>; def MRM_EB : Format<63>; def MRM_EC : Format<64>; +def MRM_ED : Format<65>; def MRM_EE : Format<66>; def MRM_F0 : Format<67>; +def MRM_F1 : Format<68>; def MRM_F2 : Format<69>; def MRM_F3 : Format<70>; +def MRM_F4 : Format<71>; def MRM_F5 : Format<72>; def MRM_F6 : Format<73>; +def MRM_F7 : Format<74>; def MRM_F8 : Format<75>; def MRM_F9 : Format<76>; +def MRM_FA : Format<77>; def MRM_FB : Format<78>; def MRM_FC : Format<79>; +def MRM_FD : Format<80>; def MRM_FE : Format<81>; def MRM_FF : Format<82>; // ImmType - This specifies the immediate type used by an instruction. This is // part of the ad-hoc solution used to emit machine instruction encodings by our // machine code emitter. -class ImmType<bits<3> val> { - bits<3> Value = val; +class ImmType<bits<4> val> { + bits<4> Value = val; } def NoImm : ImmType<0>; def Imm8 : ImmType<1>; @@ -70,7 +64,8 @@ def Imm16 : ImmType<3>; def Imm16PCRel : ImmType<4>; def Imm32 : ImmType<5>; def Imm32PCRel : ImmType<6>; -def Imm64 : ImmType<7>; +def Imm32S : ImmType<7>; +def Imm64 : ImmType<8>; // FPFormat - This specifies what form this FP instruction has. This is used by // the Floating-Point stackifier pass. @@ -110,48 +105,84 @@ def CD8VT2 : CD8VForm<5>; // v := 2 def CD8VT4 : CD8VForm<6>; // v := 4 def CD8VT8 : CD8VForm<7>; // v := 8 +// Class specifying the prefix used an opcode extension. +class Prefix<bits<3> val> { + bits<3> Value = val; +} +def NoPrfx : Prefix<0>; +def PS : Prefix<1>; +def PD : Prefix<2>; +def XS : Prefix<3>; +def XD : Prefix<4>; + +// Class specifying the opcode map. +class Map<bits<3> val> { + bits<3> Value = val; +} +def OB : Map<0>; +def TB : Map<1>; +def T8 : Map<2>; +def TA : Map<3>; +def XOP8 : Map<4>; +def XOP9 : Map<5>; +def XOPA : Map<6>; + +// Class specifying the encoding +class Encoding<bits<2> val> { + bits<2> Value = val; +} +def EncNormal : Encoding<0>; +def EncVEX : Encoding<1>; +def EncXOP : Encoding<2>; +def EncEVEX : Encoding<3>; + +// Operand size for encodings that change based on mode. +class OperandSize<bits<2> val> { + bits<2> Value = val; +} +def OpSizeFixed : OperandSize<0>; // Never needs a 0x66 prefix. +def OpSize16 : OperandSize<1>; // Needs 0x66 prefix in 32-bit mode. +def OpSize32 : OperandSize<2>; // Needs 0x66 prefix in 16-bit mode. + // Prefix byte classes which are used to indicate to the ad-hoc machine code // emitter that various prefix bytes are required. -class OpSize { bit hasOpSizePrefix = 1; } +class OpSize16 { OperandSize OpSize = OpSize16; } +class OpSize32 { OperandSize OpSize = OpSize32; } class AdSize { bit hasAdSizePrefix = 1; } class REX_W { bit hasREX_WPrefix = 1; } class LOCK { bit hasLockPrefix = 1; } -class SegFS { bits<2> SegOvrBits = 1; } -class SegGS { bits<2> SegOvrBits = 2; } -class TB { bits<5> Prefix = 1; } -class REP { bits<5> Prefix = 2; } -class D8 { bits<5> Prefix = 3; } -class D9 { bits<5> Prefix = 4; } -class DA { bits<5> Prefix = 5; } -class DB { bits<5> Prefix = 6; } -class DC { bits<5> Prefix = 7; } -class DD { bits<5> Prefix = 8; } -class DE { bits<5> Prefix = 9; } -class DF { bits<5> Prefix = 10; } -class XD { bits<5> Prefix = 11; } -class XS { bits<5> Prefix = 12; } -class T8 { bits<5> Prefix = 13; } -class TA { bits<5> Prefix = 14; } -class A6 { bits<5> Prefix = 15; } -class A7 { bits<5> Prefix = 16; } -class T8XD { bits<5> Prefix = 17; } -class T8XS { bits<5> Prefix = 18; } -class TAXD { bits<5> Prefix = 19; } -class XOP8 { bits<5> Prefix = 20; } -class XOP9 { bits<5> Prefix = 21; } -class XOPA { bits<5> Prefix = 22; } -class VEX { bit hasVEXPrefix = 1; } +class REP { bit hasREPPrefix = 1; } +class TB { Map OpMap = TB; } +class T8 { Map OpMap = T8; } +class TA { Map OpMap = TA; } +class XOP8 { Map OpMap = XOP8; Prefix OpPrefix = PS; } +class XOP9 { Map OpMap = XOP9; Prefix OpPrefix = PS; } +class XOPA { Map OpMap = XOPA; Prefix OpPrefix = PS; } +class OBXS { Prefix OpPrefix = XS; } +class PS : TB { Prefix OpPrefix = PS; } +class PD : TB { Prefix OpPrefix = PD; } +class XD : TB { Prefix OpPrefix = XD; } +class XS : TB { Prefix OpPrefix = XS; } +class T8PS : T8 { Prefix OpPrefix = PS; } +class T8PD : T8 { Prefix OpPrefix = PD; } +class T8XD : T8 { Prefix OpPrefix = XD; } +class T8XS : T8 { Prefix OpPrefix = XS; } +class TAPS : TA { Prefix OpPrefix = PS; } +class TAPD : TA { Prefix OpPrefix = PD; } +class TAXD : TA { Prefix OpPrefix = XD; } +class VEX { Encoding OpEnc = EncVEX; } class VEX_W { bit hasVEX_WPrefix = 1; } -class VEX_4V : VEX { bit hasVEX_4VPrefix = 1; } -class VEX_4VOp3 : VEX { bit hasVEX_4VOp3Prefix = 1; } +class VEX_4V : VEX { bit hasVEX_4V = 1; } +class VEX_4VOp3 : VEX { bit hasVEX_4VOp3 = 1; } class VEX_I8IMM { bit hasVEX_i8ImmReg = 1; } class VEX_L { bit hasVEX_L = 1; } class VEX_LIG { bit ignoresVEX_L = 1; } -class EVEX : VEX { bit hasEVEXPrefix = 1; } -class EVEX_4V : VEX_4V { bit hasEVEXPrefix = 1; } +class EVEX : VEX { Encoding OpEnc = EncEVEX; } +class EVEX_4V : VEX_4V { Encoding OpEnc = EncEVEX; } class EVEX_K { bit hasEVEX_K = 1; } class EVEX_KZ : EVEX_K { bit hasEVEX_Z = 1; } class EVEX_B { bit hasEVEX_B = 1; } +class EVEX_RC { bit hasEVEX_RC = 1; } class EVEX_V512 { bit hasEVEX_L2 = 1; bit hasVEX_L = 0; } class EVEX_CD8<int esize, CD8VForm form> { bits<2> EVEX_CD8E = !if(!eq(esize, 8), 0b00, @@ -162,7 +193,10 @@ class EVEX_CD8<int esize, CD8VForm form> { } class Has3DNow0F0FOpcode { bit has3DNow0F0FOpcode = 1; } class MemOp4 { bit hasMemOp4Prefix = 1; } -class XOP { bit hasXOP_Prefix = 1; } +class XOP { Encoding OpEnc = EncXOP; } +class XOP_4V : XOP { bit hasVEX_4V = 1; } +class XOP_4VOp3 : XOP { bit hasVEX_4VOp3 = 1; } + class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, string AsmStr, InstrItinClass itin, @@ -172,7 +206,7 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, bits<8> Opcode = opcod; Format Form = f; - bits<6> FormBits = Form.Value; + bits<7> FormBits = Form.Value; ImmType ImmT = i; dag OutOperandList = outs; @@ -187,25 +221,34 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, // // Attributes specific to X86 instructions... // - bit hasOpSizePrefix = 0; // Does this inst have a 0x66 prefix? + bit ForceDisassemble = 0; // Force instruction to disassemble even though it's + // isCodeGenonly. Needed to hide an ambiguous + // AsmString from the parser, but still disassemble. + + OperandSize OpSize = OpSizeFixed; // Does this instruction's encoding change + // based on operand size of the mode + bits<2> OpSizeBits = OpSize.Value; bit hasAdSizePrefix = 0; // Does this inst have a 0x67 prefix? - bits<5> Prefix = 0; // Which prefix byte does this inst have? + Prefix OpPrefix = NoPrfx; // Which prefix byte does this inst have? + bits<3> OpPrefixBits = OpPrefix.Value; + Map OpMap = OB; // Which opcode map does this inst have? + bits<3> OpMapBits = OpMap.Value; bit hasREX_WPrefix = 0; // Does this inst require the REX.W prefix? FPFormat FPForm = NotFP; // What flavor of FP instruction is this? bit hasLockPrefix = 0; // Does this inst have a 0xF0 prefix? - bits<2> SegOvrBits = 0; // Segment override prefix. Domain ExeDomain = d; - bit hasVEXPrefix = 0; // Does this inst require a VEX prefix? + bit hasREPPrefix = 0; // Does this inst have a REP prefix? + Encoding OpEnc = EncNormal; // Encoding used by this instruction + bits<2> OpEncBits = OpEnc.Value; bit hasVEX_WPrefix = 0; // Does this inst set the VEX_W field? - bit hasVEX_4VPrefix = 0; // Does this inst require the VEX.VVVV field? - bit hasVEX_4VOp3Prefix = 0; // Does this inst require the VEX.VVVV field to - // encode the third operand? + bit hasVEX_4V = 0; // Does this inst require the VEX.VVVV field? + bit hasVEX_4VOp3 = 0; // Does this inst require the VEX.VVVV field to + // encode the third operand? bit hasVEX_i8ImmReg = 0; // Does this inst require the last source register // to be encoded in a immediate field? bit hasVEX_L = 0; // Does this inst use large (256-bit) registers? bit ignoresVEX_L = 0; // Does this instruction ignore the L-bit - bit hasEVEXPrefix = 0; // Does this inst require EVEX form? bit hasEVEX_K = 0; // Does this inst require masking? bit hasEVEX_Z = 0; // Does this inst set the EVEX_Z field? bit hasEVEX_L2 = 0; // Does this inst set the EVEX_L2 field? @@ -214,37 +257,37 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, bits<3> EVEX_CD8V = 0; // Compressed disp8 form - vector-width. bit has3DNow0F0FOpcode =0;// Wacky 3dNow! encoding? bit hasMemOp4Prefix = 0; // Same bit as VEX_W, but used for swapping operands - bit hasXOP_Prefix = 0; // Does this inst require an XOP prefix? + bit hasEVEX_RC = 0; // Explicitly specified rounding control in FP instruction. // TSFlags layout should be kept in sync with X86InstrInfo.h. - let TSFlags{5-0} = FormBits; - let TSFlags{6} = hasOpSizePrefix; - let TSFlags{7} = hasAdSizePrefix; - let TSFlags{12-8} = Prefix; - let TSFlags{13} = hasREX_WPrefix; - let TSFlags{16-14} = ImmT.Value; - let TSFlags{19-17} = FPForm.Value; - let TSFlags{20} = hasLockPrefix; - let TSFlags{22-21} = SegOvrBits; - let TSFlags{24-23} = ExeDomain.Value; - let TSFlags{32-25} = Opcode; - let TSFlags{33} = hasVEXPrefix; - let TSFlags{34} = hasVEX_WPrefix; - let TSFlags{35} = hasVEX_4VPrefix; - let TSFlags{36} = hasVEX_4VOp3Prefix; - let TSFlags{37} = hasVEX_i8ImmReg; - let TSFlags{38} = hasVEX_L; - let TSFlags{39} = ignoresVEX_L; - let TSFlags{40} = hasEVEXPrefix; - let TSFlags{41} = hasEVEX_K; - let TSFlags{42} = hasEVEX_Z; - let TSFlags{43} = hasEVEX_L2; - let TSFlags{44} = hasEVEX_B; - let TSFlags{46-45} = EVEX_CD8E; - let TSFlags{49-47} = EVEX_CD8V; - let TSFlags{50} = has3DNow0F0FOpcode; - let TSFlags{51} = hasMemOp4Prefix; - let TSFlags{52} = hasXOP_Prefix; + let TSFlags{6-0} = FormBits; + let TSFlags{8-7} = OpSizeBits; + let TSFlags{9} = hasAdSizePrefix; + let TSFlags{12-10} = OpPrefixBits; + let TSFlags{15-13} = OpMapBits; + let TSFlags{16} = hasREX_WPrefix; + let TSFlags{20-17} = ImmT.Value; + let TSFlags{23-21} = FPForm.Value; + let TSFlags{24} = hasLockPrefix; + let TSFlags{25} = hasREPPrefix; + let TSFlags{27-26} = ExeDomain.Value; + let TSFlags{29-28} = OpEncBits; + let TSFlags{37-30} = Opcode; + let TSFlags{38} = hasVEX_WPrefix; + let TSFlags{39} = hasVEX_4V; + let TSFlags{40} = hasVEX_4VOp3; + let TSFlags{41} = hasVEX_i8ImmReg; + let TSFlags{42} = hasVEX_L; + let TSFlags{43} = ignoresVEX_L; + let TSFlags{44} = hasEVEX_K; + let TSFlags{45} = hasEVEX_Z; + let TSFlags{46} = hasEVEX_L2; + let TSFlags{47} = hasEVEX_B; + let TSFlags{49-48} = EVEX_CD8E; + let TSFlags{52-50} = EVEX_CD8V; + let TSFlags{53} = has3DNow0F0FOpcode; + let TSFlags{54} = hasMemOp4Prefix; + let TSFlags{55} = hasEVEX_RC; } class PseudoI<dag oops, dag iops, list<dag> pattern> @@ -284,6 +327,12 @@ class Ii32<bits<8> o, Format f, dag outs, dag ins, string asm, let Pattern = pattern; let CodeSize = 3; } +class Ii32S<bits<8> o, Format f, dag outs, dag ins, string asm, + list<dag> pattern, InstrItinClass itin = NoItinerary> + : X86Inst<o, f, Imm32S, outs, ins, asm, itin> { + let Pattern = pattern; + let CodeSize = 3; +} class Ii16PCRel<bits<8> o, Format f, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> @@ -333,73 +382,83 @@ class Iseg32 <bits<8> o, Format f, dag outs, dag ins, string asm, let CodeSize = 3; } -def __xs : XS; -def __xd : XD; - // SI - SSE 1 & 2 scalar instructions class SI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : I<o, F, outs, ins, asm, pattern, itin> { - let Predicates = !if(hasEVEXPrefix /* EVEX */, [HasAVX512], - !if(hasVEXPrefix /* VEX */, [UseAVX], - !if(!eq(Prefix, __xs.Prefix), [UseSSE1], - !if(!eq(Prefix, __xd.Prefix), [UseSSE2], - !if(hasOpSizePrefix, [UseSSE2], [UseSSE1]))))); + let Predicates = !if(!eq(OpEnc.Value, EncEVEX.Value), [HasAVX512], + !if(!eq(OpEnc.Value, EncVEX.Value), [UseAVX], + !if(!eq(OpPrefix.Value, XS.Value), [UseSSE1], + !if(!eq(OpPrefix.Value, XD.Value), [UseSSE2], + !if(!eq(OpPrefix.Value, PD.Value), [UseSSE2], + [UseSSE1]))))); // AVX instructions have a 'v' prefix in the mnemonic - let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm); + let AsmString = !if(!eq(OpEnc.Value, EncEVEX.Value), !strconcat("v", asm), + !if(!eq(OpEnc.Value, EncVEX.Value), !strconcat("v", asm), + asm)); } // SIi8 - SSE 1 & 2 scalar instructions class SIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : Ii8<o, F, outs, ins, asm, pattern, itin> { - let Predicates = !if(hasEVEXPrefix /* EVEX */, [HasAVX512], - !if(hasVEXPrefix /* VEX */, [UseAVX], - !if(!eq(Prefix, __xs.Prefix), [UseSSE1], [UseSSE2]))); + let Predicates = !if(!eq(OpEnc.Value, EncEVEX.Value), [HasAVX512], + !if(!eq(OpEnc.Value, EncVEX.Value), [UseAVX], + !if(!eq(OpPrefix.Value, XS.Value), [UseSSE1], + [UseSSE2]))); // AVX instructions have a 'v' prefix in the mnemonic - let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm); + let AsmString = !if(!eq(OpEnc.Value, EncEVEX.Value), !strconcat("v", asm), + !if(!eq(OpEnc.Value, EncVEX.Value), !strconcat("v", asm), + asm)); } // PI - SSE 1 & 2 packed instructions class PI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin, Domain d> : I<o, F, outs, ins, asm, pattern, itin, d> { - let Predicates = !if(hasEVEXPrefix /* EVEX */, [HasAVX512], - !if(hasVEXPrefix /* VEX */, [HasAVX], - !if(hasOpSizePrefix /* OpSize */, [UseSSE2], [UseSSE1]))); + let Predicates = !if(!eq(OpEnc.Value, EncEVEX.Value), [HasAVX512], + !if(!eq(OpEnc.Value, EncVEX.Value), [HasAVX], + !if(!eq(OpPrefix.Value, PD.Value), [UseSSE2], + [UseSSE1]))); // AVX instructions have a 'v' prefix in the mnemonic - let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm); + let AsmString = !if(!eq(OpEnc.Value, EncEVEX.Value), !strconcat("v", asm), + !if(!eq(OpEnc.Value, EncVEX.Value), !strconcat("v", asm), + asm)); } // MMXPI - SSE 1 & 2 packed instructions with MMX operands class MMXPI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin, Domain d> : I<o, F, outs, ins, asm, pattern, itin, d> { - let Predicates = !if(hasOpSizePrefix /* OpSize */, [HasSSE2], [HasSSE1]); + let Predicates = !if(!eq(OpPrefix.Value, PD.Value), [HasSSE2], + [HasSSE1]); } // PIi8 - SSE 1 & 2 packed instructions with immediate class PIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin, Domain d> : Ii8<o, F, outs, ins, asm, pattern, itin, d> { - let Predicates = !if(hasEVEXPrefix /* EVEX */, [HasAVX512], - !if(hasVEXPrefix /* VEX */, [HasAVX], - !if(hasOpSizePrefix /* OpSize */, [UseSSE2], [UseSSE1]))); + let Predicates = !if(!eq(OpEnc.Value, EncEVEX.Value), [HasAVX512], + !if(!eq(OpEnc.Value, EncVEX.Value), [HasAVX], + !if(!eq(OpPrefix.Value, PD.Value), [UseSSE2], + [UseSSE1]))); // AVX instructions have a 'v' prefix in the mnemonic - let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm); + let AsmString = !if(!eq(OpEnc.Value, EncEVEX.Value), !strconcat("v", asm), + !if(!eq(OpEnc.Value, EncVEX.Value), !strconcat("v", asm), + asm)); } // SSE1 Instruction Templates: // // SSI - SSE1 instructions with XS prefix. -// PSI - SSE1 instructions with TB prefix. -// PSIi8 - SSE1 instructions with ImmT == Imm8 and TB prefix. +// PSI - SSE1 instructions with PS prefix. +// PSIi8 - SSE1 instructions with ImmT == Imm8 and PS prefix. // VSSI - SSE1 instructions with XS prefix in AVX form. -// VPSI - SSE1 instructions with TB prefix in AVX form, packed single. +// VPSI - SSE1 instructions with PS prefix in AVX form, packed single. class SSI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> @@ -409,11 +468,11 @@ class SSIi8<bits<8> o, Format F, dag outs, dag ins, string asm, : Ii8<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[UseSSE1]>; class PSI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, TB, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, PS, Requires<[UseSSE1]>; class PSIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, TB, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, PS, Requires<[UseSSE1]>; class VSSI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> @@ -421,7 +480,7 @@ class VSSI<bits<8> o, Format F, dag outs, dag ins, string asm, Requires<[HasAVX]>; class VPSI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin, SSEPackedSingle>, TB, + : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin, SSEPackedSingle>, PS, Requires<[HasAVX]>; // SSE2 Instruction Templates: @@ -430,13 +489,13 @@ class VPSI<bits<8> o, Format F, dag outs, dag ins, string asm, // SDIi8 - SSE2 instructions with ImmT == Imm8 and XD prefix. // S2SI - SSE2 instructions with XS prefix. // SSDIi8 - SSE2 instructions with ImmT == Imm8 and XS prefix. -// PDI - SSE2 instructions with TB and OpSize prefixes, packed double domain. -// PDIi8 - SSE2 instructions with ImmT == Imm8 and TB and OpSize prefixes. +// PDI - SSE2 instructions with PD prefix, packed double domain. +// PDIi8 - SSE2 instructions with ImmT == Imm8 and PD prefix. // VSDI - SSE2 scalar instructions with XD prefix in AVX form. -// VPDI - SSE2 vector instructions with TB and OpSize prefixes in AVX form, +// VPDI - SSE2 vector instructions with PD prefix in AVX form, // packed double domain. -// VS2I - SSE2 scalar instructions with TB and OpSize prefixes in AVX form. -// S2I - SSE2 scalar instructions with TB and OpSize prefixes. +// VS2I - SSE2 scalar instructions with PD prefix in AVX form. +// S2I - SSE2 scalar instructions with PD prefix. // MMXSDIi8 - SSE2 instructions with ImmT == Imm8 and XD prefix as well as // MMX operands. // MMXSSDIi8 - SSE2 instructions with ImmT == Imm8 and XS prefix as well as @@ -456,11 +515,11 @@ class S2SIi8<bits<8> o, Format F, dag outs, dag ins, string asm, : Ii8<o, F, outs, ins, asm, pattern>, XS, Requires<[UseSSE2]>; class PDI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, OpSize, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, PD, Requires<[UseSSE2]>; class PDIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, OpSize, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, PD, Requires<[UseSSE2]>; class VSDI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> @@ -472,16 +531,15 @@ class VS2SI<bits<8> o, Format F, dag outs, dag ins, string asm, Requires<[HasAVX]>; class VPDI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin, SSEPackedDouble>, TB, - OpSize, Requires<[HasAVX]>; + : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin, SSEPackedDouble>, + PD, Requires<[HasAVX]>; class VS2I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin>, TB, - OpSize, Requires<[UseAVX]>; + : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin>, PD, + Requires<[UseAVX]>; class S2I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, TB, - OpSize, Requires<[UseSSE2]>; + : I<o, F, outs, ins, asm, pattern, itin>, PD, Requires<[UseSSE2]>; class MMXSDIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : Ii8<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[HasSSE2]>; @@ -491,7 +549,7 @@ class MMXS2SIi8<bits<8> o, Format F, dag outs, dag ins, string asm, // SSE3 Instruction Templates: // -// S3I - SSE3 instructions with TB and OpSize prefixes. +// S3I - SSE3 instructions with PD prefixes. // S3SI - SSE3 instructions with XS prefix. // S3DI - SSE3 instructions with XD prefix. @@ -505,7 +563,7 @@ class S3DI<bits<8> o, Format F, dag outs, dag ins, string asm, Requires<[UseSSE3]>; class S3I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, OpSize, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, PD, Requires<[UseSSE3]>; @@ -522,19 +580,19 @@ class S3I<bits<8> o, Format F, dag outs, dag ins, string asm, class SS38I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[UseSSSE3]>; class SS3AI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[UseSSSE3]>; class MMXSS38I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PS, Requires<[HasSSSE3]>; class MMXSS3AI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPS, Requires<[HasSSSE3]>; // SSE4.1 Instruction Templates: @@ -544,11 +602,11 @@ class MMXSS3AI<bits<8> o, Format F, dag outs, dag ins, string asm, // class SS48I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[UseSSE41]>; class SS4AIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[UseSSE41]>; // SSE4.2 Instruction Templates: @@ -556,7 +614,7 @@ class SS4AIi8<bits<8> o, Format F, dag outs, dag ins, string asm, // SS428I - SSE 4.2 instructions with T8 prefix. class SS428I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[UseSSE42]>; // SS42FI - SSE 4.2 instructions with T8XD prefix. @@ -568,53 +626,53 @@ class SS42FI<bits<8> o, Format F, dag outs, dag ins, string asm, // SS42AI = SSE 4.2 instructions with TA prefix class SS42AI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[UseSSE42]>; // AVX Instruction Templates: // Instructions introduced in AVX (no SSE equivalent forms) // -// AVX8I - AVX instructions with T8 and OpSize prefix. -// AVXAIi8 - AVX instructions with TA, OpSize prefix and ImmT = Imm8. +// AVX8I - AVX instructions with T8PD prefix. +// AVXAIi8 - AVX instructions with TAPD prefix and ImmT = Imm8. class AVX8I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, OpSize, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[HasAVX]>; class AVXAIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, OpSize, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[HasAVX]>; // AVX2 Instruction Templates: // Instructions introduced in AVX2 (no SSE equivalent forms) // -// AVX28I - AVX2 instructions with T8 and OpSize prefix. -// AVX2AIi8 - AVX2 instructions with TA, OpSize prefix and ImmT = Imm8. +// AVX28I - AVX2 instructions with T8PD prefix. +// AVX2AIi8 - AVX2 instructions with TAPD prefix and ImmT = Imm8. class AVX28I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, OpSize, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[HasAVX2]>; class AVX2AIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, OpSize, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[HasAVX2]>; // AVX-512 Instruction Templates: // Instructions introduced in AVX-512 (no SSE equivalent forms) // -// AVX5128I - AVX-512 instructions with T8 and OpSize prefix. -// AVX512AIi8 - AVX-512 instructions with TA, OpSize prefix and ImmT = Imm8. -// AVX512PDI - AVX-512 instructions with TB, OpSize, double packed. -// AVX512PSI - AVX-512 instructions with TB, single packed. +// AVX5128I - AVX-512 instructions with T8PD prefix. +// AVX512AIi8 - AVX-512 instructions with TAPD prefix and ImmT = Imm8. +// AVX512PDI - AVX-512 instructions with PD, double packed. +// AVX512PSI - AVX-512 instructions with PS, single packed. // AVX512XS8I - AVX-512 instructions with T8 and XS prefixes. // AVX512XSI - AVX-512 instructions with XS prefix, generic domain. -// AVX512BI - AVX-512 instructions with TB, OpSize, int packed domain. -// AVX512SI - AVX-512 scalar instructions with TB and OpSize prefixes. +// AVX512BI - AVX-512 instructions with PD, int packed domain. +// AVX512SI - AVX-512 scalar instructions with PD prefix. class AVX5128I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, OpSize, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[HasAVX512]>; class AVX512XS8I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> @@ -630,42 +688,38 @@ class AVX512XDI<bits<8> o, Format F, dag outs, dag ins, string asm, Requires<[HasAVX512]>; class AVX512BI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TB, OpSize, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, PD, Requires<[HasAVX512]>; class AVX512BIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TB, OpSize, - Requires<[HasAVX512]>; -class AVX512SI<bits<8> o, Format F, dag outs, dag ins, string asm, - list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TB, OpSize, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, PD, Requires<[HasAVX512]>; class AVX512AIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, OpSize, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[HasAVX512]>; class AVX512Ii8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TB, - Requires<[HasAVX512]>; + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, + Requires<[HasAVX512]>; class AVX512PDI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, - OpSize, Requires<[HasAVX512]>; + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, PD, + Requires<[HasAVX512]>; class AVX512PSI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, TB, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, PS, Requires<[HasAVX512]>; class AVX512PIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, Domain d, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, d>, TB, Requires<[HasAVX512]>; + : Ii8<o, F, outs, ins, asm, pattern, itin, d>, Requires<[HasAVX512]>; class AVX512PI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, Domain d, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin, d>, TB, Requires<[HasAVX512]>; + : I<o, F, outs, ins, asm, pattern, itin, d>, Requires<[HasAVX512]>; class AVX512FMA3<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, T8, - OpSize, EVEX_4V, Requires<[HasAVX512]>; + : I<o, F, outs, ins, asm, pattern, itin>, T8PD, + EVEX_4V, Requires<[HasAVX512]>; // AES Instruction Templates: // @@ -673,54 +727,54 @@ class AVX512FMA3<bits<8> o, Format F, dag outs, dag ins, string asm, // These use the same encoding as the SSE4.2 T8 and TA encodings. class AES8I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = IIC_AES> - : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8, + : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8PD, Requires<[HasAES]>; class AESAI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, Requires<[HasAES]>; // PCLMUL Instruction Templates class PCLMULIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, - OpSize, Requires<[HasPCLMUL]>; + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, + Requires<[HasPCLMUL]>; class AVXPCLMULIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, - OpSize, VEX_4V, Requires<[HasAVX, HasPCLMUL]>; + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, + VEX_4V, Requires<[HasAVX, HasPCLMUL]>; // FMA3 Instruction Templates class FMA3<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, T8, - OpSize, VEX_4V, FMASC, Requires<[HasFMA]>; + : I<o, F, outs, ins, asm, pattern, itin>, T8PD, + VEX_4V, FMASC, Requires<[HasFMA]>; // FMA4 Instruction Templates class FMA4<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin>, TA, - OpSize, VEX_4V, VEX_I8IMM, FMASC, Requires<[HasFMA4]>; + : Ii8<o, F, outs, ins, asm, pattern, itin>, TAPD, + VEX_4V, VEX_I8IMM, FMASC, Requires<[HasFMA4]>; // XOP 2, 3 and 4 Operand Instruction Template class IXOP<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, - XOP, XOP9, Requires<[HasXOP]>; + XOP9, Requires<[HasXOP]>; // XOP 2, 3 and 4 Operand Instruction Templates with imm byte class IXOPi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, - XOP, XOP8, Requires<[HasXOP]>; + XOP8, Requires<[HasXOP]>; // XOP 5 operand instruction (VEX encoding!) class IXOP5<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag>pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA, - OpSize, VEX_4V, VEX_I8IMM, Requires<[HasXOP]>; + : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD, + VEX_4V, VEX_I8IMM, Requires<[HasXOP]>; // X86-64 Instruction templates... // @@ -731,9 +785,15 @@ class RI<bits<8> o, Format F, dag outs, dag ins, string asm, class RIi8 <bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : Ii8<o, F, outs, ins, asm, pattern, itin>, REX_W; +class RIi16 <bits<8> o, Format F, dag outs, dag ins, string asm, + list<dag> pattern, InstrItinClass itin = NoItinerary> + : Ii16<o, F, outs, ins, asm, pattern, itin>, REX_W; class RIi32 <bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : Ii32<o, F, outs, ins, asm, pattern, itin>, REX_W; +class RIi32S <bits<8> o, Format F, dag outs, dag ins, string asm, + list<dag> pattern, InstrItinClass itin = NoItinerary> + : Ii32S<o, F, outs, ins, asm, pattern, itin>, REX_W; class RIi64<bits<8> o, Format f, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> @@ -749,18 +809,6 @@ class RIi64_NOREX<bits<8> o, Format f, dag outs, dag ins, string asm, let CodeSize = 3; } -class RSSI<bits<8> o, Format F, dag outs, dag ins, string asm, - list<dag> pattern, InstrItinClass itin = NoItinerary> - : SSI<o, F, outs, ins, asm, pattern, itin>, REX_W; -class RSDI<bits<8> o, Format F, dag outs, dag ins, string asm, - list<dag> pattern, InstrItinClass itin = NoItinerary> - : SDI<o, F, outs, ins, asm, pattern, itin>, REX_W; -class RPDI<bits<8> o, Format F, dag outs, dag ins, string asm, - list<dag> pattern, InstrItinClass itin = NoItinerary> - : PDI<o, F, outs, ins, asm, pattern, itin>, REX_W; -class VRPDI<bits<8> o, Format F, dag outs, dag ins, string asm, - list<dag> pattern, InstrItinClass itin = NoItinerary> - : VPDI<o, F, outs, ins, asm, pattern, itin>, VEX_W; class RS2I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : S2I<o, F, outs, ins, asm, pattern, itin>, REX_W; @@ -774,29 +822,29 @@ class VRS2I<bits<8> o, Format F, dag outs, dag ins, string asm, // MMXI - MMX instructions with TB prefix. // MMXI32 - MMX instructions with TB prefix valid only in 32 bit mode. // MMXI64 - MMX instructions with TB prefix valid only in 64 bit mode. -// MMX2I - MMX / SSE2 instructions with TB and OpSize prefixes. -// MMXIi8 - MMX instructions with ImmT == Imm8 and TB prefix. -// MMXIi8 - MMX instructions with ImmT == Imm8 and TB prefix. +// MMX2I - MMX / SSE2 instructions with PD prefix. +// MMXIi8 - MMX instructions with ImmT == Imm8 and PS prefix. +// MMXIi8 - MMX instructions with ImmT == Imm8 and PS prefix. // MMXID - MMX instructions with XD prefix. // MMXIS - MMX instructions with XS prefix. class MMXI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, TB, Requires<[HasMMX]>; + : I<o, F, outs, ins, asm, pattern, itin>, PS, Requires<[HasMMX]>; class MMXI32<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, TB, Requires<[HasMMX,In32BitMode]>; + : I<o, F, outs, ins, asm, pattern, itin>, PS, Requires<[HasMMX,Not64BitMode]>; class MMXI64<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, TB, Requires<[HasMMX,In64BitMode]>; + : I<o, F, outs, ins, asm, pattern, itin>, PS, Requires<[HasMMX,In64BitMode]>; class MMXRI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, TB, REX_W, Requires<[HasMMX]>; + : I<o, F, outs, ins, asm, pattern, itin>, PS, REX_W, Requires<[HasMMX]>; class MMX2I<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : I<o, F, outs, ins, asm, pattern, itin>, TB, OpSize, Requires<[HasMMX]>; + : I<o, F, outs, ins, asm, pattern, itin>, PD, Requires<[HasMMX]>; class MMXIi8<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> - : Ii8<o, F, outs, ins, asm, pattern, itin>, TB, Requires<[HasMMX]>; + : Ii8<o, F, outs, ins, asm, pattern, itin>, PS, Requires<[HasMMX]>; class MMXID<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern, InstrItinClass itin = NoItinerary> : Ii8<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[HasMMX]>; diff --git a/lib/Target/X86/X86InstrFragmentsSIMD.td b/lib/Target/X86/X86InstrFragmentsSIMD.td index 1fed424..486e5a9 100644 --- a/lib/Target/X86/X86InstrFragmentsSIMD.td +++ b/lib/Target/X86/X86InstrFragmentsSIMD.td @@ -25,7 +25,8 @@ def bc_mmx : PatFrag<(ops node:$in), (x86mmx (bitconvert node:$in))>; def SDTX86FPShiftOp : SDTypeProfile<1, 2, [ SDTCisSameAs<0, 1>, SDTCisFP<0>, SDTCisInt<2> ]>; def SDTX86VFCMP : SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<1, 2>, - SDTCisFP<1>, SDTCisVT<3, i8>]>; + SDTCisFP<1>, SDTCisVT<3, i8>, + SDTCisVec<1>]>; def X86umin : SDNode<"X86ISD::UMIN", SDTIntBinOp>; def X86umax : SDNode<"X86ISD::UMAX", SDTIntBinOp>; @@ -59,8 +60,8 @@ def X86hadd : SDNode<"X86ISD::HADD", SDTIntBinOp>; def X86hsub : SDNode<"X86ISD::HSUB", SDTIntBinOp>; def X86comi : SDNode<"X86ISD::COMI", SDTX86CmpTest>; def X86ucomi : SDNode<"X86ISD::UCOMI", SDTX86CmpTest>; -def X86cmpss : SDNode<"X86ISD::FSETCCss", SDTX86Cmpss>; -def X86cmpsd : SDNode<"X86ISD::FSETCCsd", SDTX86Cmpsd>; +def X86cmps : SDNode<"X86ISD::FSETCC", SDTX86Cmps>; +//def X86cmpsd : SDNode<"X86ISD::FSETCCsd", SDTX86Cmpsd>; def X86pshufb : SDNode<"X86ISD::PSHUFB", SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>, SDTCisSameAs<0,2>]>>; @@ -86,38 +87,40 @@ def X86insrtps : SDNode<"X86ISD::INSERTPS", def X86vzmovl : SDNode<"X86ISD::VZEXT_MOVL", SDTypeProfile<1, 1, [SDTCisSameAs<0,1>]>>; -def X86vzmovly : SDNode<"X86ISD::VZEXT_MOVL", - SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, - SDTCisOpSmallerThanOp<1, 0> ]>>; - -def X86vsmovl : SDNode<"X86ISD::VSEXT_MOVL", - SDTypeProfile<1, 1, - [SDTCisVec<0>, SDTCisInt<1>, SDTCisInt<0>]>>; - def X86vzload : SDNode<"X86ISD::VZEXT_LOAD", SDTLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def X86vzext : SDNode<"X86ISD::VZEXT", SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, - SDTCisInt<0>, SDTCisInt<1>]>>; + SDTCisInt<0>, SDTCisInt<1>, + SDTCisOpSmallerThanOp<1, 0>]>>; def X86vsext : SDNode<"X86ISD::VSEXT", SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, - SDTCisInt<0>, SDTCisInt<1>]>>; + SDTCisInt<0>, SDTCisInt<1>, + SDTCisOpSmallerThanOp<1, 0>]>>; def X86vtrunc : SDNode<"X86ISD::VTRUNC", SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, - SDTCisInt<0>, SDTCisInt<1>]>>; + SDTCisInt<0>, SDTCisInt<1>, + SDTCisOpSmallerThanOp<0, 1>]>>; +def X86trunc : SDNode<"X86ISD::TRUNC", + SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisInt<1>, + SDTCisOpSmallerThanOp<0, 1>]>>; + def X86vtruncm : SDNode<"X86ISD::VTRUNCM", SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisInt<0>, SDTCisInt<1>, - SDTCisVec<2>, SDTCisInt<2>]>>; + SDTCisVec<2>, SDTCisInt<2>, + SDTCisOpSmallerThanOp<0, 2>]>>; def X86vfpext : SDNode<"X86ISD::VFPEXT", SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, - SDTCisFP<0>, SDTCisFP<1>]>>; + SDTCisFP<0>, SDTCisFP<1>, + SDTCisOpSmallerThanOp<1, 0>]>>; def X86vfpround: SDNode<"X86ISD::VFPROUND", SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, - SDTCisFP<0>, SDTCisFP<1>]>>; + SDTCisFP<0>, SDTCisFP<1>, + SDTCisOpSmallerThanOp<0, 1>]>>; def X86vshldq : SDNode<"X86ISD::VSHLDQ", SDTIntShiftOp>; def X86vshrdq : SDNode<"X86ISD::VSRLDQ", SDTIntShiftOp>; @@ -130,9 +133,15 @@ def X86IntCmpMask : SDTypeProfile<1, 2, def X86pcmpeqm : SDNode<"X86ISD::PCMPEQM", X86IntCmpMask, [SDNPCommutative]>; def X86pcmpgtm : SDNode<"X86ISD::PCMPGTM", X86IntCmpMask>; -def X86CmpMaskCC : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; +def X86CmpMaskCC : + SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisInt<0>, SDTCisVec<1>, + SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; +def X86CmpMaskCCScalar : + SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; + def X86cmpm : SDNode<"X86ISD::CMPM", X86CmpMaskCC>; def X86cmpmu : SDNode<"X86ISD::CMPMU", X86CmpMaskCC>; +def X86cmpms : SDNode<"X86ISD::FSETCC", X86CmpMaskCCScalar>; def X86vshl : SDNode<"X86ISD::VSHL", SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>, @@ -155,10 +164,13 @@ def X86subus : SDNode<"X86ISD::SUBUS", SDTIntBinOp>; def X86ptest : SDNode<"X86ISD::PTEST", SDTX86CmpPTest>; def X86testp : SDNode<"X86ISD::TESTP", SDTX86CmpPTest>; def X86kortest : SDNode<"X86ISD::KORTEST", SDTX86CmpPTest>; -def X86ktest : SDNode<"X86ISD::KTEST", SDTX86CmpPTest>; -def X86testm : SDNode<"X86ISD::TESTM", SDTypeProfile<1, 2, [SDTCisVec<0>, +def X86testm : SDNode<"X86ISD::TESTM", SDTypeProfile<1, 2, [SDTCisVec<0>, + SDTCisVec<1>, + SDTCisSameAs<2, 1>]>>; +def X86testnm : SDNode<"X86ISD::TESTNM", SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<2, 1>]>>; +def X86select : SDNode<"X86ISD::SELECT" , SDTSelect>; def X86pmuludq : SDNode<"X86ISD::PMULUDQ", SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, @@ -216,6 +228,7 @@ def X86VPermilp : SDNode<"X86ISD::VPERMILP", SDTShuff2OpI>; def X86VPermv : SDNode<"X86ISD::VPERMV", SDTShuff2Op>; def X86VPermi : SDNode<"X86ISD::VPERMI", SDTShuff2OpI>; def X86VPermv3 : SDNode<"X86ISD::VPERMV3", SDTShuff3Op>; +def X86VPermiv3 : SDNode<"X86ISD::VPERMIV3", SDTShuff3Op>; def X86VPerm2x128 : SDNode<"X86ISD::VPERM2X128", SDTShuff3OpI>; @@ -223,6 +236,8 @@ def X86VBroadcast : SDNode<"X86ISD::VBROADCAST", SDTVBroadcast>; def X86VBroadcastm : SDNode<"X86ISD::VBROADCASTM", SDTVBroadcastm>; def X86Vinsert : SDNode<"X86ISD::VINSERT", SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisPtrTy<3>]>, []>; +def X86Vextract : SDNode<"X86ISD::VEXTRACT", SDTypeProfile<1, 2, + [SDTCisVec<1>, SDTCisPtrTy<2>]>, []>; def X86Blendi : SDNode<"X86ISD::BLENDI", SDTBlend>; def X86Fmadd : SDNode<"X86ISD::FMADD", SDTFma>; @@ -458,10 +473,13 @@ def bc_v32i8 : PatFrag<(ops node:$in), (v32i8 (bitconvert node:$in))>; def bc_v16i16 : PatFrag<(ops node:$in), (v16i16 (bitconvert node:$in))>; def bc_v8i32 : PatFrag<(ops node:$in), (v8i32 (bitconvert node:$in))>; def bc_v4i64 : PatFrag<(ops node:$in), (v4i64 (bitconvert node:$in))>; +def bc_v8f32 : PatFrag<(ops node:$in), (v8f32 (bitconvert node:$in))>; // 512-bit bitconvert pattern fragments def bc_v16i32 : PatFrag<(ops node:$in), (v16i32 (bitconvert node:$in))>; def bc_v8i64 : PatFrag<(ops node:$in), (v8i64 (bitconvert node:$in))>; +def bc_v8f64 : PatFrag<(ops node:$in), (v8f64 (bitconvert node:$in))>; +def bc_v16f32 : PatFrag<(ops node:$in), (v16f32 (bitconvert node:$in))>; def vzmovl_v2i64 : PatFrag<(ops node:$src), (bitconvert (v2i64 (X86vzmovl @@ -478,6 +496,14 @@ def fp32imm0 : PatLeaf<(f32 fpimm), [{ return N->isExactlyValue(+0.0); }]>; +def I8Imm : SDNodeXForm<imm, [{ + // Transformation function: get the low 8 bits. + return getI8Imm((uint8_t)N->getZExtValue()); +}]>; + +def FROUND_NO_EXC : ImmLeaf<i32, [{ return Imm == 8; }]>; +def FROUND_CURRENT : ImmLeaf<i32, [{ return Imm == 4; }]>; + // BYTE_imm - Transform bit immediates into byte immediates. def BYTE_imm : SDNodeXForm<imm, [{ // Transformation function: imm >> 3 diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp index 2461773..6450f2a 100644 --- a/lib/Target/X86/X86InstrInfo.cpp +++ b/lib/Target/X86/X86InstrInfo.cpp @@ -605,6 +605,8 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm) { X86::VMOVDQA64rr, X86::VMOVDQA64rm, TB_ALIGN_64 }, { X86::VMOVDQU32rr, X86::VMOVDQU32rm, 0 }, { X86::VMOVDQU64rr, X86::VMOVDQU64rm, 0 }, + { X86::VPABSDZrr, X86::VPABSDZrm, 0 }, + { X86::VPABSQZrr, X86::VPABSQZrm, 0 }, // AES foldable instructions { X86::AESIMCrr, X86::AESIMCrm, TB_ALIGN_16 }, @@ -1210,8 +1212,6 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm) { X86::PEXT64rr, X86::PEXT64rm, 0 }, // AVX-512 foldable instructions - { X86::VPADDDZrr, X86::VPADDDZrm, 0 }, - { X86::VPADDQZrr, X86::VPADDQZrm, 0 }, { X86::VADDPSZrr, X86::VADDPSZrm, 0 }, { X86::VADDPDZrr, X86::VADDPDZrm, 0 }, { X86::VSUBPSZrr, X86::VSUBPSZrm, 0 }, @@ -1224,17 +1224,31 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm) { X86::VMINPDZrr, X86::VMINPDZrm, 0 }, { X86::VMAXPSZrr, X86::VMAXPSZrm, 0 }, { X86::VMAXPDZrr, X86::VMAXPDZrm, 0 }, + { X86::VPADDDZrr, X86::VPADDDZrm, 0 }, + { X86::VPADDQZrr, X86::VPADDQZrm, 0 }, { X86::VPERMPDZri, X86::VPERMPDZmi, 0 }, { X86::VPERMPSZrr, X86::VPERMPSZrm, 0 }, + { X86::VPMAXSDZrr, X86::VPMAXSDZrm, 0 }, + { X86::VPMAXSQZrr, X86::VPMAXSQZrm, 0 }, + { X86::VPMAXUDZrr, X86::VPMAXUDZrm, 0 }, + { X86::VPMAXUQZrr, X86::VPMAXUQZrm, 0 }, + { X86::VPMINSDZrr, X86::VPMINSDZrm, 0 }, + { X86::VPMINSQZrr, X86::VPMINSQZrm, 0 }, + { X86::VPMINUDZrr, X86::VPMINUDZrm, 0 }, + { X86::VPMINUQZrr, X86::VPMINUQZrm, 0 }, + { X86::VPMULDQZrr, X86::VPMULDQZrm, 0 }, { X86::VPSLLVDZrr, X86::VPSLLVDZrm, 0 }, { X86::VPSLLVQZrr, X86::VPSLLVQZrm, 0 }, { X86::VPSRAVDZrr, X86::VPSRAVDZrm, 0 }, { X86::VPSRLVDZrr, X86::VPSRLVDZrm, 0 }, { X86::VPSRLVQZrr, X86::VPSRLVQZrm, 0 }, + { X86::VPSUBDZrr, X86::VPSUBDZrm, 0 }, + { X86::VPSUBQZrr, X86::VPSUBQZrm, 0 }, { X86::VSHUFPDZrri, X86::VSHUFPDZrmi, 0 }, { X86::VSHUFPSZrri, X86::VSHUFPSZrmi, 0 }, { X86::VALIGNQrri, X86::VALIGNQrmi, 0 }, { X86::VALIGNDrri, X86::VALIGNDrmi, 0 }, + { X86::VPMULUDQZrr, X86::VPMULUDQZrm, 0 }, // AES foldable instructions { X86::AESDECLASTrr, X86::AESDECLASTrm, TB_ALIGN_16 }, @@ -1268,119 +1282,111 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm) static const X86OpTblEntry OpTbl3[] = { // FMA foldable instructions - { X86::VFMADDSSr231r, X86::VFMADDSSr231m, 0 }, - { X86::VFMADDSDr231r, X86::VFMADDSDr231m, 0 }, - { X86::VFMADDSSr132r, X86::VFMADDSSr132m, 0 }, - { X86::VFMADDSDr132r, X86::VFMADDSDr132m, 0 }, - { X86::VFMADDSSr213r, X86::VFMADDSSr213m, 0 }, - { X86::VFMADDSDr213r, X86::VFMADDSDr213m, 0 }, - { X86::VFMADDSSr213r_Int, X86::VFMADDSSr213m_Int, 0 }, - { X86::VFMADDSDr213r_Int, X86::VFMADDSDr213m_Int, 0 }, - - { X86::VFMADDPSr231r, X86::VFMADDPSr231m, TB_ALIGN_16 }, - { X86::VFMADDPDr231r, X86::VFMADDPDr231m, TB_ALIGN_16 }, - { X86::VFMADDPSr132r, X86::VFMADDPSr132m, TB_ALIGN_16 }, - { X86::VFMADDPDr132r, X86::VFMADDPDr132m, TB_ALIGN_16 }, - { X86::VFMADDPSr213r, X86::VFMADDPSr213m, TB_ALIGN_16 }, - { X86::VFMADDPDr213r, X86::VFMADDPDr213m, TB_ALIGN_16 }, - { X86::VFMADDPSr231rY, X86::VFMADDPSr231mY, TB_ALIGN_32 }, - { X86::VFMADDPDr231rY, X86::VFMADDPDr231mY, TB_ALIGN_32 }, - { X86::VFMADDPSr132rY, X86::VFMADDPSr132mY, TB_ALIGN_32 }, - { X86::VFMADDPDr132rY, X86::VFMADDPDr132mY, TB_ALIGN_32 }, - { X86::VFMADDPSr213rY, X86::VFMADDPSr213mY, TB_ALIGN_32 }, - { X86::VFMADDPDr213rY, X86::VFMADDPDr213mY, TB_ALIGN_32 }, - - { X86::VFNMADDSSr231r, X86::VFNMADDSSr231m, 0 }, - { X86::VFNMADDSDr231r, X86::VFNMADDSDr231m, 0 }, - { X86::VFNMADDSSr132r, X86::VFNMADDSSr132m, 0 }, - { X86::VFNMADDSDr132r, X86::VFNMADDSDr132m, 0 }, - { X86::VFNMADDSSr213r, X86::VFNMADDSSr213m, 0 }, - { X86::VFNMADDSDr213r, X86::VFNMADDSDr213m, 0 }, - { X86::VFNMADDSSr213r_Int, X86::VFNMADDSSr213m_Int, 0 }, - { X86::VFNMADDSDr213r_Int, X86::VFNMADDSDr213m_Int, 0 }, - - { X86::VFNMADDPSr231r, X86::VFNMADDPSr231m, TB_ALIGN_16 }, - { X86::VFNMADDPDr231r, X86::VFNMADDPDr231m, TB_ALIGN_16 }, - { X86::VFNMADDPSr132r, X86::VFNMADDPSr132m, TB_ALIGN_16 }, - { X86::VFNMADDPDr132r, X86::VFNMADDPDr132m, TB_ALIGN_16 }, - { X86::VFNMADDPSr213r, X86::VFNMADDPSr213m, TB_ALIGN_16 }, - { X86::VFNMADDPDr213r, X86::VFNMADDPDr213m, TB_ALIGN_16 }, - { X86::VFNMADDPSr231rY, X86::VFNMADDPSr231mY, TB_ALIGN_32 }, - { X86::VFNMADDPDr231rY, X86::VFNMADDPDr231mY, TB_ALIGN_32 }, - { X86::VFNMADDPSr132rY, X86::VFNMADDPSr132mY, TB_ALIGN_32 }, - { X86::VFNMADDPDr132rY, X86::VFNMADDPDr132mY, TB_ALIGN_32 }, - { X86::VFNMADDPSr213rY, X86::VFNMADDPSr213mY, TB_ALIGN_32 }, - { X86::VFNMADDPDr213rY, X86::VFNMADDPDr213mY, TB_ALIGN_32 }, - - { X86::VFMSUBSSr231r, X86::VFMSUBSSr231m, 0 }, - { X86::VFMSUBSDr231r, X86::VFMSUBSDr231m, 0 }, - { X86::VFMSUBSSr132r, X86::VFMSUBSSr132m, 0 }, - { X86::VFMSUBSDr132r, X86::VFMSUBSDr132m, 0 }, - { X86::VFMSUBSSr213r, X86::VFMSUBSSr213m, 0 }, - { X86::VFMSUBSDr213r, X86::VFMSUBSDr213m, 0 }, - { X86::VFMSUBSSr213r_Int, X86::VFMSUBSSr213m_Int, 0 }, - { X86::VFMSUBSDr213r_Int, X86::VFMSUBSDr213m_Int, 0 }, - - { X86::VFMSUBPSr231r, X86::VFMSUBPSr231m, TB_ALIGN_16 }, - { X86::VFMSUBPDr231r, X86::VFMSUBPDr231m, TB_ALIGN_16 }, - { X86::VFMSUBPSr132r, X86::VFMSUBPSr132m, TB_ALIGN_16 }, - { X86::VFMSUBPDr132r, X86::VFMSUBPDr132m, TB_ALIGN_16 }, - { X86::VFMSUBPSr213r, X86::VFMSUBPSr213m, TB_ALIGN_16 }, - { X86::VFMSUBPDr213r, X86::VFMSUBPDr213m, TB_ALIGN_16 }, - { X86::VFMSUBPSr231rY, X86::VFMSUBPSr231mY, TB_ALIGN_32 }, - { X86::VFMSUBPDr231rY, X86::VFMSUBPDr231mY, TB_ALIGN_32 }, - { X86::VFMSUBPSr132rY, X86::VFMSUBPSr132mY, TB_ALIGN_32 }, - { X86::VFMSUBPDr132rY, X86::VFMSUBPDr132mY, TB_ALIGN_32 }, - { X86::VFMSUBPSr213rY, X86::VFMSUBPSr213mY, TB_ALIGN_32 }, - { X86::VFMSUBPDr213rY, X86::VFMSUBPDr213mY, TB_ALIGN_32 }, - - { X86::VFNMSUBSSr231r, X86::VFNMSUBSSr231m, 0 }, - { X86::VFNMSUBSDr231r, X86::VFNMSUBSDr231m, 0 }, - { X86::VFNMSUBSSr132r, X86::VFNMSUBSSr132m, 0 }, - { X86::VFNMSUBSDr132r, X86::VFNMSUBSDr132m, 0 }, - { X86::VFNMSUBSSr213r, X86::VFNMSUBSSr213m, 0 }, - { X86::VFNMSUBSDr213r, X86::VFNMSUBSDr213m, 0 }, - { X86::VFNMSUBSSr213r_Int, X86::VFNMSUBSSr213m_Int, 0 }, - { X86::VFNMSUBSDr213r_Int, X86::VFNMSUBSDr213m_Int, 0 }, - - { X86::VFNMSUBPSr231r, X86::VFNMSUBPSr231m, TB_ALIGN_16 }, - { X86::VFNMSUBPDr231r, X86::VFNMSUBPDr231m, TB_ALIGN_16 }, - { X86::VFNMSUBPSr132r, X86::VFNMSUBPSr132m, TB_ALIGN_16 }, - { X86::VFNMSUBPDr132r, X86::VFNMSUBPDr132m, TB_ALIGN_16 }, - { X86::VFNMSUBPSr213r, X86::VFNMSUBPSr213m, TB_ALIGN_16 }, - { X86::VFNMSUBPDr213r, X86::VFNMSUBPDr213m, TB_ALIGN_16 }, - { X86::VFNMSUBPSr231rY, X86::VFNMSUBPSr231mY, TB_ALIGN_32 }, - { X86::VFNMSUBPDr231rY, X86::VFNMSUBPDr231mY, TB_ALIGN_32 }, - { X86::VFNMSUBPSr132rY, X86::VFNMSUBPSr132mY, TB_ALIGN_32 }, - { X86::VFNMSUBPDr132rY, X86::VFNMSUBPDr132mY, TB_ALIGN_32 }, - { X86::VFNMSUBPSr213rY, X86::VFNMSUBPSr213mY, TB_ALIGN_32 }, - { X86::VFNMSUBPDr213rY, X86::VFNMSUBPDr213mY, TB_ALIGN_32 }, - - { X86::VFMADDSUBPSr231r, X86::VFMADDSUBPSr231m, TB_ALIGN_16 }, - { X86::VFMADDSUBPDr231r, X86::VFMADDSUBPDr231m, TB_ALIGN_16 }, - { X86::VFMADDSUBPSr132r, X86::VFMADDSUBPSr132m, TB_ALIGN_16 }, - { X86::VFMADDSUBPDr132r, X86::VFMADDSUBPDr132m, TB_ALIGN_16 }, - { X86::VFMADDSUBPSr213r, X86::VFMADDSUBPSr213m, TB_ALIGN_16 }, - { X86::VFMADDSUBPDr213r, X86::VFMADDSUBPDr213m, TB_ALIGN_16 }, - { X86::VFMADDSUBPSr231rY, X86::VFMADDSUBPSr231mY, TB_ALIGN_32 }, - { X86::VFMADDSUBPDr231rY, X86::VFMADDSUBPDr231mY, TB_ALIGN_32 }, - { X86::VFMADDSUBPSr132rY, X86::VFMADDSUBPSr132mY, TB_ALIGN_32 }, - { X86::VFMADDSUBPDr132rY, X86::VFMADDSUBPDr132mY, TB_ALIGN_32 }, - { X86::VFMADDSUBPSr213rY, X86::VFMADDSUBPSr213mY, TB_ALIGN_32 }, - { X86::VFMADDSUBPDr213rY, X86::VFMADDSUBPDr213mY, TB_ALIGN_32 }, - - { X86::VFMSUBADDPSr231r, X86::VFMSUBADDPSr231m, TB_ALIGN_16 }, - { X86::VFMSUBADDPDr231r, X86::VFMSUBADDPDr231m, TB_ALIGN_16 }, - { X86::VFMSUBADDPSr132r, X86::VFMSUBADDPSr132m, TB_ALIGN_16 }, - { X86::VFMSUBADDPDr132r, X86::VFMSUBADDPDr132m, TB_ALIGN_16 }, - { X86::VFMSUBADDPSr213r, X86::VFMSUBADDPSr213m, TB_ALIGN_16 }, - { X86::VFMSUBADDPDr213r, X86::VFMSUBADDPDr213m, TB_ALIGN_16 }, - { X86::VFMSUBADDPSr231rY, X86::VFMSUBADDPSr231mY, TB_ALIGN_32 }, - { X86::VFMSUBADDPDr231rY, X86::VFMSUBADDPDr231mY, TB_ALIGN_32 }, - { X86::VFMSUBADDPSr132rY, X86::VFMSUBADDPSr132mY, TB_ALIGN_32 }, - { X86::VFMSUBADDPDr132rY, X86::VFMSUBADDPDr132mY, TB_ALIGN_32 }, - { X86::VFMSUBADDPSr213rY, X86::VFMSUBADDPSr213mY, TB_ALIGN_32 }, - { X86::VFMSUBADDPDr213rY, X86::VFMSUBADDPDr213mY, TB_ALIGN_32 }, + { X86::VFMADDSSr231r, X86::VFMADDSSr231m, TB_ALIGN_NONE }, + { X86::VFMADDSDr231r, X86::VFMADDSDr231m, TB_ALIGN_NONE }, + { X86::VFMADDSSr132r, X86::VFMADDSSr132m, TB_ALIGN_NONE }, + { X86::VFMADDSDr132r, X86::VFMADDSDr132m, TB_ALIGN_NONE }, + { X86::VFMADDSSr213r, X86::VFMADDSSr213m, TB_ALIGN_NONE }, + { X86::VFMADDSDr213r, X86::VFMADDSDr213m, TB_ALIGN_NONE }, + + { X86::VFMADDPSr231r, X86::VFMADDPSr231m, TB_ALIGN_NONE }, + { X86::VFMADDPDr231r, X86::VFMADDPDr231m, TB_ALIGN_NONE }, + { X86::VFMADDPSr132r, X86::VFMADDPSr132m, TB_ALIGN_NONE }, + { X86::VFMADDPDr132r, X86::VFMADDPDr132m, TB_ALIGN_NONE }, + { X86::VFMADDPSr213r, X86::VFMADDPSr213m, TB_ALIGN_NONE }, + { X86::VFMADDPDr213r, X86::VFMADDPDr213m, TB_ALIGN_NONE }, + { X86::VFMADDPSr231rY, X86::VFMADDPSr231mY, TB_ALIGN_NONE }, + { X86::VFMADDPDr231rY, X86::VFMADDPDr231mY, TB_ALIGN_NONE }, + { X86::VFMADDPSr132rY, X86::VFMADDPSr132mY, TB_ALIGN_NONE }, + { X86::VFMADDPDr132rY, X86::VFMADDPDr132mY, TB_ALIGN_NONE }, + { X86::VFMADDPSr213rY, X86::VFMADDPSr213mY, TB_ALIGN_NONE }, + { X86::VFMADDPDr213rY, X86::VFMADDPDr213mY, TB_ALIGN_NONE }, + + { X86::VFNMADDSSr231r, X86::VFNMADDSSr231m, TB_ALIGN_NONE }, + { X86::VFNMADDSDr231r, X86::VFNMADDSDr231m, TB_ALIGN_NONE }, + { X86::VFNMADDSSr132r, X86::VFNMADDSSr132m, TB_ALIGN_NONE }, + { X86::VFNMADDSDr132r, X86::VFNMADDSDr132m, TB_ALIGN_NONE }, + { X86::VFNMADDSSr213r, X86::VFNMADDSSr213m, TB_ALIGN_NONE }, + { X86::VFNMADDSDr213r, X86::VFNMADDSDr213m, TB_ALIGN_NONE }, + + { X86::VFNMADDPSr231r, X86::VFNMADDPSr231m, TB_ALIGN_NONE }, + { X86::VFNMADDPDr231r, X86::VFNMADDPDr231m, TB_ALIGN_NONE }, + { X86::VFNMADDPSr132r, X86::VFNMADDPSr132m, TB_ALIGN_NONE }, + { X86::VFNMADDPDr132r, X86::VFNMADDPDr132m, TB_ALIGN_NONE }, + { X86::VFNMADDPSr213r, X86::VFNMADDPSr213m, TB_ALIGN_NONE }, + { X86::VFNMADDPDr213r, X86::VFNMADDPDr213m, TB_ALIGN_NONE }, + { X86::VFNMADDPSr231rY, X86::VFNMADDPSr231mY, TB_ALIGN_NONE }, + { X86::VFNMADDPDr231rY, X86::VFNMADDPDr231mY, TB_ALIGN_NONE }, + { X86::VFNMADDPSr132rY, X86::VFNMADDPSr132mY, TB_ALIGN_NONE }, + { X86::VFNMADDPDr132rY, X86::VFNMADDPDr132mY, TB_ALIGN_NONE }, + { X86::VFNMADDPSr213rY, X86::VFNMADDPSr213mY, TB_ALIGN_NONE }, + { X86::VFNMADDPDr213rY, X86::VFNMADDPDr213mY, TB_ALIGN_NONE }, + + { X86::VFMSUBSSr231r, X86::VFMSUBSSr231m, TB_ALIGN_NONE }, + { X86::VFMSUBSDr231r, X86::VFMSUBSDr231m, TB_ALIGN_NONE }, + { X86::VFMSUBSSr132r, X86::VFMSUBSSr132m, TB_ALIGN_NONE }, + { X86::VFMSUBSDr132r, X86::VFMSUBSDr132m, TB_ALIGN_NONE }, + { X86::VFMSUBSSr213r, X86::VFMSUBSSr213m, TB_ALIGN_NONE }, + { X86::VFMSUBSDr213r, X86::VFMSUBSDr213m, TB_ALIGN_NONE }, + + { X86::VFMSUBPSr231r, X86::VFMSUBPSr231m, TB_ALIGN_NONE }, + { X86::VFMSUBPDr231r, X86::VFMSUBPDr231m, TB_ALIGN_NONE }, + { X86::VFMSUBPSr132r, X86::VFMSUBPSr132m, TB_ALIGN_NONE }, + { X86::VFMSUBPDr132r, X86::VFMSUBPDr132m, TB_ALIGN_NONE }, + { X86::VFMSUBPSr213r, X86::VFMSUBPSr213m, TB_ALIGN_NONE }, + { X86::VFMSUBPDr213r, X86::VFMSUBPDr213m, TB_ALIGN_NONE }, + { X86::VFMSUBPSr231rY, X86::VFMSUBPSr231mY, TB_ALIGN_NONE }, + { X86::VFMSUBPDr231rY, X86::VFMSUBPDr231mY, TB_ALIGN_NONE }, + { X86::VFMSUBPSr132rY, X86::VFMSUBPSr132mY, TB_ALIGN_NONE }, + { X86::VFMSUBPDr132rY, X86::VFMSUBPDr132mY, TB_ALIGN_NONE }, + { X86::VFMSUBPSr213rY, X86::VFMSUBPSr213mY, TB_ALIGN_NONE }, + { X86::VFMSUBPDr213rY, X86::VFMSUBPDr213mY, TB_ALIGN_NONE }, + + { X86::VFNMSUBSSr231r, X86::VFNMSUBSSr231m, TB_ALIGN_NONE }, + { X86::VFNMSUBSDr231r, X86::VFNMSUBSDr231m, TB_ALIGN_NONE }, + { X86::VFNMSUBSSr132r, X86::VFNMSUBSSr132m, TB_ALIGN_NONE }, + { X86::VFNMSUBSDr132r, X86::VFNMSUBSDr132m, TB_ALIGN_NONE }, + { X86::VFNMSUBSSr213r, X86::VFNMSUBSSr213m, TB_ALIGN_NONE }, + { X86::VFNMSUBSDr213r, X86::VFNMSUBSDr213m, TB_ALIGN_NONE }, + + { X86::VFNMSUBPSr231r, X86::VFNMSUBPSr231m, TB_ALIGN_NONE }, + { X86::VFNMSUBPDr231r, X86::VFNMSUBPDr231m, TB_ALIGN_NONE }, + { X86::VFNMSUBPSr132r, X86::VFNMSUBPSr132m, TB_ALIGN_NONE }, + { X86::VFNMSUBPDr132r, X86::VFNMSUBPDr132m, TB_ALIGN_NONE }, + { X86::VFNMSUBPSr213r, X86::VFNMSUBPSr213m, TB_ALIGN_NONE }, + { X86::VFNMSUBPDr213r, X86::VFNMSUBPDr213m, TB_ALIGN_NONE }, + { X86::VFNMSUBPSr231rY, X86::VFNMSUBPSr231mY, TB_ALIGN_NONE }, + { X86::VFNMSUBPDr231rY, X86::VFNMSUBPDr231mY, TB_ALIGN_NONE }, + { X86::VFNMSUBPSr132rY, X86::VFNMSUBPSr132mY, TB_ALIGN_NONE }, + { X86::VFNMSUBPDr132rY, X86::VFNMSUBPDr132mY, TB_ALIGN_NONE }, + { X86::VFNMSUBPSr213rY, X86::VFNMSUBPSr213mY, TB_ALIGN_NONE }, + { X86::VFNMSUBPDr213rY, X86::VFNMSUBPDr213mY, TB_ALIGN_NONE }, + + { X86::VFMADDSUBPSr231r, X86::VFMADDSUBPSr231m, TB_ALIGN_NONE }, + { X86::VFMADDSUBPDr231r, X86::VFMADDSUBPDr231m, TB_ALIGN_NONE }, + { X86::VFMADDSUBPSr132r, X86::VFMADDSUBPSr132m, TB_ALIGN_NONE }, + { X86::VFMADDSUBPDr132r, X86::VFMADDSUBPDr132m, TB_ALIGN_NONE }, + { X86::VFMADDSUBPSr213r, X86::VFMADDSUBPSr213m, TB_ALIGN_NONE }, + { X86::VFMADDSUBPDr213r, X86::VFMADDSUBPDr213m, TB_ALIGN_NONE }, + { X86::VFMADDSUBPSr231rY, X86::VFMADDSUBPSr231mY, TB_ALIGN_NONE }, + { X86::VFMADDSUBPDr231rY, X86::VFMADDSUBPDr231mY, TB_ALIGN_NONE }, + { X86::VFMADDSUBPSr132rY, X86::VFMADDSUBPSr132mY, TB_ALIGN_NONE }, + { X86::VFMADDSUBPDr132rY, X86::VFMADDSUBPDr132mY, TB_ALIGN_NONE }, + { X86::VFMADDSUBPSr213rY, X86::VFMADDSUBPSr213mY, TB_ALIGN_NONE }, + { X86::VFMADDSUBPDr213rY, X86::VFMADDSUBPDr213mY, TB_ALIGN_NONE }, + + { X86::VFMSUBADDPSr231r, X86::VFMSUBADDPSr231m, TB_ALIGN_NONE }, + { X86::VFMSUBADDPDr231r, X86::VFMSUBADDPDr231m, TB_ALIGN_NONE }, + { X86::VFMSUBADDPSr132r, X86::VFMSUBADDPSr132m, TB_ALIGN_NONE }, + { X86::VFMSUBADDPDr132r, X86::VFMSUBADDPDr132m, TB_ALIGN_NONE }, + { X86::VFMSUBADDPSr213r, X86::VFMSUBADDPSr213m, TB_ALIGN_NONE }, + { X86::VFMSUBADDPDr213r, X86::VFMSUBADDPDr213m, TB_ALIGN_NONE }, + { X86::VFMSUBADDPSr231rY, X86::VFMSUBADDPSr231mY, TB_ALIGN_NONE }, + { X86::VFMSUBADDPDr231rY, X86::VFMSUBADDPDr231mY, TB_ALIGN_NONE }, + { X86::VFMSUBADDPSr132rY, X86::VFMSUBADDPSr132mY, TB_ALIGN_NONE }, + { X86::VFMSUBADDPDr132rY, X86::VFMSUBADDPDr132mY, TB_ALIGN_NONE }, + { X86::VFMSUBADDPSr213rY, X86::VFMSUBADDPSr213mY, TB_ALIGN_NONE }, + { X86::VFMSUBADDPDr213rY, X86::VFMSUBADDPDr213mY, TB_ALIGN_NONE }, // FMA4 foldable patterns { X86::VFMADDSS4rr, X86::VFMADDSS4rm, 0 }, @@ -1420,6 +1426,10 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm) { X86::VPERMI2Qrr, X86::VPERMI2Qrm, 0 }, { X86::VPERMI2PSrr, X86::VPERMI2PSrm, 0 }, { X86::VPERMI2PDrr, X86::VPERMI2PDrm, 0 }, + { X86::VBLENDMPDZrr, X86::VBLENDMPDZrm, 0 }, + { X86::VBLENDMPSZrr, X86::VBLENDMPSZrm, 0 }, + { X86::VPBLENDMDZrr, X86::VPBLENDMDZrm, 0 }, + { X86::VPBLENDMQZrr, X86::VPBLENDMQZrm, 0 } }; for (unsigned i = 0, e = array_lengthof(OpTbl3); i != e; ++i) { @@ -1536,8 +1546,8 @@ static bool isFrameLoadOpcode(int Opcode) { case X86::VMOVDQAYrm: case X86::MMX_MOVD64rm: case X86::MMX_MOVQ64rm: - case X86::VMOVDQA32rm: - case X86::VMOVDQA64rm: + case X86::VMOVAPSZrm: + case X86::VMOVUPSZrm: return true; } } @@ -1563,6 +1573,8 @@ static bool isFrameStoreOpcode(int Opcode) { case X86::VMOVAPSYmr: case X86::VMOVAPDYmr: case X86::VMOVDQAYmr: + case X86::VMOVUPSZmr: + case X86::VMOVAPSZmr: case X86::MMX_MOVD64mr: case X86::MMX_MOVQ64mr: case X86::MMX_MOVNTQmr: @@ -1621,9 +1633,9 @@ static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI) { if (!TargetRegisterInfo::isVirtualRegister(BaseReg)) return false; bool isPICBase = false; - for (MachineRegisterInfo::def_iterator I = MRI.def_begin(BaseReg), - E = MRI.def_end(); I != E; ++I) { - MachineInstr *DefMI = I.getOperand().getParent(); + for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), + E = MRI.def_instr_end(); I != E; ++I) { + MachineInstr *DefMI = &*I; if (DefMI->getOpcode() != X86::MOVPC32r) return false; assert(!isPICBase && "More than one PIC base?"); @@ -1809,7 +1821,7 @@ void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, MBB.insert(I, MI); } - MachineInstr *NewMI = prior(I); + MachineInstr *NewMI = std::prev(I); NewMI->substituteRegister(Orig->getOperand(0).getReg(), DestReg, SubIdx, TRI); } @@ -2452,6 +2464,41 @@ X86InstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const { } } +bool X86InstrInfo::findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1, + unsigned &SrcOpIdx2) const { + switch (MI->getOpcode()) { + case X86::VFMADDPDr231r: + case X86::VFMADDPSr231r: + case X86::VFMADDSDr231r: + case X86::VFMADDSSr231r: + case X86::VFMSUBPDr231r: + case X86::VFMSUBPSr231r: + case X86::VFMSUBSDr231r: + case X86::VFMSUBSSr231r: + case X86::VFNMADDPDr231r: + case X86::VFNMADDPSr231r: + case X86::VFNMADDSDr231r: + case X86::VFNMADDSSr231r: + case X86::VFNMSUBPDr231r: + case X86::VFNMSUBPSr231r: + case X86::VFNMSUBSDr231r: + case X86::VFNMSUBSSr231r: + case X86::VFMADDPDr231rY: + case X86::VFMADDPSr231rY: + case X86::VFMSUBPDr231rY: + case X86::VFMSUBPSr231rY: + case X86::VFNMADDPDr231rY: + case X86::VFNMADDPSr231rY: + case X86::VFNMSUBPDr231rY: + case X86::VFNMSUBPSr231rY: + SrcOpIdx1 = 2; + SrcOpIdx2 = 3; + return true; + default: + return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); + } +} + static X86::CondCode getCondFromBranchOpc(unsigned BrOpc) { switch (BrOpc) { default: return X86::COND_INVALID; @@ -2738,8 +2785,8 @@ bool X86InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB, } // If the block has any instructions after a JMP, delete them. - while (llvm::next(I) != MBB.end()) - llvm::next(I)->eraseFromParent(); + while (std::next(I) != MBB.end()) + std::next(I)->eraseFromParent(); Cond.clear(); FBB = 0; @@ -3015,6 +3062,11 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, return 0; } +inline static bool MaskRegClassContains(unsigned Reg) { + return X86::VK8RegClass.contains(Reg) || + X86::VK16RegClass.contains(Reg) || + X86::VK1RegClass.contains(Reg); +} static unsigned copyPhysRegOpcode_AVX512(unsigned& DestReg, unsigned& SrcReg) { if (X86::VR128XRegClass.contains(DestReg, SrcReg) || @@ -3024,11 +3076,23 @@ unsigned copyPhysRegOpcode_AVX512(unsigned& DestReg, unsigned& SrcReg) { SrcReg = get512BitSuperRegister(SrcReg); return X86::VMOVAPSZrr; } - if ((X86::VK8RegClass.contains(DestReg) || - X86::VK16RegClass.contains(DestReg)) && - (X86::VK8RegClass.contains(SrcReg) || - X86::VK16RegClass.contains(SrcReg))) + if (MaskRegClassContains(DestReg) && + MaskRegClassContains(SrcReg)) return X86::KMOVWkk; + if (MaskRegClassContains(DestReg) && + (X86::GR32RegClass.contains(SrcReg) || + X86::GR16RegClass.contains(SrcReg) || + X86::GR8RegClass.contains(SrcReg))) { + SrcReg = getX86SubSuperRegister(SrcReg, MVT::i32); + return X86::KMOVWkr; + } + if ((X86::GR32RegClass.contains(DestReg) || + X86::GR16RegClass.contains(DestReg) || + X86::GR8RegClass.contains(DestReg)) && + MaskRegClassContains(SrcReg)) { + DestReg = getX86SubSuperRegister(DestReg, MVT::i32); + return X86::KMOVWrk; + } return 0; } @@ -3837,6 +3901,8 @@ bool X86InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const { bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX(); MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI); switch (MI->getOpcode()) { + case X86::MOV32r0: + return Expand2AddrUndef(MIB, get(X86::XOR32rr)); case X86::SETB_C8r: return Expand2AddrUndef(MIB, get(X86::SBB8rr)); case X86::SETB_C16r: @@ -3861,6 +3927,7 @@ bool X86InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const { case X86::TEST8ri_NOREX: MI->setDesc(get(X86::TEST8ri)); return true; + case X86::KSET0B: case X86::KSET0W: return Expand2AddrUndef(MIB, get(X86::KXORWrr)); case X86::KSET1B: case X86::KSET1W: return Expand2AddrUndef(MIB, get(X86::KXNORWrr)); @@ -4198,75 +4265,10 @@ breakPartialRegDependency(MachineBasicBlock::iterator MI, unsigned OpNum, MI->addRegisterKilled(Reg, TRI, true); } -static MachineInstr* foldPatchpoint(MachineFunction &MF, - MachineInstr *MI, - const SmallVectorImpl<unsigned> &Ops, - int FrameIndex, - const TargetInstrInfo &TII) { - unsigned StartIdx = 0; - switch (MI->getOpcode()) { - case TargetOpcode::STACKMAP: - StartIdx = 2; // Skip ID, nShadowBytes. - break; - case TargetOpcode::PATCHPOINT: { - // For PatchPoint, the call args are not foldable. - PatchPointOpers opers(MI); - StartIdx = opers.getVarIdx(); - break; - } - default: - llvm_unreachable("unexpected stackmap opcode"); - } - - // Return false if any operands requested for folding are not foldable (not - // part of the stackmap's live values). - for (SmallVectorImpl<unsigned>::const_iterator I = Ops.begin(), E = Ops.end(); - I != E; ++I) { - if (*I < StartIdx) - return 0; - } - - MachineInstr *NewMI = - MF.CreateMachineInstr(TII.get(MI->getOpcode()), MI->getDebugLoc(), true); - MachineInstrBuilder MIB(MF, NewMI); - - // No need to fold return, the meta data, and function arguments - for (unsigned i = 0; i < StartIdx; ++i) - MIB.addOperand(MI->getOperand(i)); - - for (unsigned i = StartIdx; i < MI->getNumOperands(); ++i) { - MachineOperand &MO = MI->getOperand(i); - if (std::find(Ops.begin(), Ops.end(), i) != Ops.end()) { - assert(MO.getReg() && "patchpoint can only fold a vreg operand"); - // Compute the spill slot size and offset. - const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(MO.getReg()); - unsigned SpillSize; - unsigned SpillOffset; - bool Valid = TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, - SpillOffset, &MF.getTarget()); - if (!Valid) - report_fatal_error("cannot spill patchpoint subregister operand"); - - MIB.addOperand(MachineOperand::CreateImm(StackMaps::IndirectMemRefOp)); - MIB.addOperand(MachineOperand::CreateImm(SpillSize)); - MIB.addOperand(MachineOperand::CreateFI(FrameIndex)); - addOffset(MIB, SpillOffset); - } - else - MIB.addOperand(MO); - } - return NewMI; -} - MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI, const SmallVectorImpl<unsigned> &Ops, int FrameIndex) const { - // Special case stack map and patch point intrinsics. - if (MI->getOpcode() == TargetOpcode::STACKMAP - || MI->getOpcode() == TargetOpcode::PATCHPOINT) { - return foldPatchpoint(MF, MI, Ops, FrameIndex, *this); - } // Check switch flag if (NoFusing) return NULL; @@ -5172,7 +5174,13 @@ static const uint16_t ReplaceableInstrsAVX2[][3] = { { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, - { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr } + { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, + { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, + { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, + { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, + { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, + { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, + { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm} }; // FIXME: Some shuffle and unpack instructions have equivalents in different @@ -5315,7 +5323,7 @@ namespace { static char ID; CGBR() : MachineFunctionPass(ID) {} - virtual bool runOnMachineFunction(MachineFunction &MF) { + bool runOnMachineFunction(MachineFunction &MF) override { const X86TargetMachine *TM = static_cast<const X86TargetMachine *>(&MF.getTarget()); @@ -5362,11 +5370,11 @@ namespace { return true; } - virtual const char *getPassName() const { + const char *getPassName() const override { return "X86 PIC Global Base Reg Initialization"; } - virtual void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); MachineFunctionPass::getAnalysisUsage(AU); } @@ -5382,7 +5390,7 @@ namespace { static char ID; LDTLSCleanup() : MachineFunctionPass(ID) {} - virtual bool runOnMachineFunction(MachineFunction &MF) { + bool runOnMachineFunction(MachineFunction &MF) override { X86MachineFunctionInfo* MFI = MF.getInfo<X86MachineFunctionInfo>(); if (MFI->getNumLocalDynamicTLSAccesses() < 2) { // No point folding accesses if there isn't at least two. @@ -5475,11 +5483,11 @@ namespace { return Copy; } - virtual const char *getPassName() const { + const char *getPassName() const override { return "Local Dynamic TLS Access Clean-up"; } - virtual void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); AU.addRequired<MachineDominatorTree>(); MachineFunctionPass::getAnalysisUsage(AU); diff --git a/lib/Target/X86/X86InstrInfo.h b/lib/Target/X86/X86InstrInfo.h index 600e392..156291e 100644 --- a/lib/Target/X86/X86InstrInfo.h +++ b/lib/Target/X86/X86InstrInfo.h @@ -14,7 +14,7 @@ #ifndef X86INSTRUCTIONINFO_H #define X86INSTRUCTIONINFO_H -#include "X86.h" +#include "MCTargetDesc/X86BaseInfo.h" #include "X86RegisterInfo.h" #include "llvm/ADT/DenseMap.h" #include "llvm/Target/TargetInstrInfo.h" @@ -111,23 +111,24 @@ inline static bool isScale(const MachineOperand &MO) { inline static bool isLeaMem(const MachineInstr *MI, unsigned Op) { if (MI->getOperand(Op).isFI()) return true; - return Op+4 <= MI->getNumOperands() && - MI->getOperand(Op ).isReg() && isScale(MI->getOperand(Op+1)) && - MI->getOperand(Op+2).isReg() && - (MI->getOperand(Op+3).isImm() || - MI->getOperand(Op+3).isGlobal() || - MI->getOperand(Op+3).isCPI() || - MI->getOperand(Op+3).isJTI()); + return Op+X86::AddrSegmentReg <= MI->getNumOperands() && + MI->getOperand(Op+X86::AddrBaseReg).isReg() && + isScale(MI->getOperand(Op+X86::AddrScaleAmt)) && + MI->getOperand(Op+X86::AddrIndexReg).isReg() && + (MI->getOperand(Op+X86::AddrDisp).isImm() || + MI->getOperand(Op+X86::AddrDisp).isGlobal() || + MI->getOperand(Op+X86::AddrDisp).isCPI() || + MI->getOperand(Op+X86::AddrDisp).isJTI()); } inline static bool isMem(const MachineInstr *MI, unsigned Op) { if (MI->getOperand(Op).isFI()) return true; - return Op+5 <= MI->getNumOperands() && - MI->getOperand(Op+4).isReg() && + return Op+X86::AddrNumOperands <= MI->getNumOperands() && + MI->getOperand(Op+X86::AddrSegmentReg).isReg() && isLeaMem(MI, Op); } -class X86InstrInfo : public X86GenInstrInfo { +class X86InstrInfo final : public X86GenInstrInfo { X86TargetMachine &TM; const X86RegisterInfo RI; @@ -161,7 +162,7 @@ public: /// such, whenever a client has an instance of instruction info, it should /// always be able to get register info as well (through this method). /// - virtual const X86RegisterInfo &getRegisterInfo() const { return RI; } + const X86RegisterInfo &getRegisterInfo() const { return RI; } /// isCoalescableExtInstr - Return true if the instruction is a "coalescable" /// extension instruction. That is, it's like a copy where it's legal for the @@ -169,30 +170,32 @@ public: /// true, then it's expected the pre-extension value is available as a subreg /// of the result register. This also returns the sub-register index in /// SubIdx. - virtual bool isCoalescableExtInstr(const MachineInstr &MI, - unsigned &SrcReg, unsigned &DstReg, - unsigned &SubIdx) const; + bool isCoalescableExtInstr(const MachineInstr &MI, + unsigned &SrcReg, unsigned &DstReg, + unsigned &SubIdx) const override; - unsigned isLoadFromStackSlot(const MachineInstr *MI, int &FrameIndex) const; + unsigned isLoadFromStackSlot(const MachineInstr *MI, + int &FrameIndex) const override; /// isLoadFromStackSlotPostFE - Check for post-frame ptr elimination /// stack locations as well. This uses a heuristic so it isn't /// reliable for correctness. unsigned isLoadFromStackSlotPostFE(const MachineInstr *MI, - int &FrameIndex) const; + int &FrameIndex) const override; - unsigned isStoreToStackSlot(const MachineInstr *MI, int &FrameIndex) const; + unsigned isStoreToStackSlot(const MachineInstr *MI, + int &FrameIndex) const override; /// isStoreToStackSlotPostFE - Check for post-frame ptr elimination /// stack locations as well. This uses a heuristic so it isn't /// reliable for correctness. unsigned isStoreToStackSlotPostFE(const MachineInstr *MI, - int &FrameIndex) const; + int &FrameIndex) const override; bool isReallyTriviallyReMaterializable(const MachineInstr *MI, - AliasAnalysis *AA) const; + AliasAnalysis *AA) const override; void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, unsigned SubIdx, const MachineInstr *Orig, - const TargetRegisterInfo &TRI) const; + const TargetRegisterInfo &TRI) const override; /// Given an operand within a MachineInstr, insert preceding code to put it /// into the right format for a particular kind of LEA instruction. This may @@ -217,65 +220,68 @@ public: /// This method returns a null pointer if the transformation cannot be /// performed, otherwise it returns the new instruction. /// - virtual MachineInstr *convertToThreeAddress(MachineFunction::iterator &MFI, - MachineBasicBlock::iterator &MBBI, - LiveVariables *LV) const; + MachineInstr *convertToThreeAddress(MachineFunction::iterator &MFI, + MachineBasicBlock::iterator &MBBI, + LiveVariables *LV) const override; /// commuteInstruction - We have a few instructions that must be hacked on to /// commute them. /// - virtual MachineInstr *commuteInstruction(MachineInstr *MI, bool NewMI) const; + MachineInstr *commuteInstruction(MachineInstr *MI, bool NewMI) const override; + + bool findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1, + unsigned &SrcOpIdx2) const override; // Branch analysis. - virtual bool isUnpredicatedTerminator(const MachineInstr* MI) const; - virtual bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, - MachineBasicBlock *&FBB, - SmallVectorImpl<MachineOperand> &Cond, - bool AllowModify) const; - virtual unsigned RemoveBranch(MachineBasicBlock &MBB) const; - virtual unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, - MachineBasicBlock *FBB, - const SmallVectorImpl<MachineOperand> &Cond, - DebugLoc DL) const; - virtual bool canInsertSelect(const MachineBasicBlock&, - const SmallVectorImpl<MachineOperand> &Cond, - unsigned, unsigned, int&, int&, int&) const; - virtual void insertSelect(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI, DebugLoc DL, - unsigned DstReg, - const SmallVectorImpl<MachineOperand> &Cond, - unsigned TrueReg, unsigned FalseReg) const; - virtual void copyPhysReg(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI, DebugLoc DL, - unsigned DestReg, unsigned SrcReg, - bool KillSrc) const; - virtual void storeRegToStackSlot(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI, - unsigned SrcReg, bool isKill, int FrameIndex, - const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) const; - - virtual void storeRegToAddr(MachineFunction &MF, unsigned SrcReg, bool isKill, - SmallVectorImpl<MachineOperand> &Addr, - const TargetRegisterClass *RC, - MachineInstr::mmo_iterator MMOBegin, - MachineInstr::mmo_iterator MMOEnd, - SmallVectorImpl<MachineInstr*> &NewMIs) const; - - virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI, - unsigned DestReg, int FrameIndex, - const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) const; - - virtual void loadRegFromAddr(MachineFunction &MF, unsigned DestReg, - SmallVectorImpl<MachineOperand> &Addr, - const TargetRegisterClass *RC, - MachineInstr::mmo_iterator MMOBegin, - MachineInstr::mmo_iterator MMOEnd, - SmallVectorImpl<MachineInstr*> &NewMIs) const; - - virtual bool expandPostRAPseudo(MachineBasicBlock::iterator MI) const; + bool isUnpredicatedTerminator(const MachineInstr* MI) const override; + bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, + MachineBasicBlock *&FBB, + SmallVectorImpl<MachineOperand> &Cond, + bool AllowModify) const override; + unsigned RemoveBranch(MachineBasicBlock &MBB) const override; + unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, + MachineBasicBlock *FBB, + const SmallVectorImpl<MachineOperand> &Cond, + DebugLoc DL) const override; + bool canInsertSelect(const MachineBasicBlock&, + const SmallVectorImpl<MachineOperand> &Cond, + unsigned, unsigned, int&, int&, int&) const override; + void insertSelect(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, DebugLoc DL, + unsigned DstReg, + const SmallVectorImpl<MachineOperand> &Cond, + unsigned TrueReg, unsigned FalseReg) const override; + void copyPhysReg(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, DebugLoc DL, + unsigned DestReg, unsigned SrcReg, + bool KillSrc) const override; + void storeRegToStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, + unsigned SrcReg, bool isKill, int FrameIndex, + const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI) const override; + + void storeRegToAddr(MachineFunction &MF, unsigned SrcReg, bool isKill, + SmallVectorImpl<MachineOperand> &Addr, + const TargetRegisterClass *RC, + MachineInstr::mmo_iterator MMOBegin, + MachineInstr::mmo_iterator MMOEnd, + SmallVectorImpl<MachineInstr*> &NewMIs) const; + + void loadRegFromStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, + unsigned DestReg, int FrameIndex, + const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI) const override; + + void loadRegFromAddr(MachineFunction &MF, unsigned DestReg, + SmallVectorImpl<MachineOperand> &Addr, + const TargetRegisterClass *RC, + MachineInstr::mmo_iterator MMOBegin, + MachineInstr::mmo_iterator MMOEnd, + SmallVectorImpl<MachineInstr*> &NewMIs) const; + + bool expandPostRAPseudo(MachineBasicBlock::iterator MI) const override; /// foldMemoryOperand - If this target supports it, fold a load or store of /// the specified stack slot into the specified machine instruction for the @@ -283,33 +289,33 @@ public: /// folding and return true, otherwise it should return false. If it folds /// the instruction, it is likely that the MachineInstruction the iterator /// references has been changed. - virtual MachineInstr* foldMemoryOperandImpl(MachineFunction &MF, - MachineInstr* MI, - const SmallVectorImpl<unsigned> &Ops, - int FrameIndex) const; + MachineInstr* foldMemoryOperandImpl(MachineFunction &MF, + MachineInstr* MI, + const SmallVectorImpl<unsigned> &Ops, + int FrameIndex) const override; /// foldMemoryOperand - Same as the previous version except it allows folding /// of any load and store from / to any address, not just from a specific /// stack slot. - virtual MachineInstr* foldMemoryOperandImpl(MachineFunction &MF, - MachineInstr* MI, - const SmallVectorImpl<unsigned> &Ops, - MachineInstr* LoadMI) const; + MachineInstr* foldMemoryOperandImpl(MachineFunction &MF, + MachineInstr* MI, + const SmallVectorImpl<unsigned> &Ops, + MachineInstr* LoadMI) const override; /// canFoldMemoryOperand - Returns true if the specified load / store is /// folding is possible. - virtual bool canFoldMemoryOperand(const MachineInstr*, - const SmallVectorImpl<unsigned> &) const; + bool canFoldMemoryOperand(const MachineInstr*, + const SmallVectorImpl<unsigned> &) const override; /// unfoldMemoryOperand - Separate a single instruction which folded a load or /// a store or a load and a store into two or more instruction. If this is /// possible, returns true as well as the new instructions by reference. - virtual bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI, - unsigned Reg, bool UnfoldLoad, bool UnfoldStore, - SmallVectorImpl<MachineInstr*> &NewMIs) const; + bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI, + unsigned Reg, bool UnfoldLoad, bool UnfoldStore, + SmallVectorImpl<MachineInstr*> &NewMIs) const override; - virtual bool unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, - SmallVectorImpl<SDNode*> &NewNodes) const; + bool unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, + SmallVectorImpl<SDNode*> &NewNodes) const override; /// getOpcodeAfterMemoryUnfold - Returns the opcode of the would be new /// instruction after load / store are unfolded from an instruction of the @@ -317,17 +323,17 @@ public: /// possible. If LoadRegIndex is non-null, it is filled in with the operand /// index of the operand which will hold the register holding the loaded /// value. - virtual unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, - bool UnfoldLoad, bool UnfoldStore, - unsigned *LoadRegIndex = 0) const; + unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, + bool UnfoldLoad, bool UnfoldStore, + unsigned *LoadRegIndex = 0) const override; /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler /// to determine if two loads are loading from the same base address. It /// should only return true if the base pointers are the same and the /// only differences between the two addresses are the offset. It also returns /// the offsets by reference. - virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, - int64_t &Offset1, int64_t &Offset2) const; + bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1, + int64_t &Offset2) const override; /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should @@ -337,21 +343,21 @@ public: /// from the common base address. It returns true if it decides it's desirable /// to schedule the two loads together. "NumLoads" is the number of loads that /// have already been scheduled after Load1. - virtual bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, - int64_t Offset1, int64_t Offset2, - unsigned NumLoads) const; + bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, + int64_t Offset1, int64_t Offset2, + unsigned NumLoads) const override; - virtual bool shouldScheduleAdjacent(MachineInstr* First, - MachineInstr *Second) const LLVM_OVERRIDE; + bool shouldScheduleAdjacent(MachineInstr* First, + MachineInstr *Second) const override; - virtual void getNoopForMachoTarget(MCInst &NopInst) const; + void getNoopForMachoTarget(MCInst &NopInst) const override; - virtual - bool ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const; + bool + ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override; /// isSafeToMoveRegClassDefs - Return true if it's safe to move a machine /// instruction that defines the specified register class. - bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const; + bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const override; static bool isX86_64ExtendedReg(const MachineOperand &MO) { if (!MO.isReg()) return false; @@ -365,16 +371,17 @@ public: unsigned getGlobalBaseReg(MachineFunction *MF) const; std::pair<uint16_t, uint16_t> - getExecutionDomain(const MachineInstr *MI) const; + getExecutionDomain(const MachineInstr *MI) const override; - void setExecutionDomain(MachineInstr *MI, unsigned Domain) const; + void setExecutionDomain(MachineInstr *MI, unsigned Domain) const override; - unsigned getPartialRegUpdateClearance(const MachineInstr *MI, unsigned OpNum, - const TargetRegisterInfo *TRI) const; + unsigned + getPartialRegUpdateClearance(const MachineInstr *MI, unsigned OpNum, + const TargetRegisterInfo *TRI) const override; unsigned getUndefRegClearance(const MachineInstr *MI, unsigned &OpNum, - const TargetRegisterInfo *TRI) const; + const TargetRegisterInfo *TRI) const override; void breakPartialRegDependency(MachineBasicBlock::iterator MI, unsigned OpNum, - const TargetRegisterInfo *TRI) const; + const TargetRegisterInfo *TRI) const override; MachineInstr* foldMemoryOperandImpl(MachineFunction &MF, MachineInstr* MI, @@ -382,27 +389,28 @@ public: const SmallVectorImpl<MachineOperand> &MOs, unsigned Size, unsigned Alignment) const; - bool isHighLatencyDef(int opc) const; + bool isHighLatencyDef(int opc) const override; bool hasHighOperandLatency(const InstrItineraryData *ItinData, const MachineRegisterInfo *MRI, const MachineInstr *DefMI, unsigned DefIdx, - const MachineInstr *UseMI, unsigned UseIdx) const; + const MachineInstr *UseMI, + unsigned UseIdx) const override; /// analyzeCompare - For a comparison instruction, return the source registers /// in SrcReg and SrcReg2 if having two register operands, and the value it /// compares against in CmpValue. Return true if the comparison instruction /// can be analyzed. - virtual bool analyzeCompare(const MachineInstr *MI, unsigned &SrcReg, - unsigned &SrcReg2, - int &CmpMask, int &CmpValue) const; + bool analyzeCompare(const MachineInstr *MI, unsigned &SrcReg, + unsigned &SrcReg2, int &CmpMask, + int &CmpValue) const override; /// optimizeCompareInstr - Check if there exists an earlier instruction that /// operates on the same source operands and sets flags in the same way as /// Compare; remove Compare if possible. - virtual bool optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, - unsigned SrcReg2, int CmpMask, int CmpValue, - const MachineRegisterInfo *MRI) const; + bool optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, + unsigned SrcReg2, int CmpMask, int CmpValue, + const MachineRegisterInfo *MRI) const override; /// optimizeLoadInstr - Try to remove the load by folding it to a register /// operand at the use. We fold the load instructions if and only if the @@ -411,10 +419,10 @@ public: /// defined by the load we are trying to fold. DefMI returns the machine /// instruction that defines FoldAsLoadDefReg, and the function returns /// the machine instruction generated due to folding. - virtual MachineInstr* optimizeLoadInstr(MachineInstr *MI, - const MachineRegisterInfo *MRI, - unsigned &FoldAsLoadDefReg, - MachineInstr *&DefMI) const; + MachineInstr* optimizeLoadInstr(MachineInstr *MI, + const MachineRegisterInfo *MRI, + unsigned &FoldAsLoadDefReg, + MachineInstr *&DefMI) const override; private: MachineInstr * convertToThreeAddressWithLEA(unsigned MIOpc, diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td index 6e5d543..8edf873 100644 --- a/lib/Target/X86/X86InstrInfo.td +++ b/lib/Target/X86/X86InstrInfo.td @@ -23,8 +23,8 @@ def SDTIntShiftDOp: SDTypeProfile<1, 3, def SDTX86CmpTest : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisSameAs<1, 2>]>; -def SDTX86Cmpsd : SDTypeProfile<1, 3, [SDTCisVT<0, f64>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; -def SDTX86Cmpss : SDTypeProfile<1, 3, [SDTCisVT<0, f32>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; +def SDTX86Cmps : SDTypeProfile<1, 3, [SDTCisFP<0>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; +//def SDTX86Cmpss : SDTypeProfile<1, 3, [SDTCisVT<0, f32>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>; def SDTX86Cmov : SDTypeProfile<1, 4, [SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, @@ -249,9 +249,6 @@ def X86xor_flag : SDNode<"X86ISD::XOR", SDTBinaryArithWithFlags, def X86and_flag : SDNode<"X86ISD::AND", SDTBinaryArithWithFlags, [SDNPCommutative]>; -def X86blsi : SDNode<"X86ISD::BLSI", SDTIntUnaryOp>; -def X86blsmsk : SDNode<"X86ISD::BLSMSK", SDTIntUnaryOp>; -def X86blsr : SDNode<"X86ISD::BLSR", SDTIntUnaryOp>; def X86bzhi : SDNode<"X86ISD::BZHI", SDTIntShiftOp>; def X86bextr : SDNode<"X86ISD::BEXTR", SDTIntBinOp>; @@ -445,6 +442,46 @@ def brtarget8 : Operand<OtherVT>; } +def X86SrcIdx8Operand : AsmOperandClass { + let Name = "SrcIdx8"; + let RenderMethod = "addSrcIdxOperands"; + let SuperClasses = [X86Mem8AsmOperand]; +} +def X86SrcIdx16Operand : AsmOperandClass { + let Name = "SrcIdx16"; + let RenderMethod = "addSrcIdxOperands"; + let SuperClasses = [X86Mem16AsmOperand]; +} +def X86SrcIdx32Operand : AsmOperandClass { + let Name = "SrcIdx32"; + let RenderMethod = "addSrcIdxOperands"; + let SuperClasses = [X86Mem32AsmOperand]; +} +def X86SrcIdx64Operand : AsmOperandClass { + let Name = "SrcIdx64"; + let RenderMethod = "addSrcIdxOperands"; + let SuperClasses = [X86Mem64AsmOperand]; +} +def X86DstIdx8Operand : AsmOperandClass { + let Name = "DstIdx8"; + let RenderMethod = "addDstIdxOperands"; + let SuperClasses = [X86Mem8AsmOperand]; +} +def X86DstIdx16Operand : AsmOperandClass { + let Name = "DstIdx16"; + let RenderMethod = "addDstIdxOperands"; + let SuperClasses = [X86Mem16AsmOperand]; +} +def X86DstIdx32Operand : AsmOperandClass { + let Name = "DstIdx32"; + let RenderMethod = "addDstIdxOperands"; + let SuperClasses = [X86Mem32AsmOperand]; +} +def X86DstIdx64Operand : AsmOperandClass { + let Name = "DstIdx64"; + let RenderMethod = "addDstIdxOperands"; + let SuperClasses = [X86Mem64AsmOperand]; +} def X86MemOffs8AsmOperand : AsmOperandClass { let Name = "MemOffs8"; let RenderMethod = "addMemOffsOperands"; @@ -465,19 +502,54 @@ def X86MemOffs64AsmOperand : AsmOperandClass { let RenderMethod = "addMemOffsOperands"; let SuperClasses = [X86Mem64AsmOperand]; } - let OperandType = "OPERAND_MEMORY" in { -def offset8 : Operand<i64> { +def srcidx8 : Operand<iPTR> { + let ParserMatchClass = X86SrcIdx8Operand; + let MIOperandInfo = (ops ptr_rc, i8imm); + let PrintMethod = "printSrcIdx8"; } +def srcidx16 : Operand<iPTR> { + let ParserMatchClass = X86SrcIdx16Operand; + let MIOperandInfo = (ops ptr_rc, i8imm); + let PrintMethod = "printSrcIdx16"; } +def srcidx32 : Operand<iPTR> { + let ParserMatchClass = X86SrcIdx32Operand; + let MIOperandInfo = (ops ptr_rc, i8imm); + let PrintMethod = "printSrcIdx32"; } +def srcidx64 : Operand<iPTR> { + let ParserMatchClass = X86SrcIdx64Operand; + let MIOperandInfo = (ops ptr_rc, i8imm); + let PrintMethod = "printSrcIdx64"; } +def dstidx8 : Operand<iPTR> { + let ParserMatchClass = X86DstIdx8Operand; + let MIOperandInfo = (ops ptr_rc); + let PrintMethod = "printDstIdx8"; } +def dstidx16 : Operand<iPTR> { + let ParserMatchClass = X86DstIdx16Operand; + let MIOperandInfo = (ops ptr_rc); + let PrintMethod = "printDstIdx16"; } +def dstidx32 : Operand<iPTR> { + let ParserMatchClass = X86DstIdx32Operand; + let MIOperandInfo = (ops ptr_rc); + let PrintMethod = "printDstIdx32"; } +def dstidx64 : Operand<iPTR> { + let ParserMatchClass = X86DstIdx64Operand; + let MIOperandInfo = (ops ptr_rc); + let PrintMethod = "printDstIdx64"; } +def offset8 : Operand<iPTR> { let ParserMatchClass = X86MemOffs8AsmOperand; + let MIOperandInfo = (ops i64imm, i8imm); let PrintMethod = "printMemOffs8"; } -def offset16 : Operand<i64> { +def offset16 : Operand<iPTR> { let ParserMatchClass = X86MemOffs16AsmOperand; + let MIOperandInfo = (ops i64imm, i8imm); let PrintMethod = "printMemOffs16"; } -def offset32 : Operand<i64> { +def offset32 : Operand<iPTR> { let ParserMatchClass = X86MemOffs32AsmOperand; + let MIOperandInfo = (ops i64imm, i8imm); let PrintMethod = "printMemOffs32"; } -def offset64 : Operand<i64> { +def offset64 : Operand<iPTR> { let ParserMatchClass = X86MemOffs64AsmOperand; + let MIOperandInfo = (ops i64imm, i8imm); let PrintMethod = "printMemOffs64"; } } @@ -510,6 +582,10 @@ def GR32orGR64 : RegisterOperand<GR32> { let ParserMatchClass = X86GR32orGR64AsmOperand; } +def AVX512RC : Operand<i32> { + let PrintMethod = "printRoundingControl"; + let OperandType = "OPERAND_IMMEDIATE"; +} // Sign-extended immediate classes. We don't need to define the full lattice // here because there is no instruction with an ambiguity between ImmSExti64i32 // and ImmSExti32i8. @@ -657,7 +733,8 @@ def HasSSE4A : Predicate<"Subtarget->hasSSE4A()">; def HasAVX : Predicate<"Subtarget->hasAVX()">; def HasAVX2 : Predicate<"Subtarget->hasAVX2()">; def HasAVX1Only : Predicate<"Subtarget->hasAVX() && !Subtarget->hasAVX2()">; -def HasAVX512 : Predicate<"Subtarget->hasAVX512()">; +def HasAVX512 : Predicate<"Subtarget->hasAVX512()">, + AssemblerPredicate<"FeatureAVX512", "AVX-512 ISA">; def UseAVX : Predicate<"Subtarget->hasAVX() && !Subtarget->hasAVX512()">; def UseAVX2 : Predicate<"Subtarget->hasAVX2() && !Subtarget->hasAVX512()">; def NoAVX512 : Predicate<"!Subtarget->hasAVX512()">; @@ -691,10 +768,16 @@ def HasPrefetchW : Predicate<"Subtarget->hasPRFCHW()">; def FPStackf32 : Predicate<"!Subtarget->hasSSE1()">; def FPStackf64 : Predicate<"!Subtarget->hasSSE2()">; def HasCmpxchg16b: Predicate<"Subtarget->hasCmpxchg16b()">; -def In32BitMode : Predicate<"!Subtarget->is64Bit()">, - AssemblerPredicate<"!Mode64Bit", "32-bit mode">; +def Not64BitMode : Predicate<"!Subtarget->is64Bit()">, + AssemblerPredicate<"!Mode64Bit", "Not 64-bit mode">; def In64BitMode : Predicate<"Subtarget->is64Bit()">, AssemblerPredicate<"Mode64Bit", "64-bit mode">; +def In16BitMode : Predicate<"Subtarget->is16Bit()">, + AssemblerPredicate<"Mode16Bit", "16-bit mode">; +def Not16BitMode : Predicate<"!Subtarget->is16Bit()">, + AssemblerPredicate<"!Mode16Bit", "Not 16-bit mode">; +def In32BitMode : Predicate<"Subtarget->is32Bit()">, + AssemblerPredicate<"Mode32Bit", "32-bit mode">; def IsWin64 : Predicate<"Subtarget->isTargetWin64()">; def IsNaCl : Predicate<"Subtarget->isTargetNaCl()">; def NotNaCl : Predicate<"!Subtarget->isTargetNaCl()">; @@ -845,10 +928,10 @@ def trunc_su : PatFrag<(ops node:$src), (trunc node:$src), [{ // Nop let neverHasSideEffects = 1, SchedRW = [WriteZero] in { def NOOP : I<0x90, RawFrm, (outs), (ins), "nop", [], IIC_NOP>; - def NOOPW : I<0x1f, MRM0m, (outs), (ins i16mem:$zero), - "nop{w}\t$zero", [], IIC_NOP>, TB, OpSize; - def NOOPL : I<0x1f, MRM0m, (outs), (ins i32mem:$zero), - "nop{l}\t$zero", [], IIC_NOP>, TB; + def NOOPW : I<0x1f, MRMXm, (outs), (ins i16mem:$zero), + "nop{w}\t$zero", [], IIC_NOP>, TB, OpSize16; + def NOOPL : I<0x1f, MRMXm, (outs), (ins i32mem:$zero), + "nop{l}\t$zero", [], IIC_NOP>, TB, OpSize32; } @@ -860,7 +943,7 @@ let SchedRW = [WriteALU] in { let Defs = [EBP, ESP], Uses = [EBP, ESP], mayLoad = 1, neverHasSideEffects=1 in def LEAVE : I<0xC9, RawFrm, (outs), (ins), "leave", [], IIC_LEAVE>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; let Defs = [RBP,RSP], Uses = [RBP,RSP], mayLoad = 1, neverHasSideEffects = 1 in def LEAVE64 : I<0xC9, RawFrm, @@ -875,98 +958,110 @@ def LEAVE64 : I<0xC9, RawFrm, let Defs = [ESP], Uses = [ESP], neverHasSideEffects=1 in { let mayLoad = 1, SchedRW = [WriteLoad] in { def POP16r : I<0x58, AddRegFrm, (outs GR16:$reg), (ins), "pop{w}\t$reg", [], - IIC_POP_REG16>, OpSize; + IIC_POP_REG16>, OpSize16; def POP32r : I<0x58, AddRegFrm, (outs GR32:$reg), (ins), "pop{l}\t$reg", [], - IIC_POP_REG>; + IIC_POP_REG>, OpSize32, Requires<[Not64BitMode]>; def POP16rmr: I<0x8F, MRM0r, (outs GR16:$reg), (ins), "pop{w}\t$reg", [], - IIC_POP_REG>, OpSize; + IIC_POP_REG>, OpSize16; def POP16rmm: I<0x8F, MRM0m, (outs), (ins i16mem:$dst), "pop{w}\t$dst", [], - IIC_POP_MEM>, OpSize; + IIC_POP_MEM>, OpSize16; def POP32rmr: I<0x8F, MRM0r, (outs GR32:$reg), (ins), "pop{l}\t$reg", [], - IIC_POP_REG>; + IIC_POP_REG>, OpSize32, Requires<[Not64BitMode]>; def POP32rmm: I<0x8F, MRM0m, (outs), (ins i32mem:$dst), "pop{l}\t$dst", [], - IIC_POP_MEM>; + IIC_POP_MEM>, OpSize32, Requires<[Not64BitMode]>; -def POPF16 : I<0x9D, RawFrm, (outs), (ins), "popf{w}", [], IIC_POP_F>, OpSize; +def POPF16 : I<0x9D, RawFrm, (outs), (ins), "popf{w}", [], IIC_POP_F>, + OpSize16; def POPF32 : I<0x9D, RawFrm, (outs), (ins), "popf{l|d}", [], IIC_POP_FD>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; } // mayLoad, SchedRW let mayStore = 1, SchedRW = [WriteStore] in { def PUSH16r : I<0x50, AddRegFrm, (outs), (ins GR16:$reg), "push{w}\t$reg",[], - IIC_PUSH_REG>, OpSize; + IIC_PUSH_REG>, OpSize16; def PUSH32r : I<0x50, AddRegFrm, (outs), (ins GR32:$reg), "push{l}\t$reg",[], - IIC_PUSH_REG>; + IIC_PUSH_REG>, OpSize32, Requires<[Not64BitMode]>; def PUSH16rmr: I<0xFF, MRM6r, (outs), (ins GR16:$reg), "push{w}\t$reg",[], - IIC_PUSH_REG>, OpSize; + IIC_PUSH_REG>, OpSize16; def PUSH16rmm: I<0xFF, MRM6m, (outs), (ins i16mem:$src), "push{w}\t$src",[], - IIC_PUSH_MEM>, - OpSize; + IIC_PUSH_MEM>, OpSize16; def PUSH32rmr: I<0xFF, MRM6r, (outs), (ins GR32:$reg), "push{l}\t$reg",[], - IIC_PUSH_REG>; + IIC_PUSH_REG>, OpSize32, Requires<[Not64BitMode]>; def PUSH32rmm: I<0xFF, MRM6m, (outs), (ins i32mem:$src), "push{l}\t$src",[], - IIC_PUSH_MEM>; - -def PUSHi8 : Ii8<0x6a, RawFrm, (outs), (ins i32i8imm:$imm), - "push{l}\t$imm", [], IIC_PUSH_IMM>; + IIC_PUSH_MEM>, OpSize32, Requires<[Not64BitMode]>; + +def PUSH16i8 : Ii8<0x6a, RawFrm, (outs), (ins i16i8imm:$imm), + "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16, + Requires<[Not64BitMode]>; +def PUSH32i8 : Ii8<0x6a, RawFrm, (outs), (ins i32i8imm:$imm), + "push{l}\t$imm", [], IIC_PUSH_IMM>, OpSize32, + Requires<[Not64BitMode]>; def PUSHi16 : Ii16<0x68, RawFrm, (outs), (ins i16imm:$imm), - "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize; + "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16, + Requires<[Not64BitMode]>; def PUSHi32 : Ii32<0x68, RawFrm, (outs), (ins i32imm:$imm), - "push{l}\t$imm", [], IIC_PUSH_IMM>; + "push{l}\t$imm", [], IIC_PUSH_IMM>, OpSize32, + Requires<[Not64BitMode]>; def PUSHF16 : I<0x9C, RawFrm, (outs), (ins), "pushf{w}", [], IIC_PUSH_F>, - OpSize; + OpSize16; def PUSHF32 : I<0x9C, RawFrm, (outs), (ins), "pushf{l|d}", [], IIC_PUSH_F>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; } // mayStore, SchedRW } let Defs = [RSP], Uses = [RSP], neverHasSideEffects=1 in { let mayLoad = 1, SchedRW = [WriteLoad] in { -def POP64r : I<0x58, AddRegFrm, - (outs GR64:$reg), (ins), "pop{q}\t$reg", [], IIC_POP_REG>; +def POP64r : I<0x58, AddRegFrm, (outs GR64:$reg), (ins), "pop{q}\t$reg", [], + IIC_POP_REG>, OpSize32, Requires<[In64BitMode]>; def POP64rmr: I<0x8F, MRM0r, (outs GR64:$reg), (ins), "pop{q}\t$reg", [], - IIC_POP_REG>; + IIC_POP_REG>, OpSize32, Requires<[In64BitMode]>; def POP64rmm: I<0x8F, MRM0m, (outs), (ins i64mem:$dst), "pop{q}\t$dst", [], - IIC_POP_MEM>; + IIC_POP_MEM>, OpSize32, Requires<[In64BitMode]>; } // mayLoad, SchedRW let mayStore = 1, SchedRW = [WriteStore] in { -def PUSH64r : I<0x50, AddRegFrm, - (outs), (ins GR64:$reg), "push{q}\t$reg", [], IIC_PUSH_REG>; +def PUSH64r : I<0x50, AddRegFrm, (outs), (ins GR64:$reg), "push{q}\t$reg", [], + IIC_PUSH_REG>, OpSize32, Requires<[In64BitMode]>; def PUSH64rmr: I<0xFF, MRM6r, (outs), (ins GR64:$reg), "push{q}\t$reg", [], - IIC_PUSH_REG>; + IIC_PUSH_REG>, OpSize32, Requires<[In64BitMode]>; def PUSH64rmm: I<0xFF, MRM6m, (outs), (ins i64mem:$src), "push{q}\t$src", [], - IIC_PUSH_MEM>; + IIC_PUSH_MEM>, OpSize32, Requires<[In64BitMode]>; } // mayStore, SchedRW } let Defs = [RSP], Uses = [RSP], neverHasSideEffects = 1, mayStore = 1, SchedRW = [WriteStore] in { def PUSH64i8 : Ii8<0x6a, RawFrm, (outs), (ins i64i8imm:$imm), - "push{q}\t$imm", [], IIC_PUSH_IMM>; + "push{q}\t$imm", [], IIC_PUSH_IMM>, Requires<[In64BitMode]>; def PUSH64i16 : Ii16<0x68, RawFrm, (outs), (ins i16imm:$imm), - "push{q}\t$imm", [], IIC_PUSH_IMM>; -def PUSH64i32 : Ii32<0x68, RawFrm, (outs), (ins i64i32imm:$imm), - "push{q}\t$imm", [], IIC_PUSH_IMM>; + "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16, + Requires<[In64BitMode]>; +def PUSH64i32 : Ii32S<0x68, RawFrm, (outs), (ins i64i32imm:$imm), + "push{q}\t$imm", [], IIC_PUSH_IMM>, OpSize32, + Requires<[In64BitMode]>; } let Defs = [RSP, EFLAGS], Uses = [RSP], mayLoad = 1, neverHasSideEffects=1 in def POPF64 : I<0x9D, RawFrm, (outs), (ins), "popfq", [], IIC_POP_FD>, - Requires<[In64BitMode]>, Sched<[WriteLoad]>; + OpSize32, Requires<[In64BitMode]>, Sched<[WriteLoad]>; let Defs = [RSP], Uses = [RSP, EFLAGS], mayStore = 1, neverHasSideEffects=1 in def PUSHF64 : I<0x9C, RawFrm, (outs), (ins), "pushfq", [], IIC_PUSH_F>, - Requires<[In64BitMode]>, Sched<[WriteStore]>; + OpSize32, Requires<[In64BitMode]>, Sched<[WriteStore]>; let Defs = [EDI, ESI, EBP, EBX, EDX, ECX, EAX, ESP], Uses = [ESP], mayLoad = 1, neverHasSideEffects = 1, SchedRW = [WriteLoad] in { -def POPA32 : I<0x61, RawFrm, (outs), (ins), "popa{l}", [], IIC_POP_A>, - Requires<[In32BitMode]>; +def POPA32 : I<0x61, RawFrm, (outs), (ins), "popal", [], IIC_POP_A>, + OpSize32, Requires<[Not64BitMode]>; +def POPA16 : I<0x61, RawFrm, (outs), (ins), "popaw", [], IIC_POP_A>, + OpSize16, Requires<[Not64BitMode]>; } let Defs = [ESP], Uses = [EDI, ESI, EBP, EBX, EDX, ECX, EAX, ESP], mayStore = 1, neverHasSideEffects = 1, SchedRW = [WriteStore] in { -def PUSHA32 : I<0x60, RawFrm, (outs), (ins), "pusha{l}", [], IIC_PUSH_A>, - Requires<[In32BitMode]>; +def PUSHA32 : I<0x60, RawFrm, (outs), (ins), "pushal", [], IIC_PUSH_A>, + OpSize32, Requires<[Not64BitMode]>; +def PUSHA16 : I<0x60, RawFrm, (outs), (ins), "pushaw", [], IIC_PUSH_A>, + OpSize16, Requires<[Not64BitMode]>; } let Constraints = "$src = $dst", SchedRW = [WriteALU] in { @@ -974,7 +1069,7 @@ let Constraints = "$src = $dst", SchedRW = [WriteALU] in { def BSWAP32r : I<0xC8, AddRegFrm, (outs GR32:$dst), (ins GR32:$src), "bswap{l}\t$dst", - [(set GR32:$dst, (bswap GR32:$src))], IIC_BSWAP>, TB; + [(set GR32:$dst, (bswap GR32:$src))], IIC_BSWAP>, OpSize32, TB; def BSWAP64r : RI<0xC8, AddRegFrm, (outs GR64:$dst), (ins GR64:$src), "bswap{q}\t$dst", @@ -986,86 +1081,106 @@ let Defs = [EFLAGS] in { def BSF16rr : I<0xBC, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), "bsf{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, EFLAGS, (X86bsf GR16:$src))], - IIC_BIT_SCAN_REG>, TB, OpSize, Sched<[WriteShift]>; + IIC_BIT_SCAN_REG>, PS, OpSize16, Sched<[WriteShift]>; def BSF16rm : I<0xBC, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "bsf{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, EFLAGS, (X86bsf (loadi16 addr:$src)))], - IIC_BIT_SCAN_MEM>, TB, OpSize, Sched<[WriteShiftLd]>; + IIC_BIT_SCAN_MEM>, PS, OpSize16, Sched<[WriteShiftLd]>; def BSF32rr : I<0xBC, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), "bsf{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, EFLAGS, (X86bsf GR32:$src))], - IIC_BIT_SCAN_REG>, TB, - Sched<[WriteShift]>; + IIC_BIT_SCAN_REG>, PS, OpSize32, Sched<[WriteShift]>; def BSF32rm : I<0xBC, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "bsf{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, EFLAGS, (X86bsf (loadi32 addr:$src)))], - IIC_BIT_SCAN_MEM>, TB, Sched<[WriteShiftLd]>; + IIC_BIT_SCAN_MEM>, PS, OpSize32, Sched<[WriteShiftLd]>; def BSF64rr : RI<0xBC, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "bsf{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, EFLAGS, (X86bsf GR64:$src))], - IIC_BIT_SCAN_REG>, TB, Sched<[WriteShift]>; + IIC_BIT_SCAN_REG>, PS, Sched<[WriteShift]>; def BSF64rm : RI<0xBC, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "bsf{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, EFLAGS, (X86bsf (loadi64 addr:$src)))], - IIC_BIT_SCAN_MEM>, TB, Sched<[WriteShiftLd]>; + IIC_BIT_SCAN_MEM>, PS, Sched<[WriteShiftLd]>; def BSR16rr : I<0xBD, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), "bsr{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, EFLAGS, (X86bsr GR16:$src))], - IIC_BIT_SCAN_REG>, - TB, OpSize, Sched<[WriteShift]>; + IIC_BIT_SCAN_REG>, PS, OpSize16, Sched<[WriteShift]>; def BSR16rm : I<0xBD, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "bsr{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, EFLAGS, (X86bsr (loadi16 addr:$src)))], - IIC_BIT_SCAN_MEM>, TB, - OpSize, Sched<[WriteShiftLd]>; + IIC_BIT_SCAN_MEM>, PS, OpSize16, Sched<[WriteShiftLd]>; def BSR32rr : I<0xBD, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), "bsr{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, EFLAGS, (X86bsr GR32:$src))], - IIC_BIT_SCAN_REG>, TB, - Sched<[WriteShift]>; + IIC_BIT_SCAN_REG>, PS, OpSize32, Sched<[WriteShift]>; def BSR32rm : I<0xBD, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "bsr{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, EFLAGS, (X86bsr (loadi32 addr:$src)))], - IIC_BIT_SCAN_MEM>, TB, Sched<[WriteShiftLd]>; + IIC_BIT_SCAN_MEM>, PS, OpSize32, Sched<[WriteShiftLd]>; def BSR64rr : RI<0xBD, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "bsr{q}\t{$src, $dst|$dst, $src}", - [(set GR64:$dst, EFLAGS, (X86bsr GR64:$src))], IIC_BIT_SCAN_REG>, TB, - Sched<[WriteShift]>; + [(set GR64:$dst, EFLAGS, (X86bsr GR64:$src))], + IIC_BIT_SCAN_REG>, PS, Sched<[WriteShift]>; def BSR64rm : RI<0xBD, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "bsr{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, EFLAGS, (X86bsr (loadi64 addr:$src)))], - IIC_BIT_SCAN_MEM>, TB, Sched<[WriteShiftLd]>; + IIC_BIT_SCAN_MEM>, PS, Sched<[WriteShiftLd]>; } // Defs = [EFLAGS] let SchedRW = [WriteMicrocoded] in { // These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI let Defs = [EDI,ESI], Uses = [EDI,ESI,EFLAGS] in { -def MOVSB : I<0xA4, RawFrm, (outs), (ins), "movsb", [], IIC_MOVS>; -def MOVSW : I<0xA5, RawFrm, (outs), (ins), "movsw", [], IIC_MOVS>, OpSize; -def MOVSD : I<0xA5, RawFrm, (outs), (ins), "movs{l|d}", [], IIC_MOVS>; -def MOVSQ : RI<0xA5, RawFrm, (outs), (ins), "movsq", [], IIC_MOVS>; +def MOVSB : I<0xA4, RawFrmDstSrc, (outs dstidx8:$dst), (ins srcidx8:$src), + "movsb\t{$src, $dst|$dst, $src}", [], IIC_MOVS>; +def MOVSW : I<0xA5, RawFrmDstSrc, (outs dstidx16:$dst), (ins srcidx16:$src), + "movsw\t{$src, $dst|$dst, $src}", [], IIC_MOVS>, OpSize16; +def MOVSL : I<0xA5, RawFrmDstSrc, (outs dstidx32:$dst), (ins srcidx32:$src), + "movs{l|d}\t{$src, $dst|$dst, $src}", [], IIC_MOVS>, OpSize32; +def MOVSQ : RI<0xA5, RawFrmDstSrc, (outs dstidx64:$dst), (ins srcidx64:$src), + "movsq\t{$src, $dst|$dst, $src}", [], IIC_MOVS>; } // These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI let Defs = [EDI], Uses = [AL,EDI,EFLAGS] in -def STOSB : I<0xAA, RawFrm, (outs), (ins), "stosb", [], IIC_STOS>; +def STOSB : I<0xAA, RawFrmDst, (outs dstidx8:$dst), (ins), + "stosb\t{%al, $dst|$dst, al}", [], IIC_STOS>; let Defs = [EDI], Uses = [AX,EDI,EFLAGS] in -def STOSW : I<0xAB, RawFrm, (outs), (ins), "stosw", [], IIC_STOS>, OpSize; +def STOSW : I<0xAB, RawFrmDst, (outs dstidx16:$dst), (ins), + "stosw\t{%ax, $dst|$dst, ax}", [], IIC_STOS>, OpSize16; let Defs = [EDI], Uses = [EAX,EDI,EFLAGS] in -def STOSD : I<0xAB, RawFrm, (outs), (ins), "stos{l|d}", [], IIC_STOS>; +def STOSL : I<0xAB, RawFrmDst, (outs dstidx32:$dst), (ins), + "stos{l|d}\t{%eax, $dst|$dst, eax}", [], IIC_STOS>, OpSize32; let Defs = [RCX,RDI], Uses = [RAX,RCX,RDI,EFLAGS] in -def STOSQ : RI<0xAB, RawFrm, (outs), (ins), "stosq", [], IIC_STOS>; +def STOSQ : RI<0xAB, RawFrmDst, (outs dstidx64:$dst), (ins), + "stosq\t{%rax, $dst|$dst, rax}", [], IIC_STOS>; -def SCAS8 : I<0xAE, RawFrm, (outs), (ins), "scasb", [], IIC_SCAS>; -def SCAS16 : I<0xAF, RawFrm, (outs), (ins), "scasw", [], IIC_SCAS>, OpSize; -def SCAS32 : I<0xAF, RawFrm, (outs), (ins), "scas{l|d}", [], IIC_SCAS>; -def SCAS64 : RI<0xAF, RawFrm, (outs), (ins), "scasq", [], IIC_SCAS>; +// These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI +let Defs = [EDI,EFLAGS], Uses = [AL,EDI,EFLAGS] in +def SCASB : I<0xAE, RawFrmDst, (outs), (ins dstidx8:$dst), + "scasb\t{$dst, %al|al, $dst}", [], IIC_SCAS>; +let Defs = [EDI,EFLAGS], Uses = [AX,EDI,EFLAGS] in +def SCASW : I<0xAF, RawFrmDst, (outs), (ins dstidx16:$dst), + "scasw\t{$dst, %ax|ax, $dst}", [], IIC_SCAS>, OpSize16; +let Defs = [EDI,EFLAGS], Uses = [EAX,EDI,EFLAGS] in +def SCASL : I<0xAF, RawFrmDst, (outs), (ins dstidx32:$dst), + "scas{l|d}\t{$dst, %eax|eax, $dst}", [], IIC_SCAS>, OpSize32; +let Defs = [EDI,EFLAGS], Uses = [RAX,EDI,EFLAGS] in +def SCASQ : RI<0xAF, RawFrmDst, (outs), (ins dstidx64:$dst), + "scasq\t{$dst, %rax|rax, $dst}", [], IIC_SCAS>; -def CMPS8 : I<0xA6, RawFrm, (outs), (ins), "cmpsb", [], IIC_CMPS>; -def CMPS16 : I<0xA7, RawFrm, (outs), (ins), "cmpsw", [], IIC_CMPS>, OpSize; -def CMPS32 : I<0xA7, RawFrm, (outs), (ins), "cmps{l|d}", [], IIC_CMPS>; -def CMPS64 : RI<0xA7, RawFrm, (outs), (ins), "cmpsq", [], IIC_CMPS>; +// These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI +let Defs = [EDI,ESI,EFLAGS], Uses = [EDI,ESI,EFLAGS] in { +def CMPSB : I<0xA6, RawFrmDstSrc, (outs), (ins dstidx8:$dst, srcidx8:$src), + "cmpsb\t{$dst, $src|$src, $dst}", [], IIC_CMPS>; +def CMPSW : I<0xA7, RawFrmDstSrc, (outs), (ins dstidx16:$dst, srcidx16:$src), + "cmpsw\t{$dst, $src|$src, $dst}", [], IIC_CMPS>, OpSize16; +def CMPSL : I<0xA7, RawFrmDstSrc, (outs), (ins dstidx32:$dst, srcidx32:$src), + "cmps{l|d}\t{$dst, $src|$src, $dst}", [], IIC_CMPS>, OpSize32; +def CMPSQ : RI<0xA7, RawFrmDstSrc, (outs), (ins dstidx64:$dst, srcidx64:$src), + "cmpsq\t{$dst, $src|$src, $dst}", [], IIC_CMPS>; +} } // SchedRW //===----------------------------------------------------------------------===// @@ -1076,9 +1191,9 @@ let neverHasSideEffects = 1 in { def MOV8rr : I<0x88, MRMDestReg, (outs GR8 :$dst), (ins GR8 :$src), "mov{b}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; def MOV16rr : I<0x89, MRMDestReg, (outs GR16:$dst), (ins GR16:$src), - "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize; + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize16; def MOV32rr : I<0x89, MRMDestReg, (outs GR32:$dst), (ins GR32:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize32; def MOV64rr : RI<0x89, MRMDestReg, (outs GR64:$dst), (ins GR64:$src), "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; } @@ -1089,16 +1204,28 @@ def MOV8ri : Ii8 <0xB0, AddRegFrm, (outs GR8 :$dst), (ins i8imm :$src), [(set GR8:$dst, imm:$src)], IIC_MOV>; def MOV16ri : Ii16<0xB8, AddRegFrm, (outs GR16:$dst), (ins i16imm:$src), "mov{w}\t{$src, $dst|$dst, $src}", - [(set GR16:$dst, imm:$src)], IIC_MOV>, OpSize; + [(set GR16:$dst, imm:$src)], IIC_MOV>, OpSize16; def MOV32ri : Ii32<0xB8, AddRegFrm, (outs GR32:$dst), (ins i32imm:$src), "mov{l}\t{$src, $dst|$dst, $src}", - [(set GR32:$dst, imm:$src)], IIC_MOV>; + [(set GR32:$dst, imm:$src)], IIC_MOV>, OpSize32; +def MOV64ri32 : RIi32S<0xC7, MRM0r, (outs GR64:$dst), (ins i64i32imm:$src), + "mov{q}\t{$src, $dst|$dst, $src}", + [(set GR64:$dst, i64immSExt32:$src)], IIC_MOV>; +} +let isReMaterializable = 1 in { def MOV64ri : RIi64<0xB8, AddRegFrm, (outs GR64:$dst), (ins i64imm:$src), "movabs{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, imm:$src)], IIC_MOV>; -def MOV64ri32 : RIi32<0xC7, MRM0r, (outs GR64:$dst), (ins i64i32imm:$src), - "mov{q}\t{$src, $dst|$dst, $src}", - [(set GR64:$dst, i64immSExt32:$src)], IIC_MOV>; +} + +// Longer forms that use a ModR/M byte. Needed for disassembler +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { +def MOV8ri_alt : Ii8 <0xC6, MRM0r, (outs GR8 :$dst), (ins i8imm :$src), + "mov{b}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; +def MOV16ri_alt : Ii16<0xC7, MRM0r, (outs GR16:$dst), (ins i16imm:$src), + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize16; +def MOV32ri_alt : Ii32<0xC7, MRM0r, (outs GR32:$dst), (ins i32imm:$src), + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize32; } } // SchedRW @@ -1108,84 +1235,125 @@ def MOV8mi : Ii8 <0xC6, MRM0m, (outs), (ins i8mem :$dst, i8imm :$src), [(store (i8 imm:$src), addr:$dst)], IIC_MOV_MEM>; def MOV16mi : Ii16<0xC7, MRM0m, (outs), (ins i16mem:$dst, i16imm:$src), "mov{w}\t{$src, $dst|$dst, $src}", - [(store (i16 imm:$src), addr:$dst)], IIC_MOV_MEM>, OpSize; + [(store (i16 imm:$src), addr:$dst)], IIC_MOV_MEM>, OpSize16; def MOV32mi : Ii32<0xC7, MRM0m, (outs), (ins i32mem:$dst, i32imm:$src), "mov{l}\t{$src, $dst|$dst, $src}", - [(store (i32 imm:$src), addr:$dst)], IIC_MOV_MEM>; -def MOV64mi32 : RIi32<0xC7, MRM0m, (outs), (ins i64mem:$dst, i64i32imm:$src), - "mov{q}\t{$src, $dst|$dst, $src}", - [(store i64immSExt32:$src, addr:$dst)], IIC_MOV_MEM>; + [(store (i32 imm:$src), addr:$dst)], IIC_MOV_MEM>, OpSize32; +def MOV64mi32 : RIi32S<0xC7, MRM0m, (outs), (ins i64mem:$dst, i64i32imm:$src), + "mov{q}\t{$src, $dst|$dst, $src}", + [(store i64immSExt32:$src, addr:$dst)], IIC_MOV_MEM>; } // SchedRW let hasSideEffects = 0 in { /// moffs8, moffs16 and moffs32 versions of moves. The immediate is a -/// 32-bit offset from the PC. These are only valid in x86-32 mode. +/// 32-bit offset from the segment base. These are only valid in x86-32 mode. let SchedRW = [WriteALU] in { let mayLoad = 1 in { -def MOV8o8a : Ii32 <0xA0, RawFrm, (outs), (ins offset8:$src), +let Defs = [AL] in +def MOV8o8a : Ii32 <0xA0, RawFrmMemOffs, (outs), (ins offset8:$src), "mov{b}\t{$src, %al|al, $src}", [], IIC_MOV_MEM>, Requires<[In32BitMode]>; -def MOV16o16a : Ii32 <0xA1, RawFrm, (outs), (ins offset16:$src), - "mov{w}\t{$src, %ax|ax, $src}", [], IIC_MOV_MEM>, OpSize, - Requires<[In32BitMode]>; -def MOV32o32a : Ii32 <0xA1, RawFrm, (outs), (ins offset32:$src), +let Defs = [AX] in +def MOV16o16a : Ii32 <0xA1, RawFrmMemOffs, (outs), (ins offset16:$src), + "mov{w}\t{$src, %ax|ax, $src}", [], IIC_MOV_MEM>, + OpSize16, Requires<[In32BitMode]>; +let Defs = [EAX] in +def MOV32o32a : Ii32 <0xA1, RawFrmMemOffs, (outs), (ins offset32:$src), "mov{l}\t{$src, %eax|eax, $src}", [], IIC_MOV_MEM>, - Requires<[In32BitMode]>; + OpSize32, Requires<[In32BitMode]>; + +let Defs = [AL] in +def MOV8o8a_16 : Ii16 <0xA0, RawFrmMemOffs, (outs), (ins offset8:$src), + "mov{b}\t{$src, %al|al, $src}", [], IIC_MOV_MEM>, + AdSize, Requires<[In16BitMode]>; +let Defs = [AX] in +def MOV16o16a_16 : Ii16 <0xA1, RawFrmMemOffs, (outs), (ins offset16:$src), + "mov{w}\t{$src, %ax|ax, $src}", [], IIC_MOV_MEM>, + OpSize16, AdSize, Requires<[In16BitMode]>; +let Defs = [EAX] in +def MOV32o32a_16 : Ii16 <0xA1, RawFrmMemOffs, (outs), (ins offset32:$src), + "mov{l}\t{$src, %eax|eax, $src}", [], IIC_MOV_MEM>, + AdSize, OpSize32, Requires<[In16BitMode]>; } let mayStore = 1 in { -def MOV8ao8 : Ii32 <0xA2, RawFrm, (outs offset8:$dst), (ins), +let Uses = [AL] in +def MOV8ao8 : Ii32 <0xA2, RawFrmMemOffs, (outs offset8:$dst), (ins), "mov{b}\t{%al, $dst|$dst, al}", [], IIC_MOV_MEM>, Requires<[In32BitMode]>; -def MOV16ao16 : Ii32 <0xA3, RawFrm, (outs offset16:$dst), (ins), - "mov{w}\t{%ax, $dst|$dst, ax}", [], IIC_MOV_MEM>, OpSize, - Requires<[In32BitMode]>; -def MOV32ao32 : Ii32 <0xA3, RawFrm, (outs offset32:$dst), (ins), +let Uses = [AX] in +def MOV16ao16 : Ii32 <0xA3, RawFrmMemOffs, (outs offset16:$dst), (ins), + "mov{w}\t{%ax, $dst|$dst, ax}", [], IIC_MOV_MEM>, + OpSize16, Requires<[In32BitMode]>; +let Uses = [EAX] in +def MOV32ao32 : Ii32 <0xA3, RawFrmMemOffs, (outs offset32:$dst), (ins), "mov{l}\t{%eax, $dst|$dst, eax}", [], IIC_MOV_MEM>, - Requires<[In32BitMode]>; + OpSize32, Requires<[In32BitMode]>; + +let Uses = [AL] in +def MOV8ao8_16 : Ii16 <0xA2, RawFrmMemOffs, (outs offset8:$dst), (ins), + "mov{b}\t{%al, $dst|$dst, al}", [], IIC_MOV_MEM>, + AdSize, Requires<[In16BitMode]>; +let Uses = [AX] in +def MOV16ao16_16 : Ii16 <0xA3, RawFrmMemOffs, (outs offset16:$dst), (ins), + "mov{w}\t{%ax, $dst|$dst, ax}", [], IIC_MOV_MEM>, + OpSize16, AdSize, Requires<[In16BitMode]>; +let Uses = [EAX] in +def MOV32ao32_16 : Ii16 <0xA3, RawFrmMemOffs, (outs offset32:$dst), (ins), + "mov{l}\t{%eax, $dst|$dst, eax}", [], IIC_MOV_MEM>, + OpSize32, AdSize, Requires<[In16BitMode]>; } } // These forms all have full 64-bit absolute addresses in their instructions // and use the movabs mnemonic to indicate this specific form. let mayLoad = 1 in { -def MOV64o8a : RIi64_NOREX<0xA0, RawFrm, (outs), (ins offset8:$src), +let Defs = [AL] in +def MOV64o8a : RIi64_NOREX<0xA0, RawFrmMemOffs, (outs), (ins offset8:$src), "movabs{b}\t{$src, %al|al, $src}", []>, Requires<[In64BitMode]>; -def MOV64o16a : RIi64_NOREX<0xA1, RawFrm, (outs), (ins offset16:$src), - "movabs{w}\t{$src, %ax|ax, $src}", []>, OpSize, +let Defs = [AX] in +def MOV64o16a : RIi64_NOREX<0xA1, RawFrmMemOffs, (outs), (ins offset16:$src), + "movabs{w}\t{$src, %ax|ax, $src}", []>, OpSize16, Requires<[In64BitMode]>; -def MOV64o32a : RIi64_NOREX<0xA1, RawFrm, (outs), (ins offset32:$src), - "movabs{l}\t{$src, %eax|eax, $src}", []>, +let Defs = [EAX] in +def MOV64o32a : RIi64_NOREX<0xA1, RawFrmMemOffs, (outs), (ins offset32:$src), + "movabs{l}\t{$src, %eax|eax, $src}", []>, OpSize32, Requires<[In64BitMode]>; -def MOV64o64a : RIi64<0xA1, RawFrm, (outs), (ins offset64:$src), +let Defs = [RAX] in +def MOV64o64a : RIi64<0xA1, RawFrmMemOffs, (outs), (ins offset64:$src), "movabs{q}\t{$src, %rax|rax, $src}", []>, Requires<[In64BitMode]>; } let mayStore = 1 in { -def MOV64ao8 : RIi64_NOREX<0xA2, RawFrm, (outs offset8:$dst), (ins), +let Uses = [AL] in +def MOV64ao8 : RIi64_NOREX<0xA2, RawFrmMemOffs, (outs offset8:$dst), (ins), "movabs{b}\t{%al, $dst|$dst, al}", []>, Requires<[In64BitMode]>; -def MOV64ao16 : RIi64_NOREX<0xA3, RawFrm, (outs offset16:$dst), (ins), - "movabs{w}\t{%ax, $dst|$dst, ax}", []>, OpSize, +let Uses = [AX] in +def MOV64ao16 : RIi64_NOREX<0xA3, RawFrmMemOffs, (outs offset16:$dst), (ins), + "movabs{w}\t{%ax, $dst|$dst, ax}", []>, OpSize16, Requires<[In64BitMode]>; -def MOV64ao32 : RIi64_NOREX<0xA3, RawFrm, (outs offset32:$dst), (ins), - "movabs{l}\t{%eax, $dst|$dst, eax}", []>, +let Uses = [EAX] in +def MOV64ao32 : RIi64_NOREX<0xA3, RawFrmMemOffs, (outs offset32:$dst), (ins), + "movabs{l}\t{%eax, $dst|$dst, eax}", []>, OpSize32, Requires<[In64BitMode]>; -def MOV64ao64 : RIi64<0xA3, RawFrm, (outs offset64:$dst), (ins), +let Uses = [RAX] in +def MOV64ao64 : RIi64<0xA3, RawFrmMemOffs, (outs offset64:$dst), (ins), "movabs{q}\t{%rax, $dst|$dst, rax}", []>, Requires<[In64BitMode]>; } } // hasSideEffects = 0 -let isCodeGenOnly = 1, hasSideEffects = 0, SchedRW = [WriteMove] in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, + SchedRW = [WriteMove] in { def MOV8rr_REV : I<0x8A, MRMSrcReg, (outs GR8:$dst), (ins GR8:$src), "mov{b}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; def MOV16rr_REV : I<0x8B, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), - "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize; + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize16; def MOV32rr_REV : I<0x8B, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV>, OpSize32; def MOV64rr_REV : RI<0x8B, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV>; } @@ -1196,10 +1364,10 @@ def MOV8rm : I<0x8A, MRMSrcMem, (outs GR8 :$dst), (ins i8mem :$src), [(set GR8:$dst, (loadi8 addr:$src))], IIC_MOV_MEM>; def MOV16rm : I<0x8B, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "mov{w}\t{$src, $dst|$dst, $src}", - [(set GR16:$dst, (loadi16 addr:$src))], IIC_MOV_MEM>, OpSize; + [(set GR16:$dst, (loadi16 addr:$src))], IIC_MOV_MEM>, OpSize16; def MOV32rm : I<0x8B, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "mov{l}\t{$src, $dst|$dst, $src}", - [(set GR32:$dst, (loadi32 addr:$src))], IIC_MOV_MEM>; + [(set GR32:$dst, (loadi32 addr:$src))], IIC_MOV_MEM>, OpSize32; def MOV64rm : RI<0x8B, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "mov{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (load addr:$src))], IIC_MOV_MEM>; @@ -1211,10 +1379,10 @@ def MOV8mr : I<0x88, MRMDestMem, (outs), (ins i8mem :$dst, GR8 :$src), [(store GR8:$src, addr:$dst)], IIC_MOV_MEM>; def MOV16mr : I<0x89, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src), "mov{w}\t{$src, $dst|$dst, $src}", - [(store GR16:$src, addr:$dst)], IIC_MOV_MEM>, OpSize; + [(store GR16:$src, addr:$dst)], IIC_MOV_MEM>, OpSize16; def MOV32mr : I<0x89, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), "mov{l}\t{$src, $dst|$dst, $src}", - [(store GR32:$src, addr:$dst)], IIC_MOV_MEM>; + [(store GR32:$src, addr:$dst)], IIC_MOV_MEM>, OpSize32; def MOV64mr : RI<0x89, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), "mov{q}\t{$src, $dst|$dst, $src}", [(store GR64:$src, addr:$dst)], IIC_MOV_MEM>; @@ -1261,10 +1429,11 @@ let SchedRW = [WriteALU] in { def BT16rr : I<0xA3, MRMDestReg, (outs), (ins GR16:$src1, GR16:$src2), "bt{w}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt GR16:$src1, GR16:$src2))], IIC_BT_RR>, - OpSize, TB; + OpSize16, TB; def BT32rr : I<0xA3, MRMDestReg, (outs), (ins GR32:$src1, GR32:$src2), "bt{l}\t{$src2, $src1|$src1, $src2}", - [(set EFLAGS, (X86bt GR32:$src1, GR32:$src2))], IIC_BT_RR>, TB; + [(set EFLAGS, (X86bt GR32:$src1, GR32:$src2))], IIC_BT_RR>, + OpSize32, TB; def BT64rr : RI<0xA3, MRMDestReg, (outs), (ins GR64:$src1, GR64:$src2), "bt{q}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt GR64:$src1, GR64:$src2))], IIC_BT_RR>, TB; @@ -1281,13 +1450,13 @@ let mayLoad = 1, hasSideEffects = 0, SchedRW = [WriteALULd] in { // [(X86bt (loadi16 addr:$src1), GR16:$src2), // (implicit EFLAGS)] [], IIC_BT_MR - >, OpSize, TB, Requires<[FastBTMem]>; + >, OpSize16, TB, Requires<[FastBTMem]>; def BT32mr : I<0xA3, MRMDestMem, (outs), (ins i32mem:$src1, GR32:$src2), "bt{l}\t{$src2, $src1|$src1, $src2}", // [(X86bt (loadi32 addr:$src1), GR32:$src2), // (implicit EFLAGS)] [], IIC_BT_MR - >, TB, Requires<[FastBTMem]>; + >, OpSize32, TB, Requires<[FastBTMem]>; def BT64mr : RI<0xA3, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), "bt{q}\t{$src2, $src1|$src1, $src2}", // [(X86bt (loadi64 addr:$src1), GR64:$src2), @@ -1300,11 +1469,11 @@ let SchedRW = [WriteALU] in { def BT16ri8 : Ii8<0xBA, MRM4r, (outs), (ins GR16:$src1, i16i8imm:$src2), "bt{w}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt GR16:$src1, i16immSExt8:$src2))], - IIC_BT_RI>, OpSize, TB; + IIC_BT_RI>, OpSize16, TB; def BT32ri8 : Ii8<0xBA, MRM4r, (outs), (ins GR32:$src1, i32i8imm:$src2), "bt{l}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt GR32:$src1, i32immSExt8:$src2))], - IIC_BT_RI>, TB; + IIC_BT_RI>, OpSize32, TB; def BT64ri8 : RIi8<0xBA, MRM4r, (outs), (ins GR64:$src1, i64i8imm:$src2), "bt{q}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt GR64:$src1, i64immSExt8:$src2))], @@ -1318,11 +1487,11 @@ let SchedRW = [WriteALU] in { def BT16mi8 : Ii8<0xBA, MRM4m, (outs), (ins i16mem:$src1, i16i8imm:$src2), "bt{w}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt (loadi16 addr:$src1), i16immSExt8:$src2)) - ], IIC_BT_MI>, OpSize, TB; + ], IIC_BT_MI>, OpSize16, TB; def BT32mi8 : Ii8<0xBA, MRM4m, (outs), (ins i32mem:$src1, i32i8imm:$src2), "bt{l}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt (loadi32 addr:$src1), i32immSExt8:$src2)) - ], IIC_BT_MI>, TB; + ], IIC_BT_MI>, OpSize32, TB; def BT64mi8 : RIi8<0xBA, MRM4m, (outs), (ins i64mem:$src1, i64i8imm:$src2), "bt{q}\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86bt (loadi64 addr:$src1), @@ -1333,9 +1502,10 @@ let hasSideEffects = 0 in { let SchedRW = [WriteALU] in { def BTC16rr : I<0xBB, MRMDestReg, (outs), (ins GR16:$src1, GR16:$src2), "btc{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, - OpSize, TB; + OpSize16, TB; def BTC32rr : I<0xBB, MRMDestReg, (outs), (ins GR32:$src1, GR32:$src2), - "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, TB; + "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, + OpSize32, TB; def BTC64rr : RI<0xBB, MRMDestReg, (outs), (ins GR64:$src1, GR64:$src2), "btc{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, TB; } // SchedRW @@ -1343,9 +1513,10 @@ def BTC64rr : RI<0xBB, MRMDestReg, (outs), (ins GR64:$src1, GR64:$src2), let mayLoad = 1, mayStore = 1, SchedRW = [WriteALULd, WriteRMW] in { def BTC16mr : I<0xBB, MRMDestMem, (outs), (ins i16mem:$src1, GR16:$src2), "btc{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, - OpSize, TB; + OpSize16, TB; def BTC32mr : I<0xBB, MRMDestMem, (outs), (ins i32mem:$src1, GR32:$src2), - "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, TB; + "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, + OpSize32, TB; def BTC64mr : RI<0xBB, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), "btc{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, TB; } @@ -1353,9 +1524,10 @@ def BTC64mr : RI<0xBB, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), let SchedRW = [WriteALU] in { def BTC16ri8 : Ii8<0xBA, MRM7r, (outs), (ins GR16:$src1, i16i8imm:$src2), "btc{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, - OpSize, TB; + OpSize16, TB; def BTC32ri8 : Ii8<0xBA, MRM7r, (outs), (ins GR32:$src1, i32i8imm:$src2), - "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, TB; + "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, + OpSize32, TB; def BTC64ri8 : RIi8<0xBA, MRM7r, (outs), (ins GR64:$src1, i64i8imm:$src2), "btc{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, TB; } // SchedRW @@ -1363,9 +1535,10 @@ def BTC64ri8 : RIi8<0xBA, MRM7r, (outs), (ins GR64:$src1, i64i8imm:$src2), let mayLoad = 1, mayStore = 1, SchedRW = [WriteALULd, WriteRMW] in { def BTC16mi8 : Ii8<0xBA, MRM7m, (outs), (ins i16mem:$src1, i16i8imm:$src2), "btc{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, - OpSize, TB; + OpSize16, TB; def BTC32mi8 : Ii8<0xBA, MRM7m, (outs), (ins i32mem:$src1, i32i8imm:$src2), - "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, TB; + "btc{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, + OpSize32, TB; def BTC64mi8 : RIi8<0xBA, MRM7m, (outs), (ins i64mem:$src1, i64i8imm:$src2), "btc{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, TB; } @@ -1373,9 +1546,10 @@ def BTC64mi8 : RIi8<0xBA, MRM7m, (outs), (ins i64mem:$src1, i64i8imm:$src2), let SchedRW = [WriteALU] in { def BTR16rr : I<0xB3, MRMDestReg, (outs), (ins GR16:$src1, GR16:$src2), "btr{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, - OpSize, TB; + OpSize16, TB; def BTR32rr : I<0xB3, MRMDestReg, (outs), (ins GR32:$src1, GR32:$src2), - "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, TB; + "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, + OpSize32, TB; def BTR64rr : RI<0xB3, MRMDestReg, (outs), (ins GR64:$src1, GR64:$src2), "btr{q}\t{$src2, $src1|$src1, $src2}", []>, TB; } // SchedRW @@ -1383,9 +1557,10 @@ def BTR64rr : RI<0xB3, MRMDestReg, (outs), (ins GR64:$src1, GR64:$src2), let mayLoad = 1, mayStore = 1, SchedRW = [WriteALULd, WriteRMW] in { def BTR16mr : I<0xB3, MRMDestMem, (outs), (ins i16mem:$src1, GR16:$src2), "btr{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, - OpSize, TB; + OpSize16, TB; def BTR32mr : I<0xB3, MRMDestMem, (outs), (ins i32mem:$src1, GR32:$src2), - "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, TB; + "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, + OpSize32, TB; def BTR64mr : RI<0xB3, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), "btr{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, TB; } @@ -1393,9 +1568,10 @@ def BTR64mr : RI<0xB3, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), let SchedRW = [WriteALU] in { def BTR16ri8 : Ii8<0xBA, MRM6r, (outs), (ins GR16:$src1, i16i8imm:$src2), "btr{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, - OpSize, TB; + OpSize16, TB; def BTR32ri8 : Ii8<0xBA, MRM6r, (outs), (ins GR32:$src1, i32i8imm:$src2), - "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, TB; + "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, + OpSize32, TB; def BTR64ri8 : RIi8<0xBA, MRM6r, (outs), (ins GR64:$src1, i64i8imm:$src2), "btr{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, TB; } // SchedRW @@ -1403,9 +1579,10 @@ def BTR64ri8 : RIi8<0xBA, MRM6r, (outs), (ins GR64:$src1, i64i8imm:$src2), let mayLoad = 1, mayStore = 1, SchedRW = [WriteALULd, WriteRMW] in { def BTR16mi8 : Ii8<0xBA, MRM6m, (outs), (ins i16mem:$src1, i16i8imm:$src2), "btr{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, - OpSize, TB; + OpSize16, TB; def BTR32mi8 : Ii8<0xBA, MRM6m, (outs), (ins i32mem:$src1, i32i8imm:$src2), - "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, TB; + "btr{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, + OpSize32, TB; def BTR64mi8 : RIi8<0xBA, MRM6m, (outs), (ins i64mem:$src1, i64i8imm:$src2), "btr{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, TB; } @@ -1413,19 +1590,21 @@ def BTR64mi8 : RIi8<0xBA, MRM6m, (outs), (ins i64mem:$src1, i64i8imm:$src2), let SchedRW = [WriteALU] in { def BTS16rr : I<0xAB, MRMDestReg, (outs), (ins GR16:$src1, GR16:$src2), "bts{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, - OpSize, TB; + OpSize16, TB; def BTS32rr : I<0xAB, MRMDestReg, (outs), (ins GR32:$src1, GR32:$src2), - "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, TB; + "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, + OpSize32, TB; def BTS64rr : RI<0xAB, MRMDestReg, (outs), (ins GR64:$src1, GR64:$src2), - "bts{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, TB; + "bts{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RR>, TB; } // SchedRW let mayLoad = 1, mayStore = 1, SchedRW = [WriteALULd, WriteRMW] in { def BTS16mr : I<0xAB, MRMDestMem, (outs), (ins i16mem:$src1, GR16:$src2), - "bts{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, - OpSize, TB; + "bts{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, + OpSize16, TB; def BTS32mr : I<0xAB, MRMDestMem, (outs), (ins i32mem:$src1, GR32:$src2), - "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, TB; + "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, + OpSize32, TB; def BTS64mr : RI<0xAB, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), "bts{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MR>, TB; } @@ -1433,9 +1612,10 @@ def BTS64mr : RI<0xAB, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), let SchedRW = [WriteALU] in { def BTS16ri8 : Ii8<0xBA, MRM5r, (outs), (ins GR16:$src1, i16i8imm:$src2), "bts{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, - OpSize, TB; + OpSize16, TB; def BTS32ri8 : Ii8<0xBA, MRM5r, (outs), (ins GR32:$src1, i32i8imm:$src2), - "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, TB; + "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, + OpSize32, TB; def BTS64ri8 : RIi8<0xBA, MRM5r, (outs), (ins GR64:$src1, i64i8imm:$src2), "bts{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_RI>, TB; } // SchedRW @@ -1443,9 +1623,10 @@ def BTS64ri8 : RIi8<0xBA, MRM5r, (outs), (ins GR64:$src1, i64i8imm:$src2), let mayLoad = 1, mayStore = 1, SchedRW = [WriteALULd, WriteRMW] in { def BTS16mi8 : Ii8<0xBA, MRM5m, (outs), (ins i16mem:$src1, i16i8imm:$src2), "bts{w}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, - OpSize, TB; + OpSize16, TB; def BTS32mi8 : Ii8<0xBA, MRM5m, (outs), (ins i32mem:$src1, i32i8imm:$src2), - "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, TB; + "bts{l}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, + OpSize32, TB; def BTS64mi8 : RIi8<0xBA, MRM5m, (outs), (ins i64mem:$src1, i64i8imm:$src2), "bts{q}\t{$src2, $src1|$src1, $src2}", [], IIC_BTX_MI>, TB; } @@ -1475,14 +1656,14 @@ multiclass ATOMIC_SWAP<bits<8> opc8, bits<8> opc, string mnemonic, string frag, [(set GR16:$dst, (!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val))], - itin>, OpSize; + itin>, OpSize16; def NAME#32rm : I<opc, MRMSrcMem, (outs GR32:$dst), (ins GR32:$val, i32mem:$ptr), !strconcat(mnemonic, "{l}\t{$val, $ptr|$ptr, $val}"), [(set GR32:$dst, (!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val))], - itin>; + itin>, OpSize32; def NAME#64rm : RI<opc, MRMSrcMem, (outs GR64:$dst), (ins GR64:$val, i64mem:$ptr), !strconcat(mnemonic, "{q}\t{$val, $ptr|$ptr, $val}"), @@ -1501,24 +1682,30 @@ let Constraints = "$val = $dst" in { def XCHG8rr : I<0x86, MRMSrcReg, (outs GR8:$dst), (ins GR8:$val, GR8:$src), "xchg{b}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>; def XCHG16rr : I<0x87, MRMSrcReg, (outs GR16:$dst), (ins GR16:$val, GR16:$src), - "xchg{w}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>, OpSize; + "xchg{w}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>, + OpSize16; def XCHG32rr : I<0x87, MRMSrcReg, (outs GR32:$dst), (ins GR32:$val, GR32:$src), - "xchg{l}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>; + "xchg{l}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>, + OpSize32; def XCHG64rr : RI<0x87, MRMSrcReg, (outs GR64:$dst), (ins GR64:$val,GR64:$src), "xchg{q}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>; } // Swap between EAX and other registers. +let Uses = [AX], Defs = [AX] in def XCHG16ar : I<0x90, AddRegFrm, (outs), (ins GR16:$src), - "xchg{w}\t{$src, %ax|ax, $src}", [], IIC_XCHG_REG>, OpSize; + "xchg{w}\t{$src, %ax|ax, $src}", [], IIC_XCHG_REG>, OpSize16; +let Uses = [EAX], Defs = [EAX] in def XCHG32ar : I<0x90, AddRegFrm, (outs), (ins GR32:$src), "xchg{l}\t{$src, %eax|eax, $src}", [], IIC_XCHG_REG>, - Requires<[In32BitMode]>; + OpSize32, Requires<[Not64BitMode]>; +let Uses = [EAX], Defs = [EAX] in // Uses GR32_NOAX in 64-bit mode to prevent encoding using the 0x90 NOP encoding. // xchg %eax, %eax needs to clear upper 32-bits of RAX so is not a NOP. def XCHG32ar64 : I<0x90, AddRegFrm, (outs), (ins GR32_NOAX:$src), "xchg{l}\t{$src, %eax|eax, $src}", [], IIC_XCHG_REG>, - Requires<[In64BitMode]>; + OpSize32, Requires<[In64BitMode]>; +let Uses = [RAX], Defs = [RAX] in def XCHG64ar : RI<0x90, AddRegFrm, (outs), (ins GR64:$src), "xchg{q}\t{$src, %rax|rax, $src}", [], IIC_XCHG_REG>; } // SchedRW @@ -1528,9 +1715,10 @@ def XADD8rr : I<0xC0, MRMDestReg, (outs GR8:$dst), (ins GR8:$src), "xadd{b}\t{$src, $dst|$dst, $src}", [], IIC_XADD_REG>, TB; def XADD16rr : I<0xC1, MRMDestReg, (outs GR16:$dst), (ins GR16:$src), "xadd{w}\t{$src, $dst|$dst, $src}", [], IIC_XADD_REG>, TB, - OpSize; + OpSize16; def XADD32rr : I<0xC1, MRMDestReg, (outs GR32:$dst), (ins GR32:$src), - "xadd{l}\t{$src, $dst|$dst, $src}", [], IIC_XADD_REG>, TB; + "xadd{l}\t{$src, $dst|$dst, $src}", [], IIC_XADD_REG>, TB, + OpSize32; def XADD64rr : RI<0xC1, MRMDestReg, (outs GR64:$dst), (ins GR64:$src), "xadd{q}\t{$src, $dst|$dst, $src}", [], IIC_XADD_REG>, TB; } // SchedRW @@ -1540,9 +1728,10 @@ def XADD8rm : I<0xC0, MRMDestMem, (outs), (ins i8mem:$dst, GR8:$src), "xadd{b}\t{$src, $dst|$dst, $src}", [], IIC_XADD_MEM>, TB; def XADD16rm : I<0xC1, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src), "xadd{w}\t{$src, $dst|$dst, $src}", [], IIC_XADD_MEM>, TB, - OpSize; + OpSize16; def XADD32rm : I<0xC1, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), - "xadd{l}\t{$src, $dst|$dst, $src}", [], IIC_XADD_MEM>, TB; + "xadd{l}\t{$src, $dst|$dst, $src}", [], IIC_XADD_MEM>, TB, + OpSize32; def XADD64rm : RI<0xC1, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), "xadd{q}\t{$src, $dst|$dst, $src}", [], IIC_XADD_MEM>, TB; @@ -1554,10 +1743,10 @@ def CMPXCHG8rr : I<0xB0, MRMDestReg, (outs GR8:$dst), (ins GR8:$src), IIC_CMPXCHG_REG8>, TB; def CMPXCHG16rr : I<0xB1, MRMDestReg, (outs GR16:$dst), (ins GR16:$src), "cmpxchg{w}\t{$src, $dst|$dst, $src}", [], - IIC_CMPXCHG_REG>, TB, OpSize; + IIC_CMPXCHG_REG>, TB, OpSize16; def CMPXCHG32rr : I<0xB1, MRMDestReg, (outs GR32:$dst), (ins GR32:$src), "cmpxchg{l}\t{$src, $dst|$dst, $src}", [], - IIC_CMPXCHG_REG>, TB; + IIC_CMPXCHG_REG>, TB, OpSize32; def CMPXCHG64rr : RI<0xB1, MRMDestReg, (outs GR64:$dst), (ins GR64:$src), "cmpxchg{q}\t{$src, $dst|$dst, $src}", [], IIC_CMPXCHG_REG>, TB; @@ -1570,10 +1759,10 @@ def CMPXCHG8rm : I<0xB0, MRMDestMem, (outs), (ins i8mem:$dst, GR8:$src), IIC_CMPXCHG_MEM8>, TB; def CMPXCHG16rm : I<0xB1, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src), "cmpxchg{w}\t{$src, $dst|$dst, $src}", [], - IIC_CMPXCHG_MEM>, TB, OpSize; + IIC_CMPXCHG_MEM>, TB, OpSize16; def CMPXCHG32rm : I<0xB1, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), "cmpxchg{l}\t{$src, $dst|$dst, $src}", [], - IIC_CMPXCHG_MEM>, TB; + IIC_CMPXCHG_MEM>, TB, OpSize32; def CMPXCHG64rm : RI<0xB1, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), "cmpxchg{q}\t{$src, $dst|$dst, $src}", [], IIC_CMPXCHG_MEM>, TB; @@ -1594,7 +1783,8 @@ def CMPXCHG16B : RI<0xC7, MRM1m, (outs), (ins i128mem:$dst), def LOCK_PREFIX : I<0xF0, RawFrm, (outs), (ins), "lock", []>; // Rex64 instruction prefix -def REX64_PREFIX : I<0x48, RawFrm, (outs), (ins), "rex64", []>; +def REX64_PREFIX : I<0x48, RawFrm, (outs), (ins), "rex64", []>, + Requires<[In64BitMode]>; // Data16 instruction prefix def DATA16_PREFIX : I<0x66, RawFrm, (outs), (ins), "data16", []>; @@ -1611,16 +1801,41 @@ def REPNE_PREFIX : I<0xF2, RawFrm, (outs), (ins), "repne", []>; // String manipulation instructions let SchedRW = [WriteMicrocoded] in { -def LODSB : I<0xAC, RawFrm, (outs), (ins), "lodsb", [], IIC_LODS>; -def LODSW : I<0xAD, RawFrm, (outs), (ins), "lodsw", [], IIC_LODS>, OpSize; -def LODSD : I<0xAD, RawFrm, (outs), (ins), "lods{l|d}", [], IIC_LODS>; -def LODSQ : RI<0xAD, RawFrm, (outs), (ins), "lodsq", [], IIC_LODS>; +// These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI +let Defs = [AL,ESI], Uses = [ESI,EFLAGS] in +def LODSB : I<0xAC, RawFrmSrc, (outs), (ins srcidx8:$src), + "lodsb\t{$src, %al|al, $src}", [], IIC_LODS>; +let Defs = [AX,ESI], Uses = [ESI,EFLAGS] in +def LODSW : I<0xAD, RawFrmSrc, (outs), (ins srcidx16:$src), + "lodsw\t{$src, %ax|ax, $src}", [], IIC_LODS>, OpSize16; +let Defs = [EAX,ESI], Uses = [ESI,EFLAGS] in +def LODSL : I<0xAD, RawFrmSrc, (outs), (ins srcidx32:$src), + "lods{l|d}\t{$src, %eax|eax, $src}", [], IIC_LODS>, OpSize32; +let Defs = [RAX,ESI], Uses = [ESI,EFLAGS] in +def LODSQ : RI<0xAD, RawFrmSrc, (outs), (ins srcidx64:$src), + "lodsq\t{$src, %rax|rax, $src}", [], IIC_LODS>; } let SchedRW = [WriteSystem] in { -def OUTSB : I<0x6E, RawFrm, (outs), (ins), "outsb", [], IIC_OUTS>; -def OUTSW : I<0x6F, RawFrm, (outs), (ins), "outsw", [], IIC_OUTS>, OpSize; -def OUTSD : I<0x6F, RawFrm, (outs), (ins), "outs{l|d}", [], IIC_OUTS>; +// These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI +let Defs = [ESI], Uses = [DX,ESI,EFLAGS] in { +def OUTSB : I<0x6E, RawFrmSrc, (outs), (ins srcidx8:$src), + "outsb\t{$src, %dx|dx, $src}", [], IIC_OUTS>; +def OUTSW : I<0x6F, RawFrmSrc, (outs), (ins srcidx16:$src), + "outsw\t{$src, %dx|dx, $src}", [], IIC_OUTS>, OpSize16; +def OUTSL : I<0x6F, RawFrmSrc, (outs), (ins srcidx32:$src), + "outs{l|d}\t{$src, %dx|dx, $src}", [], IIC_OUTS>, OpSize32; +} + +// These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI +let Defs = [EDI], Uses = [DX,EDI,EFLAGS] in { +def INSB : I<0x6C, RawFrmDst, (outs dstidx8:$dst), (ins), + "insb\t{%dx, $dst|$dst, dx}", [], IIC_INS>; +def INSW : I<0x6D, RawFrmDst, (outs dstidx16:$dst), (ins), + "insw\t{%dx, $dst|$dst, dx}", [], IIC_INS>, OpSize16; +def INSL : I<0x6D, RawFrmDst, (outs dstidx32:$dst), (ins), + "ins{l|d}\t{%dx, $dst|$dst, dx}", [], IIC_INS>, OpSize32; +} } // Flag instructions @@ -1644,50 +1859,50 @@ let SchedRW = [WriteMicrocoded] in { // ASCII Adjust After Addition // sets AL, AH and CF and AF of EFLAGS and uses AL and AF of EFLAGS def AAA : I<0x37, RawFrm, (outs), (ins), "aaa", [], IIC_AAA>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // ASCII Adjust AX Before Division // sets AL, AH and EFLAGS and uses AL and AH def AAD8i8 : Ii8<0xD5, RawFrm, (outs), (ins i8imm:$src), - "aad\t$src", [], IIC_AAD>, Requires<[In32BitMode]>; + "aad\t$src", [], IIC_AAD>, Requires<[Not64BitMode]>; // ASCII Adjust AX After Multiply // sets AL, AH and EFLAGS and uses AL def AAM8i8 : Ii8<0xD4, RawFrm, (outs), (ins i8imm:$src), - "aam\t$src", [], IIC_AAM>, Requires<[In32BitMode]>; + "aam\t$src", [], IIC_AAM>, Requires<[Not64BitMode]>; // ASCII Adjust AL After Subtraction - sets // sets AL, AH and CF and AF of EFLAGS and uses AL and AF of EFLAGS def AAS : I<0x3F, RawFrm, (outs), (ins), "aas", [], IIC_AAS>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // Decimal Adjust AL after Addition // sets AL, CF and AF of EFLAGS and uses AL, CF and AF of EFLAGS def DAA : I<0x27, RawFrm, (outs), (ins), "daa", [], IIC_DAA>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; // Decimal Adjust AL after Subtraction // sets AL, CF and AF of EFLAGS and uses AL, CF and AF of EFLAGS def DAS : I<0x2F, RawFrm, (outs), (ins), "das", [], IIC_DAS>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; } // SchedRW let SchedRW = [WriteSystem] in { // Check Array Index Against Bounds def BOUNDS16rm : I<0x62, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), - "bound\t{$src, $dst|$dst, $src}", [], IIC_BOUND>, OpSize, - Requires<[In32BitMode]>; + "bound\t{$src, $dst|$dst, $src}", [], IIC_BOUND>, OpSize16, + Requires<[Not64BitMode]>; def BOUNDS32rm : I<0x62, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), - "bound\t{$src, $dst|$dst, $src}", [], IIC_BOUND>, - Requires<[In32BitMode]>; + "bound\t{$src, $dst|$dst, $src}", [], IIC_BOUND>, OpSize32, + Requires<[Not64BitMode]>; // Adjust RPL Field of Segment Selector def ARPL16rr : I<0x63, MRMDestReg, (outs GR16:$dst), (ins GR16:$src), "arpl\t{$src, $dst|$dst, $src}", [], IIC_ARPL_REG>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def ARPL16mr : I<0x63, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src), "arpl\t{$src, $dst|$dst, $src}", [], IIC_ARPL_MEM>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; } // SchedRW //===----------------------------------------------------------------------===// @@ -1698,29 +1913,29 @@ let Predicates = [HasMOVBE] in { def MOVBE16rm : I<0xF0, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "movbe{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (bswap (loadi16 addr:$src)))], IIC_MOVBE>, - OpSize, T8; + OpSize16, T8PS; def MOVBE32rm : I<0xF0, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "movbe{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (bswap (loadi32 addr:$src)))], IIC_MOVBE>, - T8; + OpSize32, T8PS; def MOVBE64rm : RI<0xF0, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "movbe{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (bswap (loadi64 addr:$src)))], IIC_MOVBE>, - T8; + T8PS; } let SchedRW = [WriteStore] in { def MOVBE16mr : I<0xF1, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src), "movbe{w}\t{$src, $dst|$dst, $src}", [(store (bswap GR16:$src), addr:$dst)], IIC_MOVBE>, - OpSize, T8; + OpSize16, T8PS; def MOVBE32mr : I<0xF1, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), "movbe{l}\t{$src, $dst|$dst, $src}", [(store (bswap GR32:$src), addr:$dst)], IIC_MOVBE>, - T8; + OpSize32, T8PS; def MOVBE64mr : RI<0xF1, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), "movbe{q}\t{$src, $dst|$dst, $src}", [(store (bswap GR64:$src), addr:$dst)], IIC_MOVBE>, - T8; + T8PS; } } @@ -1730,10 +1945,10 @@ let Predicates = [HasMOVBE] in { let Predicates = [HasRDRAND], Defs = [EFLAGS] in { def RDRAND16r : I<0xC7, MRM6r, (outs GR16:$dst), (ins), "rdrand{w}\t$dst", - [(set GR16:$dst, EFLAGS, (X86rdrand))]>, OpSize, TB; + [(set GR16:$dst, EFLAGS, (X86rdrand))]>, OpSize16, TB; def RDRAND32r : I<0xC7, MRM6r, (outs GR32:$dst), (ins), "rdrand{l}\t$dst", - [(set GR32:$dst, EFLAGS, (X86rdrand))]>, TB; + [(set GR32:$dst, EFLAGS, (X86rdrand))]>, OpSize32, TB; def RDRAND64r : RI<0xC7, MRM6r, (outs GR64:$dst), (ins), "rdrand{q}\t$dst", [(set GR64:$dst, EFLAGS, (X86rdrand))]>, TB; @@ -1745,10 +1960,10 @@ let Predicates = [HasRDRAND], Defs = [EFLAGS] in { let Predicates = [HasRDSEED], Defs = [EFLAGS] in { def RDSEED16r : I<0xC7, MRM7r, (outs GR16:$dst), (ins), "rdseed{w}\t$dst", - [(set GR16:$dst, EFLAGS, (X86rdseed))]>, OpSize, TB; + [(set GR16:$dst, EFLAGS, (X86rdseed))]>, OpSize16, TB; def RDSEED32r : I<0xC7, MRM7r, (outs GR32:$dst), (ins), "rdseed{l}\t$dst", - [(set GR32:$dst, EFLAGS, (X86rdseed))]>, TB; + [(set GR32:$dst, EFLAGS, (X86rdseed))]>, OpSize32, TB; def RDSEED64r : RI<0xC7, MRM7r, (outs GR64:$dst), (ins), "rdseed{q}\t$dst", [(set GR64:$dst, EFLAGS, (X86rdseed))]>, TB; @@ -1761,19 +1976,20 @@ let Predicates = [HasLZCNT], Defs = [EFLAGS] in { def LZCNT16rr : I<0xBD, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), "lzcnt{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (ctlz GR16:$src)), (implicit EFLAGS)]>, XS, - OpSize; + OpSize16; def LZCNT16rm : I<0xBD, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "lzcnt{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (ctlz (loadi16 addr:$src))), - (implicit EFLAGS)]>, XS, OpSize; + (implicit EFLAGS)]>, XS, OpSize16; def LZCNT32rr : I<0xBD, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), "lzcnt{l}\t{$src, $dst|$dst, $src}", - [(set GR32:$dst, (ctlz GR32:$src)), (implicit EFLAGS)]>, XS; + [(set GR32:$dst, (ctlz GR32:$src)), (implicit EFLAGS)]>, XS, + OpSize32; def LZCNT32rm : I<0xBD, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "lzcnt{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (ctlz (loadi32 addr:$src))), - (implicit EFLAGS)]>, XS; + (implicit EFLAGS)]>, XS, OpSize32; def LZCNT64rr : RI<0xBD, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "lzcnt{q}\t{$src, $dst|$dst, $src}", @@ -1792,19 +2008,20 @@ let Predicates = [HasBMI], Defs = [EFLAGS] in { def TZCNT16rr : I<0xBC, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), "tzcnt{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (cttz GR16:$src)), (implicit EFLAGS)]>, XS, - OpSize; + OpSize16; def TZCNT16rm : I<0xBC, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "tzcnt{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (cttz (loadi16 addr:$src))), - (implicit EFLAGS)]>, XS, OpSize; + (implicit EFLAGS)]>, XS, OpSize16; def TZCNT32rr : I<0xBC, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), "tzcnt{l}\t{$src, $dst|$dst, $src}", - [(set GR32:$dst, (cttz GR32:$src)), (implicit EFLAGS)]>, XS; + [(set GR32:$dst, (cttz GR32:$src)), (implicit EFLAGS)]>, XS, + OpSize32; def TZCNT32rm : I<0xBC, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "tzcnt{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (cttz (loadi32 addr:$src))), - (implicit EFLAGS)]>, XS; + (implicit EFLAGS)]>, XS, OpSize32; def TZCNT64rr : RI<0xBC, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "tzcnt{q}\t{$src, $dst|$dst, $src}", @@ -1817,30 +2034,47 @@ let Predicates = [HasBMI], Defs = [EFLAGS] in { } multiclass bmi_bls<string mnemonic, Format RegMRM, Format MemMRM, - RegisterClass RC, X86MemOperand x86memop, SDNode OpNode, - PatFrag ld_frag> { + RegisterClass RC, X86MemOperand x86memop> { +let hasSideEffects = 0 in { def rr : I<0xF3, RegMRM, (outs RC:$dst), (ins RC:$src), !strconcat(mnemonic, "\t{$src, $dst|$dst, $src}"), - [(set RC:$dst, (OpNode RC:$src)), (implicit EFLAGS)]>, T8, VEX_4V; + []>, T8PS, VEX_4V; + let mayLoad = 1 in def rm : I<0xF3, MemMRM, (outs RC:$dst), (ins x86memop:$src), !strconcat(mnemonic, "\t{$src, $dst|$dst, $src}"), - [(set RC:$dst, (OpNode (ld_frag addr:$src))), (implicit EFLAGS)]>, - T8, VEX_4V; + []>, T8PS, VEX_4V; +} } let Predicates = [HasBMI], Defs = [EFLAGS] in { - defm BLSR32 : bmi_bls<"blsr{l}", MRM1r, MRM1m, GR32, i32mem, - X86blsr, loadi32>; - defm BLSR64 : bmi_bls<"blsr{q}", MRM1r, MRM1m, GR64, i64mem, - X86blsr, loadi64>, VEX_W; - defm BLSMSK32 : bmi_bls<"blsmsk{l}", MRM2r, MRM2m, GR32, i32mem, - X86blsmsk, loadi32>; - defm BLSMSK64 : bmi_bls<"blsmsk{q}", MRM2r, MRM2m, GR64, i64mem, - X86blsmsk, loadi64>, VEX_W; - defm BLSI32 : bmi_bls<"blsi{l}", MRM3r, MRM3m, GR32, i32mem, - X86blsi, loadi32>; - defm BLSI64 : bmi_bls<"blsi{q}", MRM3r, MRM3m, GR64, i64mem, - X86blsi, loadi64>, VEX_W; + defm BLSR32 : bmi_bls<"blsr{l}", MRM1r, MRM1m, GR32, i32mem>; + defm BLSR64 : bmi_bls<"blsr{q}", MRM1r, MRM1m, GR64, i64mem>, VEX_W; + defm BLSMSK32 : bmi_bls<"blsmsk{l}", MRM2r, MRM2m, GR32, i32mem>; + defm BLSMSK64 : bmi_bls<"blsmsk{q}", MRM2r, MRM2m, GR64, i64mem>, VEX_W; + defm BLSI32 : bmi_bls<"blsi{l}", MRM3r, MRM3m, GR32, i32mem>; + defm BLSI64 : bmi_bls<"blsi{q}", MRM3r, MRM3m, GR64, i64mem>, VEX_W; +} + +//===----------------------------------------------------------------------===// +// Pattern fragments to auto generate BMI instructions. +//===----------------------------------------------------------------------===// + +let Predicates = [HasBMI] in { + // FIXME: patterns for the load versions are not implemented + def : Pat<(and GR32:$src, (add GR32:$src, -1)), + (BLSR32rr GR32:$src)>; + def : Pat<(and GR64:$src, (add GR64:$src, -1)), + (BLSR64rr GR64:$src)>; + + def : Pat<(xor GR32:$src, (add GR32:$src, -1)), + (BLSMSK32rr GR32:$src)>; + def : Pat<(xor GR64:$src, (add GR64:$src, -1)), + (BLSMSK64rr GR64:$src)>; + + def : Pat<(and GR32:$src, (ineg GR32:$src)), + (BLSI32rr GR32:$src)>; + def : Pat<(and GR64:$src, (ineg GR64:$src)), + (BLSI64rr GR64:$src)>; } multiclass bmi_bextr_bzhi<bits<8> opc, string mnemonic, RegisterClass RC, @@ -1849,11 +2083,11 @@ multiclass bmi_bextr_bzhi<bits<8> opc, string mnemonic, RegisterClass RC, def rr : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (Int RC:$src1, RC:$src2)), (implicit EFLAGS)]>, - T8, VEX_4VOp3; + T8PS, VEX_4VOp3; def rm : I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1, RC:$src2), !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (Int (ld_frag addr:$src1), RC:$src2)), - (implicit EFLAGS)]>, T8, VEX_4VOp3; + (implicit EFLAGS)]>, T8PS, VEX_4VOp3; } let Predicates = [HasBMI], Defs = [EFLAGS] in { @@ -1930,17 +2164,18 @@ multiclass tbm_ternary_imm_intr<bits<8> opc, RegisterClass RC, string OpcodeStr, !strconcat(OpcodeStr, "\t{$cntl, $src1, $dst|$dst, $src1, $cntl}"), [(set RC:$dst, (Int RC:$src1, immoperator:$cntl))]>, - XOP, XOPA, VEX; + XOP, XOPA; def mi : Ii32<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1, immtype:$cntl), !strconcat(OpcodeStr, "\t{$cntl, $src1, $dst|$dst, $src1, $cntl}"), [(set RC:$dst, (Int (ld_frag addr:$src1), immoperator:$cntl))]>, - XOP, XOPA, VEX; + XOP, XOPA; } defm BEXTRI32 : tbm_ternary_imm_intr<0x10, GR32, "bextr", i32mem, loadi32, int_x86_tbm_bextri_u32, i32imm, imm>; +let ImmT = Imm32S in defm BEXTRI64 : tbm_ternary_imm_intr<0x10, GR64, "bextr", i64mem, loadi64, int_x86_tbm_bextri_u64, i64i32imm, i64immSExt32>, VEX_W; @@ -1951,11 +2186,11 @@ multiclass tbm_binary_rm<bits<8> opc, Format FormReg, Format FormMem, let hasSideEffects = 0 in { def rr : I<opc, FormReg, (outs RC:$dst), (ins RC:$src), !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"), - []>, XOP, XOP9, VEX_4V; + []>, XOP_4V, XOP9; let mayLoad = 1 in def rm : I<opc, FormMem, (outs RC:$dst), (ins x86memop:$src), !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"), - []>, XOP, XOP9, VEX_4V; + []>, XOP_4V, XOP9; } } @@ -2088,6 +2323,7 @@ include "X86InstrCompiler.td" // Assembler Mnemonic Aliases //===----------------------------------------------------------------------===// +def : MnemonicAlias<"call", "callw", "att">, Requires<[In16BitMode]>; def : MnemonicAlias<"call", "calll", "att">, Requires<[In32BitMode]>; def : MnemonicAlias<"call", "callq", "att">, Requires<[In64BitMode]>; @@ -2098,17 +2334,20 @@ def : MnemonicAlias<"cdq", "cltd", "att">; def : MnemonicAlias<"cdqe", "cltq", "att">; def : MnemonicAlias<"cqo", "cqto", "att">; -// lret maps to lretl, it is not ambiguous with lretq. -def : MnemonicAlias<"lret", "lretl", "att">; +// In 64-bit mode lret maps to lretl; it is not ambiguous with lretq. +def : MnemonicAlias<"lret", "lretw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"lret", "lretl", "att">, Requires<[Not16BitMode]>; -def : MnemonicAlias<"leavel", "leave", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"leavel", "leave", "att">, Requires<[Not64BitMode]>; def : MnemonicAlias<"leaveq", "leave", "att">, Requires<[In64BitMode]>; def : MnemonicAlias<"loopz", "loope", "att">; def : MnemonicAlias<"loopnz", "loopne", "att">; +def : MnemonicAlias<"pop", "popw", "att">, Requires<[In16BitMode]>; def : MnemonicAlias<"pop", "popl", "att">, Requires<[In32BitMode]>; def : MnemonicAlias<"pop", "popq", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"popf", "popfw", "att">, Requires<[In16BitMode]>; def : MnemonicAlias<"popf", "popfl", "att">, Requires<[In32BitMode]>; def : MnemonicAlias<"popf", "popfq", "att">, Requires<[In64BitMode]>; def : MnemonicAlias<"popfd", "popfl", "att">; @@ -2116,21 +2355,33 @@ def : MnemonicAlias<"popfd", "popfl", "att">; // FIXME: This is wrong for "push reg". "push %bx" should turn into pushw in // all modes. However: "push (addr)" and "push $42" should default to // pushl/pushq depending on the current mode. Similar for "pop %bx" +def : MnemonicAlias<"push", "pushw", "att">, Requires<[In16BitMode]>; def : MnemonicAlias<"push", "pushl", "att">, Requires<[In32BitMode]>; def : MnemonicAlias<"push", "pushq", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"pushf", "pushfw", "att">, Requires<[In16BitMode]>; def : MnemonicAlias<"pushf", "pushfl", "att">, Requires<[In32BitMode]>; def : MnemonicAlias<"pushf", "pushfq", "att">, Requires<[In64BitMode]>; def : MnemonicAlias<"pushfd", "pushfl", "att">; -def : MnemonicAlias<"popad", "popa", "intel">, Requires<[In32BitMode]>; -def : MnemonicAlias<"pushad", "pusha", "intel">, Requires<[In32BitMode]>; +def : MnemonicAlias<"popad", "popal", "intel">, Requires<[Not64BitMode]>; +def : MnemonicAlias<"pushad", "pushal", "intel">, Requires<[Not64BitMode]>; +def : MnemonicAlias<"popa", "popaw", "intel">, Requires<[In16BitMode]>; +def : MnemonicAlias<"pusha", "pushaw", "intel">, Requires<[In16BitMode]>; +def : MnemonicAlias<"popa", "popal", "intel">, Requires<[In32BitMode]>; +def : MnemonicAlias<"pusha", "pushal", "intel">, Requires<[In32BitMode]>; + +def : MnemonicAlias<"popa", "popaw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"pusha", "pushaw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"popa", "popal", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"pusha", "pushal", "att">, Requires<[In32BitMode]>; def : MnemonicAlias<"repe", "rep", "att">; def : MnemonicAlias<"repz", "rep", "att">; def : MnemonicAlias<"repnz", "repne", "att">; -def : MnemonicAlias<"retl", "ret", "att">, Requires<[In32BitMode]>; -def : MnemonicAlias<"retq", "ret", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"ret", "retw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"ret", "retl", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"ret", "retq", "att">, Requires<[In64BitMode]>; def : MnemonicAlias<"salb", "shlb", "att">; def : MnemonicAlias<"salw", "shlw", "att">; @@ -2146,18 +2397,23 @@ def : MnemonicAlias<"ud2a", "ud2", "att">; def : MnemonicAlias<"verrw", "verr", "att">; // System instruction aliases. -def : MnemonicAlias<"iret", "iretl", "att">; +def : MnemonicAlias<"iret", "iretw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"iret", "iretl", "att">, Requires<[Not16BitMode]>; def : MnemonicAlias<"sysret", "sysretl", "att">; def : MnemonicAlias<"sysexit", "sysexitl", "att">; -def : MnemonicAlias<"lgdtl", "lgdt", "att">, Requires<[In32BitMode]>; -def : MnemonicAlias<"lgdtq", "lgdt", "att">, Requires<[In64BitMode]>; -def : MnemonicAlias<"lidtl", "lidt", "att">, Requires<[In32BitMode]>; -def : MnemonicAlias<"lidtq", "lidt", "att">, Requires<[In64BitMode]>; -def : MnemonicAlias<"sgdtl", "sgdt", "att">, Requires<[In32BitMode]>; -def : MnemonicAlias<"sgdtq", "sgdt", "att">, Requires<[In64BitMode]>; -def : MnemonicAlias<"sidtl", "sidt", "att">, Requires<[In32BitMode]>; -def : MnemonicAlias<"sidtq", "sidt", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"lgdt", "lgdtw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"lgdt", "lgdtl", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"lgdt", "lgdtq", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"lidt", "lidtw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"lidt", "lidtl", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"lidt", "lidtq", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"sgdt", "sgdtw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"sgdt", "sgdtl", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"sgdt", "sgdtq", "att">, Requires<[In64BitMode]>; +def : MnemonicAlias<"sidt", "sidtw", "att">, Requires<[In16BitMode]>; +def : MnemonicAlias<"sidt", "sidtl", "att">, Requires<[In32BitMode]>; +def : MnemonicAlias<"sidt", "sidtq", "att">, Requires<[In64BitMode]>; // Floating point stack aliases. @@ -2241,6 +2497,42 @@ def : InstAlias<"clrw $reg", (XOR16rr GR16:$reg, GR16:$reg), 0>; def : InstAlias<"clrl $reg", (XOR32rr GR32:$reg, GR32:$reg), 0>; def : InstAlias<"clrq $reg", (XOR64rr GR64:$reg, GR64:$reg), 0>; +// lods aliases. Accept the destination being omitted because it's implicit +// in the mnemonic, or the mnemonic suffix being omitted because it's implicit +// in the destination. +def : InstAlias<"lodsb $src", (LODSB srcidx8:$src), 0>; +def : InstAlias<"lodsw $src", (LODSW srcidx16:$src), 0>; +def : InstAlias<"lods{l|d} $src", (LODSL srcidx32:$src), 0>; +def : InstAlias<"lodsq $src", (LODSQ srcidx64:$src), 0>, Requires<[In64BitMode]>; +def : InstAlias<"lods {$src, %al|al, $src}", (LODSB srcidx8:$src), 0>; +def : InstAlias<"lods {$src, %ax|ax, $src}", (LODSW srcidx16:$src), 0>; +def : InstAlias<"lods {$src, %eax|eax, $src}", (LODSL srcidx32:$src), 0>; +def : InstAlias<"lods {$src, %rax|rax, $src}", (LODSQ srcidx64:$src), 0>, Requires<[In64BitMode]>; + +// stos aliases. Accept the source being omitted because it's implicit in +// the mnemonic, or the mnemonic suffix being omitted because it's implicit +// in the source. +def : InstAlias<"stosb $dst", (STOSB dstidx8:$dst), 0>; +def : InstAlias<"stosw $dst", (STOSW dstidx16:$dst), 0>; +def : InstAlias<"stos{l|d} $dst", (STOSL dstidx32:$dst), 0>; +def : InstAlias<"stosq $dst", (STOSQ dstidx64:$dst), 0>, Requires<[In64BitMode]>; +def : InstAlias<"stos {%al, $dst|$dst, al}", (STOSB dstidx8:$dst), 0>; +def : InstAlias<"stos {%ax, $dst|$dst, ax}", (STOSW dstidx16:$dst), 0>; +def : InstAlias<"stos {%eax, $dst|$dst, eax}", (STOSL dstidx32:$dst), 0>; +def : InstAlias<"stos {%rax, $dst|$dst, rax}", (STOSQ dstidx64:$dst), 0>, Requires<[In64BitMode]>; + +// scas aliases. Accept the destination being omitted because it's implicit +// in the mnemonic, or the mnemonic suffix being omitted because it's implicit +// in the destination. +def : InstAlias<"scasb $dst", (SCASB dstidx8:$dst), 0>; +def : InstAlias<"scasw $dst", (SCASW dstidx16:$dst), 0>; +def : InstAlias<"scas{l|d} $dst", (SCASL dstidx32:$dst), 0>; +def : InstAlias<"scasq $dst", (SCASQ dstidx64:$dst), 0>, Requires<[In64BitMode]>; +def : InstAlias<"scas {$dst, %al|al, $dst}", (SCASB dstidx8:$dst), 0>; +def : InstAlias<"scas {$dst, %ax|ax, $dst}", (SCASW dstidx16:$dst), 0>; +def : InstAlias<"scas {$dst, %eax|eax, $dst}", (SCASL dstidx32:$dst), 0>; +def : InstAlias<"scas {$dst, %rax|rax, $dst}", (SCASQ dstidx64:$dst), 0>, Requires<[In64BitMode]>; + // div and idiv aliases for explicit A register. def : InstAlias<"div{b}\t{$src, %al|al, $src}", (DIV8r GR8 :$src)>; def : InstAlias<"div{w}\t{$src, %ax|ax, $src}", (DIV16r GR16:$src)>; @@ -2325,10 +2617,22 @@ def : InstAlias<"fnstsw" , (FNSTSW16r)>; // lcall and ljmp aliases. This seems to be an odd mapping in 64-bit mode, but // this is compatible with what GAS does. -def : InstAlias<"lcall $seg, $off", (FARCALL32i i32imm:$off, i16imm:$seg)>; -def : InstAlias<"ljmp $seg, $off", (FARJMP32i i32imm:$off, i16imm:$seg)>; -def : InstAlias<"lcall *$dst", (FARCALL32m opaque48mem:$dst)>; -def : InstAlias<"ljmp *$dst", (FARJMP32m opaque48mem:$dst)>; +def : InstAlias<"lcall $seg, $off", (FARCALL32i i32imm:$off, i16imm:$seg)>, Requires<[Not16BitMode]>; +def : InstAlias<"ljmp $seg, $off", (FARJMP32i i32imm:$off, i16imm:$seg)>, Requires<[Not16BitMode]>; +def : InstAlias<"lcall *$dst", (FARCALL32m opaque48mem:$dst)>, Requires<[Not16BitMode]>; +def : InstAlias<"ljmp *$dst", (FARJMP32m opaque48mem:$dst)>, Requires<[Not16BitMode]>; +def : InstAlias<"lcall $seg, $off", (FARCALL16i i16imm:$off, i16imm:$seg)>, Requires<[In16BitMode]>; +def : InstAlias<"ljmp $seg, $off", (FARJMP16i i16imm:$off, i16imm:$seg)>, Requires<[In16BitMode]>; +def : InstAlias<"lcall *$dst", (FARCALL16m opaque32mem:$dst)>, Requires<[In16BitMode]>; +def : InstAlias<"ljmp *$dst", (FARJMP16m opaque32mem:$dst)>, Requires<[In16BitMode]>; + +def : InstAlias<"call *$dst", (CALL64m i16mem:$dst)>, Requires<[In64BitMode]>; +def : InstAlias<"jmp *$dst", (JMP64m i16mem:$dst)>, Requires<[In64BitMode]>; +def : InstAlias<"call *$dst", (CALL32m i16mem:$dst)>, Requires<[In32BitMode]>; +def : InstAlias<"jmp *$dst", (JMP32m i16mem:$dst)>, Requires<[In32BitMode]>; +def : InstAlias<"call *$dst", (CALL16m i16mem:$dst)>, Requires<[In16BitMode]>; +def : InstAlias<"jmp *$dst", (JMP16m i16mem:$dst)>, Requires<[In16BitMode]>; + // "imul <imm>, B" is an alias for "imul <imm>, B, B". def : InstAlias<"imulw $imm, $r", (IMUL16rri GR16:$r, GR16:$r, i16imm:$imm)>; @@ -2348,8 +2652,10 @@ def : InstAlias<"inl\t$port", (IN32ri i8imm:$port), 0>; // jmp and call aliases for lcall and ljmp. jmp $42,$5 -> ljmp -def : InstAlias<"call $seg, $off", (FARCALL32i i32imm:$off, i16imm:$seg)>; -def : InstAlias<"jmp $seg, $off", (FARJMP32i i32imm:$off, i16imm:$seg)>; +def : InstAlias<"call $seg, $off", (FARCALL16i i16imm:$off, i16imm:$seg)>, Requires<[In16BitMode]>; +def : InstAlias<"jmp $seg, $off", (FARJMP16i i16imm:$off, i16imm:$seg)>, Requires<[In16BitMode]>; +def : InstAlias<"call $seg, $off", (FARCALL32i i32imm:$off, i16imm:$seg)>, Requires<[Not16BitMode]>; +def : InstAlias<"jmp $seg, $off", (FARJMP32i i32imm:$off, i16imm:$seg)>, Requires<[Not16BitMode]>; def : InstAlias<"callw $seg, $off", (FARCALL16i i16imm:$off, i16imm:$seg)>; def : InstAlias<"jmpw $seg, $off", (FARJMP16i i16imm:$off, i16imm:$seg)>; def : InstAlias<"calll $seg, $off", (FARCALL32i i32imm:$off, i16imm:$seg)>; @@ -2370,10 +2676,6 @@ def : InstAlias<"movq $src, $dst", def : InstAlias<"movq $src, $dst", (MMX_MOVD64from64rr GR64:$dst, VR64:$src), 0>; -// movsd with no operands (as opposed to the SSE scalar move of a double) is an -// alias for movsl. (as in rep; movsd) -def : InstAlias<"movsd", (MOVSD), 0>; - // movsx aliases def : InstAlias<"movsx $src, $dst", (MOVSX16rr8 GR16:$dst, GR8:$src), 0>; def : InstAlias<"movsx $src, $dst", (MOVSX16rm8 GR16:$dst, i8mem:$src), 0>; @@ -2462,6 +2764,6 @@ def : InstAlias<"xchg{q}\t{$mem, $val|$val, $mem}", (XCHG64rm GR64:$val, i64mem: // xchg: We accept "xchgX <reg>, %eax" and "xchgX %eax, <reg>" as synonyms. def : InstAlias<"xchg{w}\t{%ax, $src|$src, ax}", (XCHG16ar GR16:$src)>; -def : InstAlias<"xchg{l}\t{%eax, $src|$src, eax}", (XCHG32ar GR32:$src)>, Requires<[In32BitMode]>; +def : InstAlias<"xchg{l}\t{%eax, $src|$src, eax}", (XCHG32ar GR32:$src)>, Requires<[Not64BitMode]>; def : InstAlias<"xchg{l}\t{%eax, $src|$src, eax}", (XCHG32ar64 GR32_NOAX:$src)>, Requires<[In64BitMode]>; def : InstAlias<"xchg{q}\t{%rax, $src|$src, rax}", (XCHG64ar GR64:$src)>; diff --git a/lib/Target/X86/X86InstrMMX.td b/lib/Target/X86/X86InstrMMX.td index ba58143..050ee39 100644 --- a/lib/Target/X86/X86InstrMMX.td +++ b/lib/Target/X86/X86InstrMMX.td @@ -524,24 +524,24 @@ def MMX_PSHUFWmi : MMXIi8<0x70, MRMSrcMem, // -- Conversion Instructions defm MMX_CVTPS2PI : sse12_cvt_pint<0x2D, VR128, VR64, int_x86_sse_cvtps2pi, f64mem, load, "cvtps2pi\t{$src, $dst|$dst, $src}", - MMX_CVT_PS_ITINS, SSEPackedSingle>, TB; + MMX_CVT_PS_ITINS, SSEPackedSingle>, PS; defm MMX_CVTPD2PI : sse12_cvt_pint<0x2D, VR128, VR64, int_x86_sse_cvtpd2pi, f128mem, memop, "cvtpd2pi\t{$src, $dst|$dst, $src}", - MMX_CVT_PD_ITINS, SSEPackedDouble>, TB, OpSize; + MMX_CVT_PD_ITINS, SSEPackedDouble>, PD; defm MMX_CVTTPS2PI : sse12_cvt_pint<0x2C, VR128, VR64, int_x86_sse_cvttps2pi, f64mem, load, "cvttps2pi\t{$src, $dst|$dst, $src}", - MMX_CVT_PS_ITINS, SSEPackedSingle>, TB; + MMX_CVT_PS_ITINS, SSEPackedSingle>, PS; defm MMX_CVTTPD2PI : sse12_cvt_pint<0x2C, VR128, VR64, int_x86_sse_cvttpd2pi, f128mem, memop, "cvttpd2pi\t{$src, $dst|$dst, $src}", - MMX_CVT_PD_ITINS, SSEPackedDouble>, TB, OpSize; + MMX_CVT_PD_ITINS, SSEPackedDouble>, PD; defm MMX_CVTPI2PD : sse12_cvt_pint<0x2A, VR64, VR128, int_x86_sse_cvtpi2pd, i64mem, load, "cvtpi2pd\t{$src, $dst|$dst, $src}", - MMX_CVT_PD_ITINS, SSEPackedDouble>, TB, OpSize; + MMX_CVT_PD_ITINS, SSEPackedDouble>, PD; let Constraints = "$src1 = $dst" in { defm MMX_CVTPI2PS : sse12_cvt_pint_3addr<0x2A, VR64, VR128, int_x86_sse_cvtpi2ps, i64mem, load, "cvtpi2ps\t{$src2, $dst|$dst, $src2}", - SSEPackedSingle>, TB; + SSEPackedSingle>, PS; } // Extract / Insert diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td index a5debc0..f2f3967 100644 --- a/lib/Target/X86/X86InstrSSE.td +++ b/lib/Target/X86/X86InstrSSE.td @@ -120,6 +120,11 @@ def SSE_DIV_ITINS_P : SizeItins< SSE_DIV_F32P, SSE_DIV_F64P >; +let Sched = WriteVecLogic in +def SSE_VEC_BIT_ITINS_P : OpndItins< + IIC_SSE_BIT_P_RR, IIC_SSE_BIT_P_RM +>; + def SSE_BIT_ITINS_P : OpndItins< IIC_SSE_BIT_P_RR, IIC_SSE_BIT_P_RM >; @@ -171,6 +176,7 @@ def SSE_INSERT_ITINS : OpndItins< IIC_SSE_INSERTPS_RR, IIC_SSE_INSERTPS_RM >; +let Sched = WriteMPSAD in def SSE_MPSADBW_ITINS : OpndItins< IIC_SSE_MPSADBW_RR, IIC_SSE_MPSADBW_RM >; @@ -179,6 +185,44 @@ def SSE_PMULLD_ITINS : OpndItins< IIC_SSE_PMULLD_RR, IIC_SSE_PMULLD_RM >; +// Definitions for backward compatibility. +// The instructions mapped on these definitions uses a different itinerary +// than the actual scheduling model. +let Sched = WriteShuffle in +def DEFAULT_ITINS_SHUFFLESCHED : OpndItins< + IIC_ALU_NONMEM, IIC_ALU_MEM +>; + +let Sched = WriteVecIMul in +def DEFAULT_ITINS_VECIMULSCHED : OpndItins< + IIC_ALU_NONMEM, IIC_ALU_MEM +>; + +let Sched = WriteShuffle in +def SSE_INTALU_ITINS_SHUFF_P : OpndItins< + IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM +>; + +let Sched = WriteMPSAD in +def DEFAULT_ITINS_MPSADSCHED : OpndItins< + IIC_ALU_NONMEM, IIC_ALU_MEM +>; + +let Sched = WriteFBlend in +def DEFAULT_ITINS_FBLENDSCHED : OpndItins< + IIC_ALU_NONMEM, IIC_ALU_MEM +>; + +let Sched = WriteBlend in +def DEFAULT_ITINS_BLENDSCHED : OpndItins< + IIC_ALU_NONMEM, IIC_ALU_MEM +>; + +let Sched = WriteFBlend in +def SSE_INTALU_ITINS_FBLEND_P : OpndItins< + IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM +>; + //===----------------------------------------------------------------------===// // SSE 1 & 2 Instructions Classes //===----------------------------------------------------------------------===// @@ -210,6 +254,7 @@ multiclass sse12_fp_scalar_int<bits<8> opc, string OpcodeStr, RegisterClass RC, Operand memopr, ComplexPattern mem_cpat, OpndItins itins, bit Is2Addr = 1> { +let isCodeGenOnly = 1 in { def rr_Int : SI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), !if(Is2Addr, !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), @@ -227,6 +272,7 @@ multiclass sse12_fp_scalar_int<bits<8> opc, string OpcodeStr, RegisterClass RC, RC:$src1, mem_cpat:$src2))], itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; } +} /// sse12_fp_packed - SSE 1 & 2 packed instructions class multiclass sse12_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, @@ -497,14 +543,14 @@ multiclass sse12_move_rr<RegisterClass RC, SDNode OpNode, ValueType vt, !strconcat(base_opc, asm_opr), [(set VR128:$dst, (vt (OpNode VR128:$src1, (scalar_to_vector RC:$src2))))], - IIC_SSE_MOV_S_RR>, Sched<[WriteMove]>; + IIC_SSE_MOV_S_RR>, Sched<[WriteFShuffle]>; // For the disassembler - let isCodeGenOnly = 1, hasSideEffects = 0 in + let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src1, RC:$src2), !strconcat(base_opc, asm_opr), - [], IIC_SSE_MOV_S_RR>, Sched<[WriteMove]>; + [], IIC_SSE_MOV_S_RR>, Sched<[WriteFShuffle]>; } multiclass sse12_move<RegisterClass RC, SDNode OpNode, ValueType vt, @@ -800,7 +846,7 @@ multiclass sse12_mov_packed<bits<8> opc, RegisterClass RC, let neverHasSideEffects = 1 in def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [], itins.rr, d>, - Sched<[WriteMove]>; + Sched<[WriteFShuffle]>; let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), !strconcat(asm, "\t{$src, $dst|$dst, $src}"), @@ -810,41 +856,41 @@ let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in defm VMOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, - TB, VEX; + PS, VEX; defm VMOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, - TB, OpSize, VEX; + PD, VEX; defm VMOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, "movups", SSEPackedSingle, SSE_MOVU_ITINS>, - TB, VEX; + PS, VEX; defm VMOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, - TB, OpSize, VEX; + PD, VEX; defm VMOVAPSY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv8f32, "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, - TB, VEX, VEX_L; + PS, VEX, VEX_L; defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64, "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, - TB, OpSize, VEX, VEX_L; + PD, VEX, VEX_L; defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32, "movups", SSEPackedSingle, SSE_MOVU_ITINS>, - TB, VEX, VEX_L; + PS, VEX, VEX_L; defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64, "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, - TB, OpSize, VEX, VEX_L; + PD, VEX, VEX_L; defm MOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, - TB; + PS; defm MOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, - TB, OpSize; + PD; defm MOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, "movups", SSEPackedSingle, SSE_MOVU_ITINS>, - TB; + PS; defm MOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, - TB, OpSize; + PD; let SchedRW = [WriteStore] in { def VMOVAPSmr : VPSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), @@ -882,7 +928,8 @@ def VMOVUPDYmr : VPDI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), } // SchedRW // For disassembler -let isCodeGenOnly = 1, hasSideEffects = 0, SchedRW = [WriteMove] in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, + SchedRW = [WriteFShuffle] in { def VMOVAPSrr_REV : VPSI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), "movaps\t{$src, $dst|$dst, $src}", [], @@ -958,7 +1005,8 @@ def MOVUPDmr : PDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), } // SchedRW // For disassembler -let isCodeGenOnly = 1, hasSideEffects = 0, SchedRW = [WriteMove] in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, + SchedRW = [WriteMove] in { def MOVAPSrr_REV : PSI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), "movaps\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>; @@ -1138,16 +1186,16 @@ multiclass sse12_mov_hilo_packed_base<bits<8>opc, SDNode psnode, SDNode pdnode, [(set VR128:$dst, (psnode VR128:$src1, (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))))], - itin, SSEPackedSingle>, TB, - Sched<[WriteShuffleLd, ReadAfterLd]>; + itin, SSEPackedSingle>, PS, + Sched<[WriteFShuffleLd, ReadAfterLd]>; def PDrm : PI<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2), !strconcat(base_opc, "d", asm_opr), [(set VR128:$dst, (v2f64 (pdnode VR128:$src1, (scalar_to_vector (loadf64 addr:$src2)))))], - itin, SSEPackedDouble>, TB, OpSize, - Sched<[WriteShuffleLd, ReadAfterLd]>; + itin, SSEPackedDouble>, PD, + Sched<[WriteFShuffleLd, ReadAfterLd]>; } @@ -1345,14 +1393,14 @@ let AddedComplexity = 20, Predicates = [UseAVX] in { [(set VR128:$dst, (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))], IIC_SSE_MOV_LH>, - VEX_4V, Sched<[WriteShuffle]>; + VEX_4V, Sched<[WriteFShuffle]>; def VMOVHLPSrr : VPSI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), "movhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))], IIC_SSE_MOV_LH>, - VEX_4V, Sched<[WriteShuffle]>; + VEX_4V, Sched<[WriteFShuffle]>; } let Constraints = "$src1 = $dst", AddedComplexity = 20 in { def MOVLHPSrr : PSI<0x16, MRMSrcReg, (outs VR128:$dst), @@ -1360,13 +1408,13 @@ let Constraints = "$src1 = $dst", AddedComplexity = 20 in { "movlhps\t{$src2, $dst|$dst, $src2}", [(set VR128:$dst, (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))], - IIC_SSE_MOV_LH>, Sched<[WriteShuffle]>; + IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; def MOVHLPSrr : PSI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), "movhlps\t{$src2, $dst|$dst, $src2}", [(set VR128:$dst, (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))], - IIC_SSE_MOV_LH>, Sched<[WriteShuffle]>; + IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; } let Predicates = [UseAVX] in { @@ -1632,40 +1680,43 @@ defm CVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse2_cvtsd2si64, sdmem, sse_load_f64, "cvtsd2si", SSE_CVT_SD2SI>, XD, REX_W; -let Predicates = [UseAVX] in { -defm Int_VCVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, - int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", - SSE_CVT_Scalar, 0>, XS, VEX_4V; -defm Int_VCVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, - int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", - SSE_CVT_Scalar, 0>, XS, VEX_4V, - VEX_W; -defm Int_VCVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, - int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", - SSE_CVT_Scalar, 0>, XD, VEX_4V; -defm Int_VCVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, - int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", - SSE_CVT_Scalar, 0>, XD, - VEX_4V, VEX_W; -} -let Constraints = "$src1 = $dst" in { - defm Int_CVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, - int_x86_sse_cvtsi2ss, i32mem, loadi32, - "cvtsi2ss{l}", SSE_CVT_Scalar>, XS; - defm Int_CVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, - int_x86_sse_cvtsi642ss, i64mem, loadi64, - "cvtsi2ss{q}", SSE_CVT_Scalar>, XS, REX_W; - defm Int_CVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, - int_x86_sse2_cvtsi2sd, i32mem, loadi32, - "cvtsi2sd{l}", SSE_CVT_Scalar>, XD; - defm Int_CVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, - int_x86_sse2_cvtsi642sd, i64mem, loadi64, - "cvtsi2sd{q}", SSE_CVT_Scalar>, XD, REX_W; -} +let isCodeGenOnly = 1 in { + let Predicates = [UseAVX] in { + defm Int_VCVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, + int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", + SSE_CVT_Scalar, 0>, XS, VEX_4V; + defm Int_VCVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, + int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", + SSE_CVT_Scalar, 0>, XS, VEX_4V, + VEX_W; + defm Int_VCVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, + int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", + SSE_CVT_Scalar, 0>, XD, VEX_4V; + defm Int_VCVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, + int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", + SSE_CVT_Scalar, 0>, XD, + VEX_4V, VEX_W; + } + let Constraints = "$src1 = $dst" in { + defm Int_CVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, + int_x86_sse_cvtsi2ss, i32mem, loadi32, + "cvtsi2ss{l}", SSE_CVT_Scalar>, XS; + defm Int_CVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, + int_x86_sse_cvtsi642ss, i64mem, loadi64, + "cvtsi2ss{q}", SSE_CVT_Scalar>, XS, REX_W; + defm Int_CVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, + int_x86_sse2_cvtsi2sd, i32mem, loadi32, + "cvtsi2sd{l}", SSE_CVT_Scalar>, XD; + defm Int_CVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, + int_x86_sse2_cvtsi642sd, i64mem, loadi64, + "cvtsi2sd{q}", SSE_CVT_Scalar>, XD, REX_W; + } +} // isCodeGenOnly = 1 /// SSE 1 Only // Aliases for intrinsics +let isCodeGenOnly = 1 in { let Predicates = [UseAVX] in { defm Int_VCVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse_cvttss2si, ssmem, sse_load_f32, "cvttss2si", @@ -1694,6 +1745,7 @@ defm Int_CVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse2_cvttsd2si, defm Int_CVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, "cvttsd2si", SSE_CVT_SD2SI>, XD, REX_W; +} // isCodeGenOnly = 1 let Predicates = [UseAVX] in { defm VCVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si, @@ -1713,16 +1765,16 @@ defm CVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse_cvtss2si64, defm VCVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem, "vcvtdq2ps\t{$src, $dst|$dst, $src}", SSEPackedSingle, SSE_CVT_PS>, - TB, VEX, Requires<[HasAVX]>; + PS, VEX, Requires<[HasAVX]>; defm VCVTDQ2PSY : sse12_cvt_p<0x5B, VR256, VR256, i256mem, "vcvtdq2ps\t{$src, $dst|$dst, $src}", SSEPackedSingle, SSE_CVT_PS>, - TB, VEX, VEX_L, Requires<[HasAVX]>; + PS, VEX, VEX_L, Requires<[HasAVX]>; defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem, "cvtdq2ps\t{$src, $dst|$dst, $src}", SSEPackedSingle, SSE_CVT_PS>, - TB, Requires<[UseSSE2]>; + PS, Requires<[UseSSE2]>; let Predicates = [UseAVX] in { def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}", @@ -1792,6 +1844,7 @@ def CVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst), (ins f64mem:$src), XD, Requires<[UseSSE2, OptForSize]>, Sched<[WriteCvtF2FLd]>; +let isCodeGenOnly = 1 in { def Int_VCVTSD2SSrr: I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", @@ -1823,6 +1876,7 @@ def Int_CVTSD2SSrm: I<0x5A, MRMSrcReg, IIC_SSE_CVT_Scalar_RM>, XD, Requires<[UseSSE2]>, Sched<[WriteCvtF2FLd, ReadAfterLd]>; } +} // isCodeGenOnly = 1 // Convert scalar single to scalar double // SSE2 instructions with XS prefix @@ -1875,6 +1929,7 @@ def : Pat<(fextend (loadf32 addr:$src)), def : Pat<(extloadf32 addr:$src), (CVTSS2SDrr (MOVSSrm addr:$src))>, Requires<[UseSSE2, OptForSpeed]>; +let isCodeGenOnly = 1 in { def Int_VCVTSS2SDrr: I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", @@ -1905,6 +1960,7 @@ def Int_CVTSS2SDrm: I<0x5A, MRMSrcMem, IIC_SSE_CVT_Scalar_RM>, XS, Requires<[UseSSE2]>, Sched<[WriteCvtF2FLd, ReadAfterLd]>; } +} // isCodeGenOnly = 1 // Convert packed single/double fp to doubleword def VCVTPS2DQrr : VPDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), @@ -2116,32 +2172,32 @@ let Predicates = [HasAVX] in { def VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), "vcvtps2pd\t{$src, $dst|$dst, $src}", [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))], - IIC_SSE_CVT_PD_RR>, TB, VEX, Sched<[WriteCvtF2F]>; + IIC_SSE_CVT_PD_RR>, PS, VEX, Sched<[WriteCvtF2F]>; def VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), "vcvtps2pd\t{$src, $dst|$dst, $src}", [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))], - IIC_SSE_CVT_PD_RM>, TB, VEX, Sched<[WriteCvtF2FLd]>; + IIC_SSE_CVT_PD_RM>, PS, VEX, Sched<[WriteCvtF2FLd]>; def VCVTPS2PDYrr : I<0x5A, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), "vcvtps2pd\t{$src, $dst|$dst, $src}", [(set VR256:$dst, (int_x86_avx_cvt_ps2_pd_256 VR128:$src))], - IIC_SSE_CVT_PD_RR>, TB, VEX, VEX_L, Sched<[WriteCvtF2F]>; + IIC_SSE_CVT_PD_RR>, PS, VEX, VEX_L, Sched<[WriteCvtF2F]>; def VCVTPS2PDYrm : I<0x5A, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src), "vcvtps2pd\t{$src, $dst|$dst, $src}", [(set VR256:$dst, (int_x86_avx_cvt_ps2_pd_256 (loadv4f32 addr:$src)))], - IIC_SSE_CVT_PD_RM>, TB, VEX, VEX_L, Sched<[WriteCvtF2FLd]>; + IIC_SSE_CVT_PD_RM>, PS, VEX, VEX_L, Sched<[WriteCvtF2FLd]>; } let Predicates = [UseSSE2] in { def CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), "cvtps2pd\t{$src, $dst|$dst, $src}", [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))], - IIC_SSE_CVT_PD_RR>, TB, Sched<[WriteCvtF2F]>; + IIC_SSE_CVT_PD_RR>, PS, Sched<[WriteCvtF2F]>; def CVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), "cvtps2pd\t{$src, $dst|$dst, $src}", [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))], - IIC_SSE_CVT_PD_RM>, TB, Sched<[WriteCvtF2FLd]>; + IIC_SSE_CVT_PD_RM>, PS, Sched<[WriteCvtF2FLd]>; } // Convert Packed DW Integers to Packed Double FP @@ -2287,7 +2343,7 @@ multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, Sched<[itins.Sched.Folded, ReadAfterLd]>; // Accept explicit immediate argument form instead of comparison code. - let neverHasSideEffects = 1 in { + let isAsmParserOnly = 1, hasSideEffects = 0 in { def rr_alt : SIi8<0xC2, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc), asm_alt, [], IIC_SSE_ALU_F32S_RR>, Sched<[itins.Sched]>; @@ -2299,23 +2355,23 @@ multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, } } -defm VCMPSS : sse12_cmp_scalar<FR32, f32mem, AVXCC, X86cmpss, f32, loadf32, +defm VCMPSS : sse12_cmp_scalar<FR32, f32mem, AVXCC, X86cmps, f32, loadf32, "cmp${cc}ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", "cmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", SSE_ALU_F32S>, XS, VEX_4V, VEX_LIG; -defm VCMPSD : sse12_cmp_scalar<FR64, f64mem, AVXCC, X86cmpsd, f64, loadf64, +defm VCMPSD : sse12_cmp_scalar<FR64, f64mem, AVXCC, X86cmps, f64, loadf64, "cmp${cc}sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", "cmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", SSE_ALU_F32S>, // same latency as 32 bit compare XD, VEX_4V, VEX_LIG; let Constraints = "$src1 = $dst" in { - defm CMPSS : sse12_cmp_scalar<FR32, f32mem, SSECC, X86cmpss, f32, loadf32, + defm CMPSS : sse12_cmp_scalar<FR32, f32mem, SSECC, X86cmps, f32, loadf32, "cmp${cc}ss\t{$src2, $dst|$dst, $src2}", "cmpss\t{$cc, $src2, $dst|$dst, $src2, $cc}", SSE_ALU_F32S>, XS; - defm CMPSD : sse12_cmp_scalar<FR64, f64mem, SSECC, X86cmpsd, f64, loadf64, + defm CMPSD : sse12_cmp_scalar<FR64, f64mem, SSECC, X86cmps, f64, loadf64, "cmp${cc}sd\t{$src2, $dst|$dst, $src2}", "cmpsd\t{$cc, $src2, $dst|$dst, $src2, $cc}", SSE_ALU_F64S>, @@ -2338,23 +2394,25 @@ multiclass sse12_cmp_scalar_int<X86MemOperand x86memop, Operand CC, Sched<[itins.Sched.Folded, ReadAfterLd]>; } -// Aliases to match intrinsics which expect XMM operand(s). -defm Int_VCMPSS : sse12_cmp_scalar_int<f32mem, AVXCC, int_x86_sse_cmp_ss, - "cmp${cc}ss\t{$src, $src1, $dst|$dst, $src1, $src}", - SSE_ALU_F32S>, - XS, VEX_4V; -defm Int_VCMPSD : sse12_cmp_scalar_int<f64mem, AVXCC, int_x86_sse2_cmp_sd, - "cmp${cc}sd\t{$src, $src1, $dst|$dst, $src1, $src}", - SSE_ALU_F32S>, // same latency as f32 - XD, VEX_4V; -let Constraints = "$src1 = $dst" in { - defm Int_CMPSS : sse12_cmp_scalar_int<f32mem, SSECC, int_x86_sse_cmp_ss, - "cmp${cc}ss\t{$src, $dst|$dst, $src}", - SSE_ALU_F32S>, XS; - defm Int_CMPSD : sse12_cmp_scalar_int<f64mem, SSECC, int_x86_sse2_cmp_sd, - "cmp${cc}sd\t{$src, $dst|$dst, $src}", - SSE_ALU_F64S>, - XD; +let isCodeGenOnly = 1 in { + // Aliases to match intrinsics which expect XMM operand(s). + defm Int_VCMPSS : sse12_cmp_scalar_int<f32mem, AVXCC, int_x86_sse_cmp_ss, + "cmp${cc}ss\t{$src, $src1, $dst|$dst, $src1, $src}", + SSE_ALU_F32S>, + XS, VEX_4V; + defm Int_VCMPSD : sse12_cmp_scalar_int<f64mem, AVXCC, int_x86_sse2_cmp_sd, + "cmp${cc}sd\t{$src, $src1, $dst|$dst, $src1, $src}", + SSE_ALU_F32S>, // same latency as f32 + XD, VEX_4V; + let Constraints = "$src1 = $dst" in { + defm Int_CMPSS : sse12_cmp_scalar_int<f32mem, SSECC, int_x86_sse_cmp_ss, + "cmp${cc}ss\t{$src, $dst|$dst, $src}", + SSE_ALU_F32S>, XS; + defm Int_CMPSD : sse12_cmp_scalar_int<f64mem, SSECC, int_x86_sse2_cmp_sd, + "cmp${cc}sd\t{$src, $dst|$dst, $src}", + SSE_ALU_F64S>, + XD; +} } @@ -2377,46 +2435,50 @@ multiclass sse12_ord_cmp<bits<8> opc, RegisterClass RC, SDNode OpNode, let Defs = [EFLAGS] in { defm VUCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32, - "ucomiss">, TB, VEX, VEX_LIG; + "ucomiss">, PS, VEX, VEX_LIG; defm VUCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64, - "ucomisd">, TB, OpSize, VEX, VEX_LIG; + "ucomisd">, PD, VEX, VEX_LIG; let Pattern = []<dag> in { defm VCOMISS : sse12_ord_cmp<0x2F, VR128, undef, v4f32, f128mem, load, - "comiss">, TB, VEX, VEX_LIG; + "comiss">, PS, VEX, VEX_LIG; defm VCOMISD : sse12_ord_cmp<0x2F, VR128, undef, v2f64, f128mem, load, - "comisd">, TB, OpSize, VEX, VEX_LIG; + "comisd">, PD, VEX, VEX_LIG; } - defm Int_VUCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, - load, "ucomiss">, TB, VEX; - defm Int_VUCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, - load, "ucomisd">, TB, OpSize, VEX; + let isCodeGenOnly = 1 in { + defm Int_VUCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, + load, "ucomiss">, PS, VEX; + defm Int_VUCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, + load, "ucomisd">, PD, VEX; - defm Int_VCOMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, - load, "comiss">, TB, VEX; - defm Int_VCOMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, - load, "comisd">, TB, OpSize, VEX; + defm Int_VCOMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, + load, "comiss">, PS, VEX; + defm Int_VCOMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, + load, "comisd">, PD, VEX; + } defm UCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32, - "ucomiss">, TB; + "ucomiss">, PS; defm UCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64, - "ucomisd">, TB, OpSize; + "ucomisd">, PD; let Pattern = []<dag> in { defm COMISS : sse12_ord_cmp<0x2F, VR128, undef, v4f32, f128mem, load, - "comiss">, TB; + "comiss">, PS; defm COMISD : sse12_ord_cmp<0x2F, VR128, undef, v2f64, f128mem, load, - "comisd">, TB, OpSize; + "comisd">, PD; } - defm Int_UCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, - load, "ucomiss">, TB; - defm Int_UCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, - load, "ucomisd">, TB, OpSize; + let isCodeGenOnly = 1 in { + defm Int_UCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, + load, "ucomiss">, PS; + defm Int_UCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, + load, "ucomisd">, PD; - defm Int_COMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, load, - "comiss">, TB; - defm Int_COMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, load, - "comisd">, TB, OpSize; + defm Int_COMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, load, + "comiss">, PS; + defm Int_COMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, load, + "comisd">, PD; + } } // Defs = [EFLAGS] // sse12_cmp_packed - sse 1 & 2 compare packed instructions @@ -2436,7 +2498,7 @@ multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop, Sched<[WriteFAddLd, ReadAfterLd]>; // Accept explicit immediate argument form instead of comparison code. - let neverHasSideEffects = 1 in { + let isAsmParserOnly = 1, hasSideEffects = 0 in { def rri_alt : PIi8<0xC2, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc), asm_alt, [], itins.rr, d>, Sched<[WriteFAdd]>; @@ -2450,28 +2512,28 @@ multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop, defm VCMPPS : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse_cmp_ps, "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", - SSEPackedSingle>, TB, VEX_4V; + SSEPackedSingle>, PS, VEX_4V; defm VCMPPD : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse2_cmp_pd, "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", - SSEPackedDouble>, TB, OpSize, VEX_4V; + SSEPackedDouble>, PD, VEX_4V; defm VCMPPSY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_ps_256, "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", - SSEPackedSingle>, TB, VEX_4V, VEX_L; + SSEPackedSingle>, PS, VEX_4V, VEX_L; defm VCMPPDY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_pd_256, "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", - SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L; + SSEPackedDouble>, PD, VEX_4V, VEX_L; let Constraints = "$src1 = $dst" in { defm CMPPS : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse_cmp_ps, "cmp${cc}ps\t{$src2, $dst|$dst, $src2}", "cmpps\t{$cc, $src2, $dst|$dst, $src2, $cc}", - SSEPackedSingle, SSE_ALU_F32P>, TB; + SSEPackedSingle, SSE_ALU_F32P>, PS; defm CMPPD : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse2_cmp_pd, "cmp${cc}pd\t{$src2, $dst|$dst, $src2}", "cmppd\t{$cc, $src2, $dst|$dst, $src2, $cc}", - SSEPackedDouble, SSE_ALU_F64P>, TB, OpSize; + SSEPackedDouble, SSE_ALU_F64P>, PD; } let Predicates = [HasAVX] in { @@ -2512,7 +2574,7 @@ def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), (memop addr:$src2), imm:$cc)), // SSE 1 & 2 - Shuffle Instructions //===----------------------------------------------------------------------===// -/// sse12_shuffle - sse 1 & 2 shuffle instructions +/// sse12_shuffle - sse 1 & 2 fp shuffle instructions multiclass sse12_shuffle<RegisterClass RC, X86MemOperand x86memop, ValueType vt, string asm, PatFrag mem_frag, Domain d, bit IsConvertibleToThreeAddress = 0> { @@ -2520,37 +2582,35 @@ multiclass sse12_shuffle<RegisterClass RC, X86MemOperand x86memop, (ins RC:$src1, x86memop:$src2, i8imm:$src3), asm, [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2), (i8 imm:$src3))))], IIC_SSE_SHUFP, d>, - Sched<[WriteShuffleLd, ReadAfterLd]>; + Sched<[WriteFShuffleLd, ReadAfterLd]>; let isConvertibleToThreeAddress = IsConvertibleToThreeAddress in def rri : PIi8<0xC6, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$src3), asm, [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2, (i8 imm:$src3))))], IIC_SSE_SHUFP, d>, - Sched<[WriteShuffle]>; + Sched<[WriteFShuffle]>; } defm VSHUFPS : sse12_shuffle<VR128, f128mem, v4f32, "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - loadv4f32, SSEPackedSingle>, TB, VEX_4V; + loadv4f32, SSEPackedSingle>, PS, VEX_4V; defm VSHUFPSY : sse12_shuffle<VR256, f256mem, v8f32, "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - loadv8f32, SSEPackedSingle>, TB, VEX_4V, VEX_L; + loadv8f32, SSEPackedSingle>, PS, VEX_4V, VEX_L; defm VSHUFPD : sse12_shuffle<VR128, f128mem, v2f64, "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - loadv2f64, SSEPackedDouble>, TB, OpSize, VEX_4V; + loadv2f64, SSEPackedDouble>, PD, VEX_4V; defm VSHUFPDY : sse12_shuffle<VR256, f256mem, v4f64, "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - loadv4f64, SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L; + loadv4f64, SSEPackedDouble>, PD, VEX_4V, VEX_L; let Constraints = "$src1 = $dst" in { defm SHUFPS : sse12_shuffle<VR128, f128mem, v4f32, "shufps\t{$src3, $src2, $dst|$dst, $src2, $src3}", - memopv4f32, SSEPackedSingle, 1 /* cvt to pshufd */>, - TB; + memopv4f32, SSEPackedSingle, 1 /* cvt to pshufd */>, PS; defm SHUFPD : sse12_shuffle<VR128, f128mem, v2f64, "shufpd\t{$src3, $src2, $dst|$dst, $src2, $src3}", - memopv2f64, SSEPackedDouble, 1 /* cvt to pshufd */>, - TB, OpSize; + memopv2f64, SSEPackedDouble, 1 /* cvt to pshufd */>, PD; } let Predicates = [HasAVX] in { @@ -2598,10 +2658,10 @@ let Predicates = [UseSSE2] in { } //===----------------------------------------------------------------------===// -// SSE 1 & 2 - Unpack Instructions +// SSE 1 & 2 - Unpack FP Instructions //===----------------------------------------------------------------------===// -/// sse12_unpack_interleave - sse 1 & 2 unpack and interleave +/// sse12_unpack_interleave - sse 1 & 2 fp unpack and interleave multiclass sse12_unpack_interleave<bits<8> opc, SDNode OpNode, ValueType vt, PatFrag mem_frag, RegisterClass RC, X86MemOperand x86memop, string asm, @@ -2610,55 +2670,55 @@ multiclass sse12_unpack_interleave<bits<8> opc, SDNode OpNode, ValueType vt, (outs RC:$dst), (ins RC:$src1, RC:$src2), asm, [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], - IIC_SSE_UNPCK, d>, Sched<[WriteShuffle]>; + IIC_SSE_UNPCK, d>, Sched<[WriteFShuffle]>; def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), asm, [(set RC:$dst, (vt (OpNode RC:$src1, (mem_frag addr:$src2))))], IIC_SSE_UNPCK, d>, - Sched<[WriteShuffleLd, ReadAfterLd]>; + Sched<[WriteFShuffleLd, ReadAfterLd]>; } defm VUNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, loadv4f32, VR128, f128mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedSingle>, TB, VEX_4V; + SSEPackedSingle>, PS, VEX_4V; defm VUNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, loadv2f64, VR128, f128mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedDouble>, TB, OpSize, VEX_4V; + SSEPackedDouble>, PD, VEX_4V; defm VUNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, loadv4f32, VR128, f128mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedSingle>, TB, VEX_4V; + SSEPackedSingle>, PS, VEX_4V; defm VUNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, loadv2f64, VR128, f128mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedDouble>, TB, OpSize, VEX_4V; + SSEPackedDouble>, PD, VEX_4V; defm VUNPCKHPSY: sse12_unpack_interleave<0x15, X86Unpckh, v8f32, loadv8f32, VR256, f256mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedSingle>, TB, VEX_4V, VEX_L; + SSEPackedSingle>, PS, VEX_4V, VEX_L; defm VUNPCKHPDY: sse12_unpack_interleave<0x15, X86Unpckh, v4f64, loadv4f64, VR256, f256mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L; + SSEPackedDouble>, PD, VEX_4V, VEX_L; defm VUNPCKLPSY: sse12_unpack_interleave<0x14, X86Unpckl, v8f32, loadv8f32, VR256, f256mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedSingle>, TB, VEX_4V, VEX_L; + SSEPackedSingle>, PS, VEX_4V, VEX_L; defm VUNPCKLPDY: sse12_unpack_interleave<0x14, X86Unpckl, v4f64, loadv4f64, VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", - SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L; + SSEPackedDouble>, PD, VEX_4V, VEX_L; let Constraints = "$src1 = $dst" in { defm UNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, memopv4f32, VR128, f128mem, "unpckhps\t{$src2, $dst|$dst, $src2}", - SSEPackedSingle>, TB; + SSEPackedSingle>, PS; defm UNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, memopv2f64, VR128, f128mem, "unpckhpd\t{$src2, $dst|$dst, $src2}", - SSEPackedDouble>, TB, OpSize; + SSEPackedDouble>, PD; defm UNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, memopv4f32, VR128, f128mem, "unpcklps\t{$src2, $dst|$dst, $src2}", - SSEPackedSingle>, TB; + SSEPackedSingle>, PS; defm UNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, memopv2f64, VR128, f128mem, "unpcklpd\t{$src2, $dst|$dst, $src2}", - SSEPackedDouble>, TB, OpSize; + SSEPackedDouble>, PD; } // Constraints = "$src1 = $dst" let Predicates = [HasAVX1Only] in { @@ -2714,16 +2774,15 @@ multiclass sse12_extr_sign_mask<RegisterClass RC, Intrinsic Int, string asm, let Predicates = [HasAVX] in { defm VMOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps, - "movmskps", SSEPackedSingle>, TB, VEX; + "movmskps", SSEPackedSingle>, PS, VEX; defm VMOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd, - "movmskpd", SSEPackedDouble>, TB, - OpSize, VEX; + "movmskpd", SSEPackedDouble>, PD, VEX; defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_ps_256, - "movmskps", SSEPackedSingle>, TB, + "movmskps", SSEPackedSingle>, PS, VEX, VEX_L; defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_pd_256, - "movmskpd", SSEPackedDouble>, TB, - OpSize, VEX, VEX_L; + "movmskpd", SSEPackedDouble>, PD, + VEX, VEX_L; def : Pat<(i32 (X86fgetsign FR32:$src)), (VMOVMSKPSrr (COPY_TO_REGCLASS FR32:$src, VR128))>; @@ -2738,9 +2797,9 @@ let Predicates = [HasAVX] in { } defm MOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps, "movmskps", - SSEPackedSingle>, TB; + SSEPackedSingle>, PS; defm MOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd, "movmskpd", - SSEPackedDouble>, TB, OpSize; + SSEPackedDouble>, PD; def : Pat<(i32 (X86fgetsign FR32:$src)), (MOVMSKPSrr (COPY_TO_REGCLASS FR32:$src, VR128))>, @@ -2807,11 +2866,14 @@ let Predicates = [HasAVX2] in // These are ordered here for pattern ordering requirements with the fp versions -defm PAND : PDI_binop_all<0xDB, "pand", and, v2i64, v4i64, SSE_BIT_ITINS_P, 1>; -defm POR : PDI_binop_all<0xEB, "por", or, v2i64, v4i64, SSE_BIT_ITINS_P, 1>; -defm PXOR : PDI_binop_all<0xEF, "pxor", xor, v2i64, v4i64, SSE_BIT_ITINS_P, 1>; +defm PAND : PDI_binop_all<0xDB, "pand", and, v2i64, v4i64, + SSE_VEC_BIT_ITINS_P, 1>; +defm POR : PDI_binop_all<0xEB, "por", or, v2i64, v4i64, + SSE_VEC_BIT_ITINS_P, 1>; +defm PXOR : PDI_binop_all<0xEF, "pxor", xor, v2i64, v4i64, + SSE_VEC_BIT_ITINS_P, 1>; defm PANDN : PDI_binop_all<0xDF, "pandn", X86andnp, v2i64, v4i64, - SSE_BIT_ITINS_P, 0>; + SSE_VEC_BIT_ITINS_P, 0>; //===----------------------------------------------------------------------===// // SSE 1 & 2 - Logical Instructions @@ -2823,20 +2885,20 @@ multiclass sse12_fp_alias_pack_logical<bits<8> opc, string OpcodeStr, SDNode OpNode, OpndItins itins> { defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, FR32, f32, f128mem, memopfsf32, SSEPackedSingle, itins, 0>, - TB, VEX_4V; + PS, VEX_4V; defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, FR64, f64, f128mem, memopfsf64, SSEPackedDouble, itins, 0>, - TB, OpSize, VEX_4V; + PD, VEX_4V; let Constraints = "$src1 = $dst" in { defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, FR32, f32, f128mem, memopfsf32, SSEPackedSingle, itins>, - TB; + PS; defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, FR64, f64, f128mem, memopfsf64, SSEPackedDouble, itins>, - TB, OpSize; + PD; } } @@ -2862,7 +2924,7 @@ multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "ps"), f256mem, [(set VR256:$dst, (v4i64 (OpNode VR256:$src1, VR256:$src2)))], [(set VR256:$dst, (OpNode (bc_v4i64 (v8f32 VR256:$src1)), - (loadv4i64 addr:$src2)))], 0>, TB, VEX_4V, VEX_L; + (loadv4i64 addr:$src2)))], 0>, PS, VEX_4V, VEX_L; defm V#NAME#PDY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedDouble, !strconcat(OpcodeStr, "pd"), f256mem, @@ -2870,7 +2932,7 @@ multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr, (bc_v4i64 (v4f64 VR256:$src2))))], [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)), (loadv4i64 addr:$src2)))], 0>, - TB, OpSize, VEX_4V, VEX_L; + PD, VEX_4V, VEX_L; // In AVX no need to add a pattern for 128-bit logical rr ps, because they // are all promoted to v2i64, and the patterns are covered by the int @@ -2879,7 +2941,7 @@ multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr, defm V#NAME#PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle, !strconcat(OpcodeStr, "ps"), f128mem, [], [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)), - (loadv2i64 addr:$src2)))], 0>, TB, VEX_4V; + (loadv2i64 addr:$src2)))], 0>, PS, VEX_4V; defm V#NAME#PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble, !strconcat(OpcodeStr, "pd"), f128mem, @@ -2887,21 +2949,21 @@ multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr, (bc_v2i64 (v2f64 VR128:$src2))))], [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), (loadv2i64 addr:$src2)))], 0>, - TB, OpSize, VEX_4V; + PD, VEX_4V; let Constraints = "$src1 = $dst" in { defm PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle, !strconcat(OpcodeStr, "ps"), f128mem, [(set VR128:$dst, (v2i64 (OpNode VR128:$src1, VR128:$src2)))], [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)), - (memopv2i64 addr:$src2)))]>, TB; + (memopv2i64 addr:$src2)))]>, PS; defm PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble, !strconcat(OpcodeStr, "pd"), f128mem, [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), (bc_v2i64 (v2f64 VR128:$src2))))], [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), - (memopv2i64 addr:$src2)))]>, TB, OpSize; + (memopv2i64 addr:$src2)))]>, PD; } } @@ -2932,25 +2994,25 @@ multiclass basic_sse12_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode, SizeItins itins> { defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128, v4f32, f128mem, loadv4f32, - SSEPackedSingle, itins.s, 0>, TB, VEX_4V; + SSEPackedSingle, itins.s, 0>, PS, VEX_4V; defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128, v2f64, f128mem, loadv2f64, - SSEPackedDouble, itins.d, 0>, TB, OpSize, VEX_4V; + SSEPackedDouble, itins.d, 0>, PD, VEX_4V; defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR256, v8f32, f256mem, loadv8f32, - SSEPackedSingle, itins.s, 0>, TB, VEX_4V, VEX_L; + SSEPackedSingle, itins.s, 0>, PS, VEX_4V, VEX_L; defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR256, v4f64, f256mem, loadv4f64, - SSEPackedDouble, itins.d, 0>, TB, OpSize, VEX_4V, VEX_L; + SSEPackedDouble, itins.d, 0>, PD, VEX_4V, VEX_L; let Constraints = "$src1 = $dst" in { defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128, v4f32, f128mem, memopv4f32, SSEPackedSingle, - itins.s>, TB; + itins.s>, PS; defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128, v2f64, f128mem, memopv2f64, SSEPackedDouble, - itins.d>, TB, OpSize; + itins.d>, PD; } } @@ -3017,6 +3079,214 @@ let isCodeGenOnly = 1 in { basic_sse12_fp_binop_s<0x5D, "min", X86fminc, SSE_ALU_ITINS_S>; } +// Patterns used to select SSE scalar fp arithmetic instructions from +// a scalar fp operation followed by a blend. +// +// These patterns know, for example, how to select an ADDSS from a +// float add plus vector insert. +// +// The effect is that the backend no longer emits unnecessary vector +// insert instructions immediately after SSE scalar fp instructions +// like addss or mulss. +// +// For example, given the following code: +// __m128 foo(__m128 A, __m128 B) { +// A[0] += B[0]; +// return A; +// } +// +// previously we generated: +// addss %xmm0, %xmm1 +// movss %xmm1, %xmm0 +// +// we now generate: +// addss %xmm1, %xmm0 + +let Predicates = [UseSSE1] in { + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), (v4f32 (scalar_to_vector (fadd + (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))))), + (ADDSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), (v4f32 (scalar_to_vector (fsub + (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))))), + (SUBSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), (v4f32 (scalar_to_vector (fmul + (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))))), + (MULSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), (v4f32 (scalar_to_vector (fdiv + (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))))), + (DIVSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; +} + +let Predicates = [UseSSE2] in { + // SSE2 patterns to select scalar double-precision fp arithmetic instructions + + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fadd + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (ADDSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fsub + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (SUBSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fmul + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (MULSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fdiv + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (DIVSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; +} + +let Predicates = [UseSSE41] in { + // If the subtarget has SSE4.1 but not AVX, the vector insert + // instruction is lowered into a X86insrtps rather than a X86Movss. + // When selecting SSE scalar single-precision fp arithmetic instructions, + // make sure that we correctly match the X86insrtps. + + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fadd (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (ADDSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fsub (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (SUBSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fmul (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (MULSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fdiv (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (DIVSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; +} + +let Predicates = [HasAVX] in { + // The following patterns select AVX Scalar single/double precision fp + // arithmetic instructions. + + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fadd + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (VADDSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fsub + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (VSUBSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fmul + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (VMULSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector (fdiv + (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), + FR64:$src))))), + (VDIVSDrr_Int v2f64:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fadd (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (VADDSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fsub (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (VSUBSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fmul (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (VMULSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; + def : Pat<(v4f32 (X86insrtps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector + (fdiv (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), + FR32:$src))), (iPTR 0))), + (VDIVSSrr_Int v4f32:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; +} + +// Patterns used to select SSE scalar fp arithmetic instructions from +// a vector packed single/double fp operation followed by a vector insert. +// +// The effect is that the backend converts the packed fp instruction +// followed by a vector insert into a single SSE scalar fp instruction. +// +// For example, given the following code: +// __m128 foo(__m128 A, __m128 B) { +// __m128 C = A + B; +// return (__m128) {c[0], a[1], a[2], a[3]}; +// } +// +// previously we generated: +// addps %xmm0, %xmm1 +// movss %xmm1, %xmm0 +// +// we now generate: +// addss %xmm1, %xmm0 + +let Predicates = [UseSSE1] in { + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fadd (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (ADDSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fsub (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (SUBSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fmul (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (MULSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fdiv (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (DIVSSrr_Int v4f32:$dst, v4f32:$src)>; +} + +let Predicates = [UseSSE2] in { + // SSE2 patterns to select scalar double-precision fp arithmetic instructions + // from a packed double-precision fp instruction plus movsd. + + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fadd (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (ADDSDrr_Int v2f64:$dst, v2f64:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fsub (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (SUBSDrr_Int v2f64:$dst, v2f64:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fmul (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (MULSDrr_Int v2f64:$dst, v2f64:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fdiv (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (DIVSDrr_Int v2f64:$dst, v2f64:$src)>; +} + +let Predicates = [HasAVX] in { + // The following patterns select AVX Scalar single/double precision fp + // arithmetic instructions from a packed single precision fp instruction + // plus movss/movsd. + + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fadd (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (VADDSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fsub (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (VSUBSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fmul (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (VMULSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), + (fdiv (v4f32 VR128:$dst), (v4f32 VR128:$src)))), + (VDIVSSrr_Int v4f32:$dst, v4f32:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fadd (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (VADDSDrr_Int v2f64:$dst, v2f64:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fsub (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (VSUBSDrr_Int v2f64:$dst, v2f64:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fmul (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (VMULSDrr_Int v2f64:$dst, v2f64:$src)>; + def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), + (fdiv (v2f64 VR128:$dst), (v2f64 VR128:$src)))), + (VDIVSDrr_Int v2f64:$dst, v2f64:$src)>; +} + /// Unop Arithmetic /// In addition, we also have a special variant of the scalar form here to /// represent the associated intrinsic operation. This form is unlike the @@ -3069,6 +3339,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, VEX_4V, VEX_LIG, Sched<[itins.Sched.Folded, ReadAfterLd]>; + let isCodeGenOnly = 1 in def V#NAME#SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2), !strconcat("v", OpcodeStr, @@ -3089,6 +3360,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"), [(set FR32:$dst, (OpNode (load addr:$src)))], itins.rm>, XS, Requires<[UseSSE1, OptForSize]>, Sched<[itins.Sched.Folded]>; +let isCodeGenOnly = 1 in { def SSr_Int : SSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (F32Int VR128:$src))], itins.rr>, @@ -3098,6 +3370,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { [(set VR128:$dst, (F32Int sse_load_f32:$src))], itins.rm>, Sched<[itins.Sched.Folded]>; } +} /// sse1_fp_unop_s_rw - SSE1 unops where vector form has a read-write operand. multiclass sse1_fp_unop_rw<bits<8> opc, string OpcodeStr, SDNode OpNode, @@ -3115,6 +3388,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, VEX_4V, VEX_LIG, Sched<[itins.Sched.Folded, ReadAfterLd]>; + let isCodeGenOnly = 1 in def V#NAME#SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2), !strconcat("v", OpcodeStr, @@ -3135,7 +3409,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"), [(set FR32:$dst, (OpNode (load addr:$src)))], itins.rm>, XS, Requires<[UseSSE1, OptForSize]>, Sched<[itins.Sched.Folded]>; - let Constraints = "$src1 = $dst" in { + let isCodeGenOnly = 1, Constraints = "$src1 = $dst" in { def SSr_Int : SSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), !strconcat(OpcodeStr, "ss\t{$src2, $dst|$dst, $src2}"), @@ -3188,6 +3462,7 @@ let Predicates = [HasAVX] in { multiclass sse1_fp_unop_p_int<bits<8> opc, string OpcodeStr, Intrinsic V4F32Int, Intrinsic V8F32Int, OpndItins itins> { +let isCodeGenOnly = 1 in { let Predicates = [HasAVX] in { def V#NAME#PSr_Int : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat("v", OpcodeStr, @@ -3220,6 +3495,7 @@ let Predicates = [HasAVX] in { !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (V4F32Int (memopv4f32 addr:$src)))], itins.rm>, Sched<[itins.Sched.Folded]>; +} // isCodeGenOnly = 1 } /// sse2_fp_unop_s - SSE2 unops in scalar form. @@ -3238,6 +3514,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, VEX_4V, VEX_LIG, Sched<[itins.Sched.Folded, ReadAfterLd]>; + let isCodeGenOnly = 1 in def V#NAME#SDm_Int : SDI<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2), !strconcat("v", OpcodeStr, @@ -3256,6 +3533,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { !strconcat(OpcodeStr, "sd\t{$src, $dst|$dst, $src}"), [(set FR64:$dst, (OpNode (load addr:$src)))], itins.rm>, XD, Requires<[UseSSE2, OptForSize]>, Sched<[itins.Sched.Folded]>; +let isCodeGenOnly = 1 in { def SDr_Int : SDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "sd\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (F64Int VR128:$src))], itins.rr>, @@ -3265,6 +3543,7 @@ let Predicates = [HasAVX], hasSideEffects = 0 in { [(set VR128:$dst, (F64Int sse_load_f64:$src))], itins.rm>, Sched<[itins.Sched.Folded]>; } +} /// sse2_fp_unop_p - SSE2 unops in vector forms. multiclass sse2_fp_unop_p<bits<8> opc, string OpcodeStr, @@ -3455,19 +3734,14 @@ def MOVNTImr : I<0xC3, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), "movnti{l}\t{$src, $dst|$dst, $src}", [(nontemporalstore (i32 GR32:$src), addr:$dst)], IIC_SSE_MOVNT>, - TB, Requires<[HasSSE2]>; + PS, Requires<[HasSSE2]>; def MOVNTI_64mr : RI<0xC3, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), "movnti{q}\t{$src, $dst|$dst, $src}", [(nontemporalstore (i64 GR64:$src), addr:$dst)], IIC_SSE_MOVNT>, - TB, Requires<[HasSSE2]>; + PS, Requires<[HasSSE2]>; } // SchedRW = [WriteStore] -def : Pat<(alignednontemporalstore (v2i64 VR128:$src), addr:$dst), - (VMOVNTDQmr addr:$dst, VR128:$src)>, Requires<[HasAVX]>; - -def : Pat<(alignednontemporalstore (v2i64 VR128:$src), addr:$dst), - (MOVNTDQmr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; } // AddedComplexity //===----------------------------------------------------------------------===// @@ -3490,17 +3764,23 @@ def PREFETCHNTA : I<0x18, MRM0m, (outs), (ins i8mem:$src), IIC_SSE_PREFETCH>, TB; } -// FIXME: How should these memory instructions be modeled? +// FIXME: How should flush instruction be modeled? let SchedRW = [WriteLoad] in { // Flush cache def CLFLUSH : I<0xAE, MRM7m, (outs), (ins i8mem:$src), "clflush\t$src", [(int_x86_sse2_clflush addr:$src)], IIC_SSE_PREFETCH>, TB, Requires<[HasSSE2]>; +} +let SchedRW = [WriteNop] in { // Pause. This "instruction" is encoded as "rep; nop", so even though it // was introduced with SSE2, it's backward compatible. -def PAUSE : I<0x90, RawFrm, (outs), (ins), "pause", [], IIC_SSE_PAUSE>, REP; +def PAUSE : I<0x90, RawFrm, (outs), (ins), + "pause", [(int_x86_sse2_pause)], IIC_SSE_PAUSE>, + OBXS, Requires<[HasSSE2]>; +} +let SchedRW = [WriteFence] in { // Load, store, and memory fence def SFENCE : I<0xAE, MRM_F8, (outs), (ins), "sfence", [(int_x86_sse_sfence)], IIC_SSE_SFENCE>, @@ -3557,7 +3837,8 @@ def VMOVDQUYrr : VSSI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), } // For Disassembler -let isCodeGenOnly = 1, hasSideEffects = 0, SchedRW = [WriteMove] in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, + SchedRW = [WriteMove] in { def VMOVDQArr_REV : VPDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>, @@ -3621,7 +3902,7 @@ def MOVDQUrr : I<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>; // For Disassembler -let isCodeGenOnly = 1, hasSideEffects = 0 in { +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { def MOVDQArr_REV : PDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>; @@ -4047,11 +4328,14 @@ defm PCMPGTD : PDI_binop_all<0x66, "pcmpgtd", X86pcmpgt, v4i32, v8i32, //===---------------------------------------------------------------------===// defm PACKSSWB : PDI_binop_all_int<0x63, "packsswb", int_x86_sse2_packsswb_128, - int_x86_avx2_packsswb, SSE_INTALU_ITINS_P, 0>; + int_x86_avx2_packsswb, + SSE_INTALU_ITINS_SHUFF_P, 0>; defm PACKSSDW : PDI_binop_all_int<0x6B, "packssdw", int_x86_sse2_packssdw_128, - int_x86_avx2_packssdw, SSE_INTALU_ITINS_P, 0>; + int_x86_avx2_packssdw, + SSE_INTALU_ITINS_SHUFF_P, 0>; defm PACKUSWB : PDI_binop_all_int<0x67, "packuswb", int_x86_sse2_packuswb_128, - int_x86_avx2_packuswb, SSE_INTALU_ITINS_P, 0>; + int_x86_avx2_packuswb, + SSE_INTALU_ITINS_SHUFF_P, 0>; //===---------------------------------------------------------------------===// // SSE2 - Packed Integer Shuffle Instructions @@ -4111,12 +4395,12 @@ let Predicates = [UseSSE2] in { [(set VR128:$dst, (vt128 (OpNode (bitconvert (memopv2i64 addr:$src1)), (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, - Sched<[WriteShuffleLd]>; + Sched<[WriteShuffleLd, ReadAfterLd]>; } } } // ExeDomain = SSEPackedInt -defm PSHUFD : sse2_pshuffle<"pshufd", v4i32, v8i32, X86PShufd>, TB, OpSize; +defm PSHUFD : sse2_pshuffle<"pshufd", v4i32, v8i32, X86PShufd>, PD; defm PSHUFHW : sse2_pshuffle<"pshufhw", v8i16, v16i16, X86PShufhw>, XS; defm PSHUFLW : sse2_pshuffle<"pshuflw", v8i16, v16i16, X86PShuflw>, XD; @@ -4269,7 +4553,7 @@ def VPEXTRWri : Ii8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst), (ins VR128:$src1, i32i8imm:$src2), "vpextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1), - imm:$src2))]>, TB, OpSize, VEX, + imm:$src2))]>, PD, VEX, Sched<[WriteShuffle]>; def PEXTRWri : PDIi8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst), (ins VR128:$src1, i32i8imm:$src2), @@ -4280,10 +4564,10 @@ def PEXTRWri : PDIi8<0xC5, MRMSrcReg, // Insert let Predicates = [HasAVX] in -defm VPINSRW : sse2_pinsrw<0>, TB, OpSize, VEX_4V; +defm VPINSRW : sse2_pinsrw<0>, PD, VEX_4V; let Predicates = [UseSSE2], Constraints = "$src1 = $dst" in -defm PINSRW : sse2_pinsrw, TB, OpSize; +defm PINSRW : sse2_pinsrw, PD; } // ExeDomain = SSEPackedInt @@ -4320,7 +4604,7 @@ def PMOVMSKBrr : PDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), (ins VR128:$src), let ExeDomain = SSEPackedInt, SchedRW = [WriteStore] in { -let Uses = [EDI], Predicates = [HasAVX,In32BitMode] in +let Uses = [EDI], Predicates = [HasAVX,Not64BitMode] in def VMASKMOVDQU : VPDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask), "maskmovdqu\t{$mask, $src|$src, $mask}", @@ -4333,7 +4617,7 @@ def VMASKMOVDQU64 : VPDI<0xF7, MRMSrcReg, (outs), [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)], IIC_SSE_MASKMOV>, VEX; -let Uses = [EDI], Predicates = [UseSSE2,In32BitMode] in +let Uses = [EDI], Predicates = [UseSSE2,Not64BitMode] in def MASKMOVDQU : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask), "maskmovdqu\t{$mask, $src|$src, $mask}", [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)], @@ -4434,7 +4718,7 @@ def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs), "movd\t{$src, $dst|$dst, $src}", [(store (i32 (vector_extract (v4i32 VR128:$src), (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, - VEX, Sched<[WriteLoad]>; + VEX, Sched<[WriteStore]>; def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), "movd\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (vector_extract (v4i32 VR128:$src), @@ -4444,7 +4728,7 @@ def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src), "movd\t{$src, $dst|$dst, $src}", [(store (i32 (vector_extract (v4i32 VR128:$src), (iPTR 0))), addr:$dst)], - IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; + IIC_SSE_MOVDQ>, Sched<[WriteStore]>; def : Pat<(v8i32 (X86Vinsert (v8i32 immAllZerosV), GR32:$src2, (iPTR 0))), (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src2), sub_xmm)>; @@ -4638,17 +4922,24 @@ def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), IIC_SSE_MOVDQ>; } // SchedRW +// For disassembler only +let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, + SchedRW = [WriteVecLogic] in { +def VMOVPQI2QIrr : VS2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), + "movq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVQ_RR>, VEX; +def MOVPQI2QIrr : S2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), + "movq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVQ_RR>; +} + //===---------------------------------------------------------------------===// // Store / copy lower 64-bits of a XMM register. // -def VMOVLQ128mr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), - "movq\t{$src, $dst|$dst, $src}", - [(int_x86_sse2_storel_dq addr:$dst, VR128:$src)]>, VEX, - Sched<[WriteStore]>; -def MOVLQ128mr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), - "movq\t{$src, $dst|$dst, $src}", - [(int_x86_sse2_storel_dq addr:$dst, VR128:$src)], - IIC_SSE_MOVDQ>, Sched<[WriteStore]>; +let Predicates = [UseAVX] in +def : Pat<(int_x86_sse2_storel_dq addr:$dst, VR128:$src), + (VMOVPQI2QImr addr:$dst, VR128:$src)>; +let Predicates = [UseSSE2] in +def : Pat<(int_x86_sse2_storel_dq addr:$dst, VR128:$src), + (MOVPQI2QImr addr:$dst, VR128:$src)>; let isCodeGenOnly = 1, AddedComplexity = 20 in { def VMOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), @@ -4745,11 +5036,11 @@ multiclass sse3_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr, def rr : S3SI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set RC:$dst, (vt (OpNode RC:$src)))], - IIC_SSE_MOV_LH>, Sched<[WriteShuffle]>; + IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; def rm : S3SI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set RC:$dst, (OpNode (mem_frag addr:$src)))], - IIC_SSE_MOV_LH>, Sched<[WriteShuffleLd]>; + IIC_SSE_MOV_LH>, Sched<[WriteLoad]>; } let Predicates = [HasAVX] in { @@ -4805,13 +5096,13 @@ multiclass sse3_replicate_dfp<string OpcodeStr> { let neverHasSideEffects = 1 in def rr : S3DI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [], IIC_SSE_MOV_LH>, Sched<[WriteShuffle]>; + [], IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; def rm : S3DI<0x12, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (v2f64 (X86Movddup (scalar_to_vector (loadf64 addr:$src)))))], - IIC_SSE_MOV_LH>, Sched<[WriteShuffleLd]>; + IIC_SSE_MOV_LH>, Sched<[WriteLoad]>; } // FIXME: Merge with above classe when there're patterns for the ymm version @@ -4819,13 +5110,13 @@ multiclass sse3_replicate_dfp_y<string OpcodeStr> { def rr : S3DI<0x12, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (v4f64 (X86Movddup VR256:$src)))]>, - Sched<[WriteShuffle]>; + Sched<[WriteFShuffle]>; def rm : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (v4f64 (X86Movddup (scalar_to_vector (loadf64 addr:$src)))))]>, - Sched<[WriteShuffleLd]>; + Sched<[WriteLoad]>; } let Predicates = [HasAVX] in { @@ -4915,24 +5206,24 @@ multiclass sse3_addsub<Intrinsic Int, string OpcodeStr, RegisterClass RC, let Predicates = [HasAVX] in { let ExeDomain = SSEPackedSingle in { defm VADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "vaddsubps", VR128, - f128mem, SSE_ALU_F32P, 0>, TB, XD, VEX_4V; + f128mem, SSE_ALU_F32P, 0>, XD, VEX_4V; defm VADDSUBPSY : sse3_addsub<int_x86_avx_addsub_ps_256, "vaddsubps", VR256, - f256mem, SSE_ALU_F32P, 0>, TB, XD, VEX_4V, VEX_L; + f256mem, SSE_ALU_F32P, 0>, XD, VEX_4V, VEX_L; } let ExeDomain = SSEPackedDouble in { defm VADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "vaddsubpd", VR128, - f128mem, SSE_ALU_F64P, 0>, TB, OpSize, VEX_4V; + f128mem, SSE_ALU_F64P, 0>, PD, VEX_4V; defm VADDSUBPDY : sse3_addsub<int_x86_avx_addsub_pd_256, "vaddsubpd", VR256, - f256mem, SSE_ALU_F64P, 0>, TB, OpSize, VEX_4V, VEX_L; + f256mem, SSE_ALU_F64P, 0>, PD, VEX_4V, VEX_L; } } let Constraints = "$src1 = $dst", Predicates = [UseSSE3] in { let ExeDomain = SSEPackedSingle in defm ADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "addsubps", VR128, - f128mem, SSE_ALU_F32P>, TB, XD; + f128mem, SSE_ALU_F32P>, XD; let ExeDomain = SSEPackedDouble in defm ADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "addsubpd", VR128, - f128mem, SSE_ALU_F64P>, TB, OpSize; + f128mem, SSE_ALU_F64P>, PD; } //===---------------------------------------------------------------------===// @@ -5019,7 +5310,7 @@ multiclass SS3I_unop_rm_int<bits<8> opc, string OpcodeStr, (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (IntId128 VR128:$src))], IIC_SSE_PABS_RR>, - OpSize, Sched<[WriteVecALU]>; + Sched<[WriteVecALU]>; def rm128 : SS38I<opc, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), @@ -5027,7 +5318,7 @@ multiclass SS3I_unop_rm_int<bits<8> opc, string OpcodeStr, [(set VR128:$dst, (IntId128 (bitconvert (memopv2i64 addr:$src))))], IIC_SSE_PABS_RM>, - OpSize, Sched<[WriteVecALULd]>; + Sched<[WriteVecALULd]>; } /// SS3I_unop_rm_int_y - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}. @@ -5037,14 +5328,14 @@ multiclass SS3I_unop_rm_int_y<bits<8> opc, string OpcodeStr, (ins VR256:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (IntId256 VR256:$src))]>, - OpSize, Sched<[WriteVecALU]>; + Sched<[WriteVecALU]>; def rm256 : SS38I<opc, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (IntId256 - (bitconvert (memopv4i64 addr:$src))))]>, OpSize, + (bitconvert (memopv4i64 addr:$src))))]>, Sched<[WriteVecALULd]>; } @@ -5164,7 +5455,7 @@ multiclass SS3I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))], itins.rr>, - OpSize, Sched<[itins.Sched]>; + Sched<[itins.Sched]>; def rm : SS38I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), !if(Is2Addr, @@ -5172,7 +5463,7 @@ multiclass SS3I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set RC:$dst, (OpVT (OpNode RC:$src1, - (bitconvert (memop_frag addr:$src2)))))], itins.rm>, OpSize, + (bitconvert (memop_frag addr:$src2)))))], itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; } @@ -5187,7 +5478,7 @@ multiclass SS3I_binop_rm_int<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>, - OpSize, Sched<[itins.Sched]>; + Sched<[itins.Sched]>; def rm128 : SS38I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), !if(Is2Addr, @@ -5195,24 +5486,25 @@ multiclass SS3I_binop_rm_int<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set VR128:$dst, (IntId128 VR128:$src1, - (bitconvert (memopv2i64 addr:$src2))))]>, OpSize, + (bitconvert (memopv2i64 addr:$src2))))]>, Sched<[itins.Sched.Folded, ReadAfterLd]>; } multiclass SS3I_binop_rm_int_y<bits<8> opc, string OpcodeStr, - Intrinsic IntId256> { + Intrinsic IntId256, + X86FoldableSchedWrite Sched> { let isCommutable = 1 in def rr256 : SS38I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, (IntId256 VR256:$src1, VR256:$src2))]>, - OpSize; + Sched<[Sched]>; def rm256 : SS38I<opc, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, - (IntId256 VR256:$src1, - (bitconvert (loadv4i64 addr:$src2))))]>, OpSize; + (IntId256 VR256:$src1, (bitconvert (loadv4i64 addr:$src2))))]>, + Sched<[Sched.Folded, ReadAfterLd]>; } let ImmT = NoImm, Predicates = [HasAVX] in { @@ -5281,16 +5573,20 @@ let isCommutable = 0 in { SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; defm VPSHUFBY : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v32i8, VR256, loadv4i64, i256mem, - SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; + SSE_PSHUFB, 0>, VEX_4V, VEX_L; defm VPHADDSW : SS3I_binop_rm_int_y<0x03, "vphaddsw", - int_x86_avx2_phadd_sw>, VEX_4V, VEX_L; + int_x86_avx2_phadd_sw, + WriteVecALU>, VEX_4V, VEX_L; defm VPHSUBSW : SS3I_binop_rm_int_y<0x07, "vphsubsw", - int_x86_avx2_phsub_sw>, VEX_4V, VEX_L; + int_x86_avx2_phsub_sw, + WriteVecALU>, VEX_4V, VEX_L; defm VPMADDUBSW : SS3I_binop_rm_int_y<0x04, "vpmaddubsw", - int_x86_avx2_pmadd_ub_sw>, VEX_4V, VEX_L; + int_x86_avx2_pmadd_ub_sw, + WriteVecIMul>, VEX_4V, VEX_L; } defm VPMULHRSW : SS3I_binop_rm_int_y<0x0B, "vpmulhrsw", - int_x86_avx2_pmul_hr_sw>, VEX_4V, VEX_L; + int_x86_avx2_pmul_hr_sw, + WriteVecIMul>, VEX_4V, VEX_L; } // None of these have i8 immediate fields. @@ -5338,7 +5634,7 @@ multiclass ssse3_palignr<string asm, bit Is2Addr = 1> { !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), !strconcat(asm, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), - [], IIC_SSE_PALIGNRR>, OpSize, Sched<[WriteShuffle]>; + [], IIC_SSE_PALIGNRR>, Sched<[WriteShuffle]>; let mayLoad = 1 in def R128rm : SS3AI<0x0F, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2, i8imm:$src3), @@ -5346,7 +5642,7 @@ multiclass ssse3_palignr<string asm, bit Is2Addr = 1> { !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), !strconcat(asm, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), - [], IIC_SSE_PALIGNRM>, OpSize, Sched<[WriteShuffleLd, ReadAfterLd]>; + [], IIC_SSE_PALIGNRM>, Sched<[WriteShuffleLd, ReadAfterLd]>; } } @@ -5356,13 +5652,13 @@ multiclass ssse3_palignr_y<string asm, bit Is2Addr = 1> { (ins VR256:$src1, VR256:$src2, i8imm:$src3), !strconcat(asm, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), - []>, OpSize, Sched<[WriteShuffle]>; + []>, Sched<[WriteShuffle]>; let mayLoad = 1 in def R256rm : SS3AI<0x0F, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2, i8imm:$src3), !strconcat(asm, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), - []>, OpSize, Sched<[WriteShuffleLd, ReadAfterLd]>; + []>, Sched<[WriteShuffleLd, ReadAfterLd]>; } } @@ -5426,11 +5722,11 @@ def MWAITrr : I<0x01, MRM_C9, (outs), (ins), "mwait", TB, Requires<[HasSSE3]>; } // SchedRW -def : InstAlias<"mwait\t{%eax, %ecx|ecx, eax}", (MWAITrr)>, Requires<[In32BitMode]>; +def : InstAlias<"mwait\t{%eax, %ecx|ecx, eax}", (MWAITrr)>, Requires<[Not64BitMode]>; def : InstAlias<"mwait\t{%rax, %rcx|rcx, rax}", (MWAITrr)>, Requires<[In64BitMode]>; def : InstAlias<"monitor\t{%eax, %ecx, %edx|edx, ecx, eax}", (MONITORrrr)>, - Requires<[In32BitMode]>; + Requires<[Not64BitMode]>; def : InstAlias<"monitor\t{%rax, %rcx, %rdx|rdx, rcx, rax}", (MONITORrrr)>, Requires<[In64BitMode]>; @@ -5442,63 +5738,82 @@ multiclass SS41I_binop_rm_int8<bits<8> opc, string OpcodeStr, Intrinsic IntId, OpndItins itins = DEFAULT_ITINS> { def rr : SS48I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (IntId VR128:$src))], itins.rr>, OpSize; + [(set VR128:$dst, (IntId VR128:$src))], itins.rr>, + Sched<[itins.Sched]>; def rm : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (IntId (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))], - itins.rm>, OpSize; + itins.rm>, Sched<[itins.Sched.Folded]>; } multiclass SS41I_binop_rm_int16_y<bits<8> opc, string OpcodeStr, - Intrinsic IntId> { + Intrinsic IntId, X86FoldableSchedWrite Sched> { def Yrr : SS48I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR256:$dst, (IntId VR128:$src))]>, OpSize; + [(set VR256:$dst, (IntId VR128:$src))]>, Sched<[Sched]>; def Yrm : SS48I<opc, MRMSrcMem, (outs VR256:$dst), (ins i128mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (IntId (load addr:$src)))]>, - OpSize; + Sched<[Sched.Folded]>; } let Predicates = [HasAVX] in { defm VPMOVSXBW : SS41I_binop_rm_int8<0x20, "vpmovsxbw", - int_x86_sse41_pmovsxbw>, VEX; + int_x86_sse41_pmovsxbw, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; defm VPMOVSXWD : SS41I_binop_rm_int8<0x23, "vpmovsxwd", - int_x86_sse41_pmovsxwd>, VEX; + int_x86_sse41_pmovsxwd, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; defm VPMOVSXDQ : SS41I_binop_rm_int8<0x25, "vpmovsxdq", - int_x86_sse41_pmovsxdq>, VEX; + int_x86_sse41_pmovsxdq, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; defm VPMOVZXBW : SS41I_binop_rm_int8<0x30, "vpmovzxbw", - int_x86_sse41_pmovzxbw>, VEX; + int_x86_sse41_pmovzxbw, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; defm VPMOVZXWD : SS41I_binop_rm_int8<0x33, "vpmovzxwd", - int_x86_sse41_pmovzxwd>, VEX; + int_x86_sse41_pmovzxwd, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; defm VPMOVZXDQ : SS41I_binop_rm_int8<0x35, "vpmovzxdq", - int_x86_sse41_pmovzxdq>, VEX; + int_x86_sse41_pmovzxdq, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; } let Predicates = [HasAVX2] in { defm VPMOVSXBW : SS41I_binop_rm_int16_y<0x20, "vpmovsxbw", - int_x86_avx2_pmovsxbw>, VEX, VEX_L; + int_x86_avx2_pmovsxbw, + WriteShuffle>, VEX, VEX_L; defm VPMOVSXWD : SS41I_binop_rm_int16_y<0x23, "vpmovsxwd", - int_x86_avx2_pmovsxwd>, VEX, VEX_L; + int_x86_avx2_pmovsxwd, + WriteShuffle>, VEX, VEX_L; defm VPMOVSXDQ : SS41I_binop_rm_int16_y<0x25, "vpmovsxdq", - int_x86_avx2_pmovsxdq>, VEX, VEX_L; + int_x86_avx2_pmovsxdq, + WriteShuffle>, VEX, VEX_L; defm VPMOVZXBW : SS41I_binop_rm_int16_y<0x30, "vpmovzxbw", - int_x86_avx2_pmovzxbw>, VEX, VEX_L; + int_x86_avx2_pmovzxbw, + WriteShuffle>, VEX, VEX_L; defm VPMOVZXWD : SS41I_binop_rm_int16_y<0x33, "vpmovzxwd", - int_x86_avx2_pmovzxwd>, VEX, VEX_L; + int_x86_avx2_pmovzxwd, + WriteShuffle>, VEX, VEX_L; defm VPMOVZXDQ : SS41I_binop_rm_int16_y<0x35, "vpmovzxdq", - int_x86_avx2_pmovzxdq>, VEX, VEX_L; -} - -defm PMOVSXBW : SS41I_binop_rm_int8<0x20, "pmovsxbw", int_x86_sse41_pmovsxbw, SSE_INTALU_ITINS_P>; -defm PMOVSXWD : SS41I_binop_rm_int8<0x23, "pmovsxwd", int_x86_sse41_pmovsxwd, SSE_INTALU_ITINS_P>; -defm PMOVSXDQ : SS41I_binop_rm_int8<0x25, "pmovsxdq", int_x86_sse41_pmovsxdq, SSE_INTALU_ITINS_P>; -defm PMOVZXBW : SS41I_binop_rm_int8<0x30, "pmovzxbw", int_x86_sse41_pmovzxbw, SSE_INTALU_ITINS_P>; -defm PMOVZXWD : SS41I_binop_rm_int8<0x33, "pmovzxwd", int_x86_sse41_pmovzxwd, SSE_INTALU_ITINS_P>; -defm PMOVZXDQ : SS41I_binop_rm_int8<0x35, "pmovzxdq", int_x86_sse41_pmovzxdq, SSE_INTALU_ITINS_P>; + int_x86_avx2_pmovzxdq, + WriteShuffle>, VEX, VEX_L; +} + +defm PMOVSXBW : SS41I_binop_rm_int8<0x20, "pmovsxbw", int_x86_sse41_pmovsxbw, + SSE_INTALU_ITINS_SHUFF_P>; +defm PMOVSXWD : SS41I_binop_rm_int8<0x23, "pmovsxwd", int_x86_sse41_pmovsxwd, + SSE_INTALU_ITINS_SHUFF_P>; +defm PMOVSXDQ : SS41I_binop_rm_int8<0x25, "pmovsxdq", int_x86_sse41_pmovsxdq, + SSE_INTALU_ITINS_SHUFF_P>; +defm PMOVZXBW : SS41I_binop_rm_int8<0x30, "pmovzxbw", int_x86_sse41_pmovzxbw, + SSE_INTALU_ITINS_SHUFF_P>; +defm PMOVZXWD : SS41I_binop_rm_int8<0x33, "pmovzxwd", int_x86_sse41_pmovzxwd, + SSE_INTALU_ITINS_SHUFF_P>; +defm PMOVZXDQ : SS41I_binop_rm_int8<0x35, "pmovzxdq", int_x86_sse41_pmovzxdq, + SSE_INTALU_ITINS_SHUFF_P>; let Predicates = [HasAVX] in { // Common patterns involving scalar load. @@ -5590,91 +5905,67 @@ let Predicates = [UseSSE41] in { (PMOVZXDQrm addr:$src)>; } -let Predicates = [HasAVX2] in { - let AddedComplexity = 15 in { - def : Pat<(v4i64 (X86vzmovly (v4i32 VR128:$src))), - (VPMOVZXDQYrr VR128:$src)>; - def : Pat<(v8i32 (X86vzmovly (v8i16 VR128:$src))), - (VPMOVZXWDYrr VR128:$src)>; - def : Pat<(v16i16 (X86vzmovly (v16i8 VR128:$src))), - (VPMOVZXBWYrr VR128:$src)>; - } - - def : Pat<(v4i64 (X86vsmovl (v4i32 VR128:$src))), (VPMOVSXDQYrr VR128:$src)>; - def : Pat<(v8i32 (X86vsmovl (v8i16 VR128:$src))), (VPMOVSXWDYrr VR128:$src)>; - def : Pat<(v16i16 (X86vsmovl (v16i8 VR128:$src))), (VPMOVSXBWYrr VR128:$src)>; -} - -let Predicates = [HasAVX] in { - def : Pat<(v2i64 (X86vsmovl (v4i32 VR128:$src))), (VPMOVSXDQrr VR128:$src)>; - def : Pat<(v4i32 (X86vsmovl (v8i16 VR128:$src))), (VPMOVSXWDrr VR128:$src)>; - def : Pat<(v8i16 (X86vsmovl (v16i8 VR128:$src))), (VPMOVSXBWrr VR128:$src)>; -} - -let Predicates = [UseSSE41] in { - def : Pat<(v2i64 (X86vsmovl (v4i32 VR128:$src))), (PMOVSXDQrr VR128:$src)>; - def : Pat<(v4i32 (X86vsmovl (v8i16 VR128:$src))), (PMOVSXWDrr VR128:$src)>; - def : Pat<(v8i16 (X86vsmovl (v16i8 VR128:$src))), (PMOVSXBWrr VR128:$src)>; -} - - multiclass SS41I_binop_rm_int4<bits<8> opc, string OpcodeStr, Intrinsic IntId, OpndItins itins = DEFAULT_ITINS> { def rr : SS48I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (IntId VR128:$src))], itins.rr>, OpSize; + [(set VR128:$dst, (IntId VR128:$src))], itins.rr>, + Sched<[itins.Sched]>; def rm : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (IntId (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))], - itins.rm>, - OpSize; + itins.rm>, Sched<[itins.Sched.Folded]>; } multiclass SS41I_binop_rm_int8_y<bits<8> opc, string OpcodeStr, - Intrinsic IntId> { + Intrinsic IntId, X86FoldableSchedWrite Sched> { def Yrr : SS48I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR256:$dst, (IntId VR128:$src))]>, OpSize; + [(set VR256:$dst, (IntId VR128:$src))]>, Sched<[Sched]>; def Yrm : SS48I<opc, MRMSrcMem, (outs VR256:$dst), (ins i32mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (IntId (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))]>, - OpSize; + Sched<[Sched.Folded]>; } let Predicates = [HasAVX] in { -defm VPMOVSXBD : SS41I_binop_rm_int4<0x21, "vpmovsxbd", int_x86_sse41_pmovsxbd>, - VEX; -defm VPMOVSXWQ : SS41I_binop_rm_int4<0x24, "vpmovsxwq", int_x86_sse41_pmovsxwq>, - VEX; -defm VPMOVZXBD : SS41I_binop_rm_int4<0x31, "vpmovzxbd", int_x86_sse41_pmovzxbd>, - VEX; -defm VPMOVZXWQ : SS41I_binop_rm_int4<0x34, "vpmovzxwq", int_x86_sse41_pmovzxwq>, - VEX; +defm VPMOVSXBD : SS41I_binop_rm_int4<0x21, "vpmovsxbd", int_x86_sse41_pmovsxbd, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; +defm VPMOVSXWQ : SS41I_binop_rm_int4<0x24, "vpmovsxwq", int_x86_sse41_pmovsxwq, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; +defm VPMOVZXBD : SS41I_binop_rm_int4<0x31, "vpmovzxbd", int_x86_sse41_pmovzxbd, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; +defm VPMOVZXWQ : SS41I_binop_rm_int4<0x34, "vpmovzxwq", int_x86_sse41_pmovzxwq, + DEFAULT_ITINS_SHUFFLESCHED>, VEX; } let Predicates = [HasAVX2] in { defm VPMOVSXBD : SS41I_binop_rm_int8_y<0x21, "vpmovsxbd", - int_x86_avx2_pmovsxbd>, VEX, VEX_L; + int_x86_avx2_pmovsxbd, WriteShuffle>, + VEX, VEX_L; defm VPMOVSXWQ : SS41I_binop_rm_int8_y<0x24, "vpmovsxwq", - int_x86_avx2_pmovsxwq>, VEX, VEX_L; + int_x86_avx2_pmovsxwq, WriteShuffle>, + VEX, VEX_L; defm VPMOVZXBD : SS41I_binop_rm_int8_y<0x31, "vpmovzxbd", - int_x86_avx2_pmovzxbd>, VEX, VEX_L; + int_x86_avx2_pmovzxbd, WriteShuffle>, + VEX, VEX_L; defm VPMOVZXWQ : SS41I_binop_rm_int8_y<0x34, "vpmovzxwq", - int_x86_avx2_pmovzxwq>, VEX, VEX_L; + int_x86_avx2_pmovzxwq, WriteShuffle>, + VEX, VEX_L; } defm PMOVSXBD : SS41I_binop_rm_int4<0x21, "pmovsxbd", int_x86_sse41_pmovsxbd, - SSE_INTALU_ITINS_P>; + SSE_INTALU_ITINS_SHUFF_P>; defm PMOVSXWQ : SS41I_binop_rm_int4<0x24, "pmovsxwq", int_x86_sse41_pmovsxwq, - SSE_INTALU_ITINS_P>; + SSE_INTALU_ITINS_SHUFF_P>; defm PMOVZXBD : SS41I_binop_rm_int4<0x31, "pmovzxbd", int_x86_sse41_pmovzxbd, - SSE_INTALU_ITINS_P>; + SSE_INTALU_ITINS_SHUFF_P>; defm PMOVZXWQ : SS41I_binop_rm_int4<0x34, "pmovzxwq", int_x86_sse41_pmovzxwq, - SSE_INTALU_ITINS_P>; + SSE_INTALU_ITINS_SHUFF_P>; let Predicates = [HasAVX] in { // Common patterns involving scalar load @@ -5703,49 +5994,49 @@ let Predicates = [UseSSE41] in { } multiclass SS41I_binop_rm_int2<bits<8> opc, string OpcodeStr, Intrinsic IntId, - OpndItins itins = DEFAULT_ITINS> { + X86FoldableSchedWrite Sched> { def rr : SS48I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (IntId VR128:$src))]>, OpSize; + [(set VR128:$dst, (IntId VR128:$src))]>, Sched<[Sched]>; // Expecting a i16 load any extended to i32 value. def rm : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins i16mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, (IntId (bitconvert (v4i32 (scalar_to_vector (loadi16_anyext addr:$src))))))]>, - OpSize; + Sched<[Sched.Folded]>; } multiclass SS41I_binop_rm_int4_y<bits<8> opc, string OpcodeStr, - Intrinsic IntId> { + Intrinsic IntId, X86FoldableSchedWrite Sched> { def Yrr : SS48I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR256:$dst, (IntId VR128:$src))]>, OpSize; + [(set VR256:$dst, (IntId VR128:$src))]>, Sched<[Sched]>; // Expecting a i16 load any extended to i32 value. def Yrm : SS48I<opc, MRMSrcMem, (outs VR256:$dst), (ins i16mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (IntId (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))]>, - OpSize; + Sched<[Sched.Folded]>; } let Predicates = [HasAVX] in { -defm VPMOVSXBQ : SS41I_binop_rm_int2<0x22, "vpmovsxbq", int_x86_sse41_pmovsxbq>, - VEX; -defm VPMOVZXBQ : SS41I_binop_rm_int2<0x32, "vpmovzxbq", int_x86_sse41_pmovzxbq>, - VEX; +defm VPMOVSXBQ : SS41I_binop_rm_int2<0x22, "vpmovsxbq", int_x86_sse41_pmovsxbq, + WriteShuffle>, VEX; +defm VPMOVZXBQ : SS41I_binop_rm_int2<0x32, "vpmovzxbq", int_x86_sse41_pmovzxbq, + WriteShuffle>, VEX; } let Predicates = [HasAVX2] in { -defm VPMOVSXBQ : SS41I_binop_rm_int4_y<0x22, "vpmovsxbq", - int_x86_avx2_pmovsxbq>, VEX, VEX_L; -defm VPMOVZXBQ : SS41I_binop_rm_int4_y<0x32, "vpmovzxbq", - int_x86_avx2_pmovzxbq>, VEX, VEX_L; +defm VPMOVSXBQ : SS41I_binop_rm_int4_y<0x22, "vpmovsxbq", int_x86_avx2_pmovsxbq, + WriteShuffle>, VEX, VEX_L; +defm VPMOVZXBQ : SS41I_binop_rm_int4_y<0x32, "vpmovzxbq", int_x86_avx2_pmovzxbq, + WriteShuffle>, VEX, VEX_L; } defm PMOVSXBQ : SS41I_binop_rm_int2<0x22, "pmovsxbq", int_x86_sse41_pmovsxbq, - SSE_INTALU_ITINS_P>; + WriteShuffle>; defm PMOVZXBQ : SS41I_binop_rm_int2<0x32, "pmovzxbq", int_x86_sse41_pmovzxbq, - SSE_INTALU_ITINS_P>; + WriteShuffle>; let Predicates = [HasAVX2] in { def : Pat<(v16i16 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBWYrr VR128:$src)>; @@ -5772,9 +6063,9 @@ let Predicates = [HasAVX2] in { def : Pat<(v4i64 (X86vsext (v8i32 VR256:$src))), (VPMOVSXDQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>; - def : Pat<(v8i32 (X86vsmovl (v8i16 (bitconvert (v2i64 (load addr:$src)))))), + def : Pat<(v8i32 (X86vsext (v8i16 (bitconvert (v2i64 (load addr:$src)))))), (VPMOVSXWDYrm addr:$src)>; - def : Pat<(v4i64 (X86vsmovl (v4i32 (bitconvert (v2i64 (load addr:$src)))))), + def : Pat<(v4i64 (X86vsext (v4i32 (bitconvert (v2i64 (load addr:$src)))))), (VPMOVSXDQYrm addr:$src)>; def : Pat<(v8i32 (X86vsext (v16i8 (bitconvert (v2i64 @@ -6003,16 +6294,15 @@ multiclass SS41I_extract8<bits<8> opc, string OpcodeStr> { "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set GR32orGR64:$dst, (X86pextrb (v16i8 VR128:$src1), imm:$src2))]>, - OpSize; - let neverHasSideEffects = 1, mayStore = 1 in + Sched<[WriteShuffle]>; + let neverHasSideEffects = 1, mayStore = 1, + SchedRW = [WriteShuffleLd, WriteRMW] in def mr : SS4AIi8<opc, MRMDestMem, (outs), (ins i8mem:$dst, VR128:$src1, i32i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, OpSize; -// FIXME: -// There's an AssertZext in the way of writing the store pattern -// (store (i8 (trunc (X86pextrb (v16i8 VR128:$src1), imm:$src2))), addr:$dst) + [(store (i8 (trunc (assertzext (X86pextrb (v16i8 VR128:$src1), + imm:$src2)))), addr:$dst)]>; } let Predicates = [HasAVX] in @@ -6023,22 +6313,21 @@ defm PEXTRB : SS41I_extract8<0x14, "pextrb">; /// SS41I_extract16 - SSE 4.1 extract 16 bits to memory destination multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> { - let isCodeGenOnly = 1, hasSideEffects = 0 in + let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rr_REV : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), (ins VR128:$src1, i32i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, OpSize; + []>, Sched<[WriteShuffle]>; - let neverHasSideEffects = 1, mayStore = 1 in + let neverHasSideEffects = 1, mayStore = 1, + SchedRW = [WriteShuffleLd, WriteRMW] in def mr : SS4AIi8<opc, MRMDestMem, (outs), (ins i16mem:$dst, VR128:$src1, i32i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - []>, OpSize; -// FIXME: -// There's an AssertZext in the way of writing the store pattern -// (store (i16 (trunc (X86pextrw (v16i8 VR128:$src1), imm:$src2))), addr:$dst) + [(store (i16 (trunc (assertzext (X86pextrw (v8i16 VR128:$src1), + imm:$src2)))), addr:$dst)]>; } let Predicates = [HasAVX] in @@ -6054,13 +6343,15 @@ multiclass SS41I_extract32<bits<8> opc, string OpcodeStr> { !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set GR32:$dst, - (extractelt (v4i32 VR128:$src1), imm:$src2))]>, OpSize; + (extractelt (v4i32 VR128:$src1), imm:$src2))]>, + Sched<[WriteShuffle]>; + let SchedRW = [WriteShuffleLd, WriteRMW] in def mr : SS4AIi8<opc, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src1, i32i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(store (extractelt (v4i32 VR128:$src1), imm:$src2), - addr:$dst)]>, OpSize; + addr:$dst)]>; } let Predicates = [HasAVX] in @@ -6075,13 +6366,15 @@ multiclass SS41I_extract64<bits<8> opc, string OpcodeStr> { !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set GR64:$dst, - (extractelt (v2i64 VR128:$src1), imm:$src2))]>, OpSize, REX_W; + (extractelt (v2i64 VR128:$src1), imm:$src2))]>, + Sched<[WriteShuffle]>, REX_W; + let SchedRW = [WriteShuffleLd, WriteRMW] in def mr : SS4AIi8<opc, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src1, i32i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(store (extractelt (v2i64 VR128:$src1), imm:$src2), - addr:$dst)]>, OpSize, REX_W; + addr:$dst)]>, REX_W; } let Predicates = [HasAVX] in @@ -6099,14 +6392,14 @@ multiclass SS41I_extractf32<bits<8> opc, string OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set GR32orGR64:$dst, (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2))], - itins.rr>, - OpSize; + itins.rr>, Sched<[WriteFBlend]>; + let SchedRW = [WriteFBlendLd, WriteRMW] in def mr : SS4AIi8<opc, MRMDestMem, (outs), (ins f32mem:$dst, VR128:$src1, i32i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(store (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2), - addr:$dst)], itins.rm>, OpSize; + addr:$dst)], itins.rm>; } let ExeDomain = SSEPackedSingle in { @@ -6139,7 +6432,8 @@ multiclass SS41I_insert8<bits<8> opc, string asm, bit Is2Addr = 1> { !strconcat(asm, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, - (X86pinsrb VR128:$src1, GR32orGR64:$src2, imm:$src3))]>, OpSize; + (X86pinsrb VR128:$src1, GR32orGR64:$src2, imm:$src3))]>, + Sched<[WriteShuffle]>; def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i8mem:$src2, i32i8imm:$src3), !if(Is2Addr, @@ -6148,7 +6442,7 @@ multiclass SS41I_insert8<bits<8> opc, string asm, bit Is2Addr = 1> { "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (X86pinsrb VR128:$src1, (extloadi8 addr:$src2), - imm:$src3))]>, OpSize; + imm:$src3))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; } let Predicates = [HasAVX] in @@ -6165,7 +6459,7 @@ multiclass SS41I_insert32<bits<8> opc, string asm, bit Is2Addr = 1> { "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (v4i32 (insertelt VR128:$src1, GR32:$src2, imm:$src3)))]>, - OpSize; + Sched<[WriteShuffle]>; def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i32mem:$src2, i32i8imm:$src3), !if(Is2Addr, @@ -6174,7 +6468,7 @@ multiclass SS41I_insert32<bits<8> opc, string asm, bit Is2Addr = 1> { "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (v4i32 (insertelt VR128:$src1, (loadi32 addr:$src2), - imm:$src3)))]>, OpSize; + imm:$src3)))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; } let Predicates = [HasAVX] in @@ -6191,7 +6485,7 @@ multiclass SS41I_insert64<bits<8> opc, string asm, bit Is2Addr = 1> { "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (v2i64 (insertelt VR128:$src1, GR64:$src2, imm:$src3)))]>, - OpSize; + Sched<[WriteShuffle]>; def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i64mem:$src2, i32i8imm:$src3), !if(Is2Addr, @@ -6200,7 +6494,7 @@ multiclass SS41I_insert64<bits<8> opc, string asm, bit Is2Addr = 1> { "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (v2i64 (insertelt VR128:$src1, (loadi64 addr:$src2), - imm:$src3)))]>, OpSize; + imm:$src3)))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; } let Predicates = [HasAVX] in @@ -6222,7 +6516,7 @@ multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (X86insrtps VR128:$src1, VR128:$src2, imm:$src3))], itins.rr>, - OpSize; + Sched<[WriteFShuffle]>; def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, f32mem:$src2, u32u8imm:$src3), !if(Is2Addr, @@ -6232,7 +6526,8 @@ multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1, [(set VR128:$dst, (X86insrtps VR128:$src1, (v4f32 (scalar_to_vector (loadf32 addr:$src2))), - imm:$src3))], itins.rm>, OpSize; + imm:$src3))], itins.rm>, + Sched<[WriteFShuffleLd, ReadAfterLd]>; } let ExeDomain = SSEPackedSingle in { @@ -6258,8 +6553,7 @@ let ExeDomain = SSEPackedSingle in { !strconcat(OpcodeStr, "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (V4F32Int RC:$src1, imm:$src2))], - IIC_SSE_ROUNDPS_REG>, - OpSize; + IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAdd]>; // Vector intrinsic operation, mem def PSm : SS4AIi8<opcps, MRMSrcMem, @@ -6268,8 +6562,7 @@ let ExeDomain = SSEPackedSingle in { "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (V4F32Int (mem_frag32 addr:$src1),imm:$src2))], - IIC_SSE_ROUNDPS_MEM>, - OpSize; + IIC_SSE_ROUNDPS_MEM>, Sched<[WriteFAddLd]>; } // ExeDomain = SSEPackedSingle let ExeDomain = SSEPackedDouble in { @@ -6279,8 +6572,7 @@ let ExeDomain = SSEPackedDouble in { !strconcat(OpcodeStr, "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (V2F64Int RC:$src1, imm:$src2))], - IIC_SSE_ROUNDPS_REG>, - OpSize; + IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAdd]>; // Vector intrinsic operation, mem def PDm : SS4AIi8<opcpd, MRMSrcMem, @@ -6289,8 +6581,7 @@ let ExeDomain = SSEPackedDouble in { "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (V2F64Int (mem_frag64 addr:$src1),imm:$src2))], - IIC_SSE_ROUNDPS_REG>, - OpSize; + IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAddLd]>; } // ExeDomain = SSEPackedDouble } @@ -6308,9 +6599,10 @@ let ExeDomain = GenericDomain in { "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), !strconcat(OpcodeStr, "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), - []>, OpSize; + []>, Sched<[WriteFAdd]>; // Intrinsic operation, reg. + let isCodeGenOnly = 1 in def SSr_Int : SS4AIi8<opcss, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32i8imm:$src3), !if(Is2Addr, @@ -6319,7 +6611,7 @@ let ExeDomain = GenericDomain in { !strconcat(OpcodeStr, "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (F32Int VR128:$src1, VR128:$src2, imm:$src3))]>, - OpSize; + Sched<[WriteFAdd]>; // Intrinsic operation, mem. def SSm : SS4AIi8<opcss, MRMSrcMem, @@ -6331,7 +6623,7 @@ let ExeDomain = GenericDomain in { "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (F32Int VR128:$src1, sse_load_f32:$src2, imm:$src3))]>, - OpSize; + Sched<[WriteFAddLd, ReadAfterLd]>; // Operation, reg. let hasSideEffects = 0 in @@ -6342,9 +6634,10 @@ let ExeDomain = GenericDomain in { "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), !strconcat(OpcodeStr, "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), - []>, OpSize; + []>, Sched<[WriteFAdd]>; // Intrinsic operation, reg. + let isCodeGenOnly = 1 in def SDr_Int : SS4AIi8<opcsd, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32i8imm:$src3), !if(Is2Addr, @@ -6353,7 +6646,7 @@ let ExeDomain = GenericDomain in { !strconcat(OpcodeStr, "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (F64Int VR128:$src1, VR128:$src2, imm:$src3))]>, - OpSize; + Sched<[WriteFAdd]>; // Intrinsic operation, mem. def SDm : SS4AIi8<opcsd, MRMSrcMem, @@ -6365,7 +6658,7 @@ let ExeDomain = GenericDomain in { "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set VR128:$dst, (F64Int VR128:$src1, sse_load_f64:$src2, imm:$src3))]>, - OpSize; + Sched<[WriteFAddLd, ReadAfterLd]>; } // ExeDomain = GenericDomain } @@ -6512,31 +6805,31 @@ let Defs = [EFLAGS], Predicates = [HasAVX] in { def VPTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2), "vptest\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>, - OpSize, VEX; + Sched<[WriteVecLogic]>, VEX; def VPTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2), "vptest\t{$src2, $src1|$src1, $src2}", [(set EFLAGS,(X86ptest VR128:$src1, (loadv2i64 addr:$src2)))]>, - OpSize, VEX; + Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX; def VPTESTYrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR256:$src1, VR256:$src2), "vptest\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86ptest VR256:$src1, (v4i64 VR256:$src2)))]>, - OpSize, VEX, VEX_L; + Sched<[WriteVecLogic]>, VEX, VEX_L; def VPTESTYrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR256:$src1, i256mem:$src2), "vptest\t{$src2, $src1|$src1, $src2}", [(set EFLAGS,(X86ptest VR256:$src1, (loadv4i64 addr:$src2)))]>, - OpSize, VEX, VEX_L; + Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX, VEX_L; } let Defs = [EFLAGS] in { def PTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2), "ptest\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>, - OpSize; + Sched<[WriteVecLogic]>; def PTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2), "ptest\t{$src2, $src1|$src1, $src2}", [(set EFLAGS, (X86ptest VR128:$src1, (memopv2i64 addr:$src2)))]>, - OpSize; + Sched<[WriteVecLogicLd, ReadAfterLd]>; } // The bit test instructions below are AVX only @@ -6544,11 +6837,12 @@ multiclass avx_bittest<bits<8> opc, string OpcodeStr, RegisterClass RC, X86MemOperand x86memop, PatFrag mem_frag, ValueType vt> { def rr : SS48I<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), - [(set EFLAGS, (X86testp RC:$src1, (vt RC:$src2)))]>, OpSize, VEX; + [(set EFLAGS, (X86testp RC:$src1, (vt RC:$src2)))]>, + Sched<[WriteVecLogic]>, VEX; def rm : SS48I<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), [(set EFLAGS, (X86testp RC:$src1, (mem_frag addr:$src2)))]>, - OpSize, VEX; + Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX; } let Defs = [EFLAGS], Predicates = [HasAVX] in { @@ -6572,56 +6866,65 @@ let Defs = [EFLAGS], Predicates = [HasPOPCNT] in { def POPCNT16rr : I<0xB8, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), "popcnt{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (ctpop GR16:$src)), (implicit EFLAGS)], - IIC_SSE_POPCNT_RR>, - OpSize, XS; + IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, + OpSize16, XS; def POPCNT16rm : I<0xB8, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), "popcnt{w}\t{$src, $dst|$dst, $src}", [(set GR16:$dst, (ctpop (loadi16 addr:$src))), - (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, OpSize, XS; + (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, + Sched<[WriteFAddLd]>, OpSize16, XS; def POPCNT32rr : I<0xB8, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), "popcnt{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (ctpop GR32:$src)), (implicit EFLAGS)], - IIC_SSE_POPCNT_RR>, - XS; + IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, + OpSize32, XS; + def POPCNT32rm : I<0xB8, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), "popcnt{l}\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (ctpop (loadi32 addr:$src))), - (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, XS; + (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, + Sched<[WriteFAddLd]>, OpSize32, XS; def POPCNT64rr : RI<0xB8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), "popcnt{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (ctpop GR64:$src)), (implicit EFLAGS)], - IIC_SSE_POPCNT_RR>, - XS; + IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, XS; def POPCNT64rm : RI<0xB8, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "popcnt{q}\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (ctpop (loadi64 addr:$src))), - (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, XS; + (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, + Sched<[WriteFAddLd]>, XS; } // SS41I_unop_rm_int_v16 - SSE 4.1 unary operator whose type is v8i16. multiclass SS41I_unop_rm_int_v16<bits<8> opc, string OpcodeStr, - Intrinsic IntId128> { + Intrinsic IntId128, + X86FoldableSchedWrite Sched> { def rr128 : SS48I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (IntId128 VR128:$src))]>, OpSize; + [(set VR128:$dst, (IntId128 VR128:$src))]>, + Sched<[Sched]>; def rm128 : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, - (IntId128 - (bitconvert (memopv2i64 addr:$src))))]>, OpSize; + (IntId128 (bitconvert (memopv2i64 addr:$src))))]>, + Sched<[Sched.Folded]>; } +// PHMIN has the same profile as PSAD, thus we use the same scheduling +// model, although the naming is misleading. let Predicates = [HasAVX] in defm VPHMINPOSUW : SS41I_unop_rm_int_v16 <0x41, "vphminposuw", - int_x86_sse41_phminposuw>, VEX; + int_x86_sse41_phminposuw, + WriteVecIMul>, VEX; defm PHMINPOSUW : SS41I_unop_rm_int_v16 <0x41, "phminposuw", - int_x86_sse41_phminposuw>; + int_x86_sse41_phminposuw, + WriteVecIMul>; /// SS41I_binop_rm_int - Simple SSE 4.1 binary operator multiclass SS41I_binop_rm_int<bits<8> opc, string OpcodeStr, @@ -6634,32 +6937,33 @@ multiclass SS41I_binop_rm_int<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))], - itins.rr>, OpSize; + itins.rr>, Sched<[itins.Sched]>; def rm : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), !if(Is2Addr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set VR128:$dst, - (IntId128 VR128:$src1, - (bitconvert (memopv2i64 addr:$src2))))], - itins.rm>, OpSize; + (IntId128 VR128:$src1, (bitconvert (memopv2i64 addr:$src2))))], + itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; } /// SS41I_binop_rm_int_y - Simple SSE 4.1 binary operator multiclass SS41I_binop_rm_int_y<bits<8> opc, string OpcodeStr, - Intrinsic IntId256> { + Intrinsic IntId256, + X86FoldableSchedWrite Sched> { let isCommutable = 1 in def Yrr : SS48I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set VR256:$dst, (IntId256 VR256:$src1, VR256:$src2))]>, OpSize; + [(set VR256:$dst, (IntId256 VR256:$src1, VR256:$src2))]>, + Sched<[Sched]>; def Yrm : SS48I<opc, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, - (IntId256 VR256:$src1, - (bitconvert (loadv4i64 addr:$src2))))]>, OpSize; + (IntId256 VR256:$src1, (bitconvert (loadv4i64 addr:$src2))))]>, + Sched<[Sched.Folded, ReadAfterLd]>; } @@ -6667,75 +6971,95 @@ multiclass SS41I_binop_rm_int_y<bits<8> opc, string OpcodeStr, multiclass SS48I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, ValueType OpVT, RegisterClass RC, PatFrag memop_frag, X86MemOperand x86memop, bit Is2Addr = 1, - OpndItins itins = DEFAULT_ITINS> { + OpndItins itins = SSE_INTALU_ITINS_P> { let isCommutable = 1 in def rr : SS48I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), !if(Is2Addr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), - [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>, OpSize; + [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>, + Sched<[itins.Sched]>; def rm : SS48I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), !if(Is2Addr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set RC:$dst, - (OpVT (OpNode RC:$src1, - (bitconvert (memop_frag addr:$src2)))))]>, OpSize; + (OpVT (OpNode RC:$src1, (bitconvert (memop_frag addr:$src2)))))]>, + Sched<[itins.Sched.Folded, ReadAfterLd]>; } let Predicates = [HasAVX] in { let isCommutable = 0 in defm VPACKUSDW : SS41I_binop_rm_int<0x2B, "vpackusdw", int_x86_sse41_packusdw, - 0>, VEX_4V; + 0, DEFAULT_ITINS_SHUFFLESCHED>, VEX_4V; defm VPMINSB : SS48I_binop_rm<0x38, "vpminsb", X86smin, v16i8, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMINSD : SS48I_binop_rm<0x39, "vpminsd", X86smin, v4i32, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMINUD : SS48I_binop_rm<0x3B, "vpminud", X86umin, v4i32, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMINUW : SS48I_binop_rm<0x3A, "vpminuw", X86umin, v8i16, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMAXSB : SS48I_binop_rm<0x3C, "vpmaxsb", X86smax, v16i8, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMAXSD : SS48I_binop_rm<0x3D, "vpmaxsd", X86smax, v4i32, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMAXUD : SS48I_binop_rm<0x3F, "vpmaxud", X86umax, v4i32, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMAXUW : SS48I_binop_rm<0x3E, "vpmaxuw", X86umax, v8i16, VR128, - loadv2i64, i128mem, 0>, VEX_4V; + loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPMULDQ : SS41I_binop_rm_int<0x28, "vpmuldq", int_x86_sse41_pmuldq, - 0>, VEX_4V; + 0, DEFAULT_ITINS_VECIMULSCHED>, VEX_4V; } let Predicates = [HasAVX2] in { let isCommutable = 0 in defm VPACKUSDW : SS41I_binop_rm_int_y<0x2B, "vpackusdw", - int_x86_avx2_packusdw>, VEX_4V, VEX_L; + int_x86_avx2_packusdw, WriteShuffle>, + VEX_4V, VEX_L; defm VPMINSBY : SS48I_binop_rm<0x38, "vpminsb", X86smin, v32i8, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMINSDY : SS48I_binop_rm<0x39, "vpminsd", X86smin, v8i32, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMINUDY : SS48I_binop_rm<0x3B, "vpminud", X86umin, v8i32, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMINUWY : SS48I_binop_rm<0x3A, "vpminuw", X86umin, v16i16, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMAXSBY : SS48I_binop_rm<0x3C, "vpmaxsb", X86smax, v32i8, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMAXSDY : SS48I_binop_rm<0x3D, "vpmaxsd", X86smax, v8i32, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMAXUDY : SS48I_binop_rm<0x3F, "vpmaxud", X86umax, v8i32, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMAXUWY : SS48I_binop_rm<0x3E, "vpmaxuw", X86umax, v16i16, VR256, - loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPMULDQ : SS41I_binop_rm_int_y<0x28, "vpmuldq", - int_x86_avx2_pmul_dq>, VEX_4V, VEX_L; + int_x86_avx2_pmul_dq, WriteVecIMul>, + VEX_4V, VEX_L; } let Constraints = "$src1 = $dst" in { let isCommutable = 0 in - defm PACKUSDW : SS41I_binop_rm_int<0x2B, "packusdw", int_x86_sse41_packusdw>; + defm PACKUSDW : SS41I_binop_rm_int<0x2B, "packusdw", int_x86_sse41_packusdw, + 1, DEFAULT_ITINS_SHUFFLESCHED>; defm PMINSB : SS48I_binop_rm<0x38, "pminsb", X86smin, v16i8, VR128, memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; defm PMINSD : SS48I_binop_rm<0x39, "pminsd", X86smin, v4i32, VR128, @@ -6758,15 +7082,19 @@ let Constraints = "$src1 = $dst" in { let Predicates = [HasAVX] in { defm VPMULLD : SS48I_binop_rm<0x40, "vpmulld", mul, v4i32, VR128, - memopv2i64, i128mem, 0>, VEX_4V; + memopv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; defm VPCMPEQQ : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v2i64, VR128, - memopv2i64, i128mem, 0>, VEX_4V; + memopv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V; } let Predicates = [HasAVX2] in { defm VPMULLDY : SS48I_binop_rm<0x40, "vpmulld", mul, v8i32, VR256, - memopv4i64, i256mem, 0>, VEX_4V, VEX_L; + memopv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; defm VPCMPEQQY : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v4i64, VR256, - memopv4i64, i256mem, 0>, VEX_4V, VEX_L; + memopv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, + VEX_4V, VEX_L; } let Constraints = "$src1 = $dst" in { @@ -6790,7 +7118,7 @@ multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), [(set RC:$dst, (IntId RC:$src1, RC:$src2, imm:$src3))], itins.rr>, - OpSize; + Sched<[itins.Sched]>; def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u32u8imm:$src3), !if(Is2Addr, @@ -6801,47 +7129,58 @@ multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr, [(set RC:$dst, (IntId RC:$src1, (bitconvert (memop_frag addr:$src2)), imm:$src3))], itins.rm>, - OpSize; + Sched<[itins.Sched.Folded, ReadAfterLd]>; } let Predicates = [HasAVX] in { let isCommutable = 0 in { let ExeDomain = SSEPackedSingle in { defm VBLENDPS : SS41I_binop_rmi_int<0x0C, "vblendps", int_x86_sse41_blendps, - VR128, loadv4f32, f128mem, 0>, VEX_4V; + VR128, loadv4f32, f128mem, 0, + DEFAULT_ITINS_FBLENDSCHED>, VEX_4V; defm VBLENDPSY : SS41I_binop_rmi_int<0x0C, "vblendps", int_x86_avx_blend_ps_256, VR256, loadv8f32, - f256mem, 0>, VEX_4V, VEX_L; + f256mem, 0, DEFAULT_ITINS_FBLENDSCHED>, + VEX_4V, VEX_L; } let ExeDomain = SSEPackedDouble in { defm VBLENDPD : SS41I_binop_rmi_int<0x0D, "vblendpd", int_x86_sse41_blendpd, - VR128, loadv2f64, f128mem, 0>, VEX_4V; + VR128, loadv2f64, f128mem, 0, + DEFAULT_ITINS_FBLENDSCHED>, VEX_4V; defm VBLENDPDY : SS41I_binop_rmi_int<0x0D, "vblendpd", int_x86_avx_blend_pd_256,VR256, loadv4f64, - f256mem, 0>, VEX_4V, VEX_L; + f256mem, 0, DEFAULT_ITINS_FBLENDSCHED>, + VEX_4V, VEX_L; } defm VPBLENDW : SS41I_binop_rmi_int<0x0E, "vpblendw", int_x86_sse41_pblendw, - VR128, loadv2i64, i128mem, 0>, VEX_4V; + VR128, loadv2i64, i128mem, 0, + DEFAULT_ITINS_BLENDSCHED>, VEX_4V; defm VMPSADBW : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_sse41_mpsadbw, - VR128, loadv2i64, i128mem, 0>, VEX_4V; + VR128, loadv2i64, i128mem, 0, + DEFAULT_ITINS_MPSADSCHED>, VEX_4V; } let ExeDomain = SSEPackedSingle in defm VDPPS : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_sse41_dpps, - VR128, loadv4f32, f128mem, 0>, VEX_4V; + VR128, loadv4f32, f128mem, 0, + SSE_DPPS_ITINS>, VEX_4V; let ExeDomain = SSEPackedDouble in defm VDPPD : SS41I_binop_rmi_int<0x41, "vdppd", int_x86_sse41_dppd, - VR128, loadv2f64, f128mem, 0>, VEX_4V; + VR128, loadv2f64, f128mem, 0, + SSE_DPPS_ITINS>, VEX_4V; let ExeDomain = SSEPackedSingle in defm VDPPSY : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_avx_dp_ps_256, - VR256, loadv8f32, i256mem, 0>, VEX_4V, VEX_L; + VR256, loadv8f32, i256mem, 0, + SSE_DPPS_ITINS>, VEX_4V, VEX_L; } let Predicates = [HasAVX2] in { let isCommutable = 0 in { defm VPBLENDWY : SS41I_binop_rmi_int<0x0E, "vpblendw", int_x86_avx2_pblendw, - VR256, loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + VR256, loadv4i64, i256mem, 0, + DEFAULT_ITINS_BLENDSCHED>, VEX_4V, VEX_L; defm VMPSADBWY : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_avx2_mpsadbw, - VR256, loadv4i64, i256mem, 0>, VEX_4V, VEX_L; + VR256, loadv4i64, i256mem, 0, + DEFAULT_ITINS_MPSADSCHED>, VEX_4V, VEX_L; } } @@ -6850,17 +7189,17 @@ let Constraints = "$src1 = $dst" in { let ExeDomain = SSEPackedSingle in defm BLENDPS : SS41I_binop_rmi_int<0x0C, "blendps", int_x86_sse41_blendps, VR128, memopv4f32, f128mem, - 1, SSE_INTALU_ITINS_P>; + 1, SSE_INTALU_ITINS_FBLEND_P>; let ExeDomain = SSEPackedDouble in defm BLENDPD : SS41I_binop_rmi_int<0x0D, "blendpd", int_x86_sse41_blendpd, VR128, memopv2f64, f128mem, - 1, SSE_INTALU_ITINS_P>; + 1, SSE_INTALU_ITINS_FBLEND_P>; defm PBLENDW : SS41I_binop_rmi_int<0x0E, "pblendw", int_x86_sse41_pblendw, VR128, memopv2i64, i128mem, - 1, SSE_INTALU_ITINS_P>; + 1, SSE_INTALU_ITINS_FBLEND_P>; defm MPSADBW : SS41I_binop_rmi_int<0x42, "mpsadbw", int_x86_sse41_mpsadbw, VR128, memopv2i64, i128mem, - 1, SSE_INTMUL_ITINS_P>; + 1, SSE_MPSADBW_ITINS>; } let ExeDomain = SSEPackedSingle in defm DPPS : SS41I_binop_rmi_int<0x40, "dpps", int_x86_sse41_dpps, @@ -6875,13 +7214,15 @@ let Constraints = "$src1 = $dst" in { /// SS41I_quaternary_int_avx - AVX SSE 4.1 with 4 operators multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr, RegisterClass RC, X86MemOperand x86memop, - PatFrag mem_frag, Intrinsic IntId> { + PatFrag mem_frag, Intrinsic IntId, + X86FoldableSchedWrite Sched> { def rr : Ii8<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2, RC:$src3), !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set RC:$dst, (IntId RC:$src1, RC:$src2, RC:$src3))], - NoItinerary, SSEPackedInt>, OpSize, TA, VEX_4V, VEX_I8IMM; + NoItinerary, SSEPackedInt>, TAPD, VEX_4V, VEX_I8IMM, + Sched<[Sched]>; def rm : Ii8<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2, RC:$src3), @@ -6890,29 +7231,36 @@ multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr, [(set RC:$dst, (IntId RC:$src1, (bitconvert (mem_frag addr:$src2)), RC:$src3))], - NoItinerary, SSEPackedInt>, OpSize, TA, VEX_4V, VEX_I8IMM; + NoItinerary, SSEPackedInt>, TAPD, VEX_4V, VEX_I8IMM, + Sched<[Sched.Folded, ReadAfterLd]>; } let Predicates = [HasAVX] in { let ExeDomain = SSEPackedDouble in { defm VBLENDVPD : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR128, f128mem, - loadv2f64, int_x86_sse41_blendvpd>; + loadv2f64, int_x86_sse41_blendvpd, + WriteFVarBlend>; defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, f256mem, - loadv4f64, int_x86_avx_blendv_pd_256>, VEX_L; + loadv4f64, int_x86_avx_blendv_pd_256, + WriteFVarBlend>, VEX_L; } // ExeDomain = SSEPackedDouble let ExeDomain = SSEPackedSingle in { defm VBLENDVPS : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR128, f128mem, - loadv4f32, int_x86_sse41_blendvps>; + loadv4f32, int_x86_sse41_blendvps, + WriteFVarBlend>; defm VBLENDVPSY : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR256, f256mem, - loadv8f32, int_x86_avx_blendv_ps_256>, VEX_L; + loadv8f32, int_x86_avx_blendv_ps_256, + WriteFVarBlend>, VEX_L; } // ExeDomain = SSEPackedSingle defm VPBLENDVB : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR128, i128mem, - loadv2i64, int_x86_sse41_pblendvb>; + loadv2i64, int_x86_sse41_pblendvb, + WriteVarBlend>; } let Predicates = [HasAVX2] in { defm VPBLENDVBY : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR256, i256mem, - loadv4i64, int_x86_avx2_pblendvb>, VEX_L; + loadv4i64, int_x86_avx2_pblendvb, + WriteVarBlend>, VEX_L; } let Predicates = [HasAVX] in { @@ -6981,7 +7329,7 @@ let Uses = [XMM0], Constraints = "$src1 = $dst" in { !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), [(set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0))], - itins.rr>, OpSize; + itins.rr>; def rm0 : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, x86memop:$src2), @@ -6990,7 +7338,7 @@ let Uses = [XMM0], Constraints = "$src1 = $dst" in { [(set VR128:$dst, (IntId VR128:$src1, (bitconvert (mem_frag addr:$src2)), XMM0))], - itins.rm>, OpSize; + itins.rm>; } } @@ -7050,16 +7398,15 @@ let Predicates = [HasAVX] in def VMOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>, - OpSize, VEX; + VEX; let Predicates = [HasAVX2] in def VMOVNTDQAYrm : SS48I<0x2A, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", [(set VR256:$dst, (int_x86_avx2_movntdqa addr:$src))]>, - OpSize, VEX, VEX_L; + VEX, VEX_L; def MOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), "movntdqa\t{$src, $dst|$dst, $src}", - [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>, - OpSize; + [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>; //===----------------------------------------------------------------------===// // SSE4.2 - Compare Instructions @@ -7074,15 +7421,14 @@ multiclass SS42I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, !if(Is2Addr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), - [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>, - OpSize; + [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>; def rm : SS428I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), !if(Is2Addr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set RC:$dst, - (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>, OpSize; + (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>; } let Predicates = [HasAVX] in @@ -7122,12 +7468,12 @@ multiclass pcmpistrm_SS42AI<string asm> { def rr : SS42AI<0x62, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2, i8imm:$src3), !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), - []>, OpSize; + []>, Sched<[WritePCmpIStrM]>; let mayLoad = 1 in def rm :SS42AI<0x62, MRMSrcMem, (outs), (ins VR128:$src1, i128mem:$src2, i8imm:$src3), !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), - []>, OpSize; + []>, Sched<[WritePCmpIStrMLd, ReadAfterLd]>; } let Defs = [XMM0, EFLAGS], neverHasSideEffects = 1 in { @@ -7157,12 +7503,12 @@ multiclass SS42AI_pcmpestrm<string asm> { def rr : SS42AI<0x60, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src3, i8imm:$src5), !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), - []>, OpSize; + []>, Sched<[WritePCmpEStrM]>; let mayLoad = 1 in def rm : SS42AI<0x60, MRMSrcMem, (outs), (ins VR128:$src1, i128mem:$src3, i8imm:$src5), !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), - []>, OpSize; + []>, Sched<[WritePCmpEStrMLd, ReadAfterLd]>; } let Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], neverHasSideEffects = 1 in { @@ -7192,12 +7538,12 @@ multiclass SS42AI_pcmpistri<string asm> { def rr : SS42AI<0x63, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2, i8imm:$src3), !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), - []>, OpSize; + []>, Sched<[WritePCmpIStrI]>; let mayLoad = 1 in def rm : SS42AI<0x63, MRMSrcMem, (outs), (ins VR128:$src1, i128mem:$src2, i8imm:$src3), !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), - []>, OpSize; + []>, Sched<[WritePCmpIStrILd, ReadAfterLd]>; } let Defs = [ECX, EFLAGS], neverHasSideEffects = 1 in { @@ -7228,12 +7574,12 @@ multiclass SS42AI_pcmpestri<string asm> { def rr : SS42AI<0x61, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src3, i8imm:$src5), !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), - []>, OpSize; + []>, Sched<[WritePCmpEStrI]>; let mayLoad = 1 in def rm : SS42AI<0x61, MRMSrcMem, (outs), (ins VR128:$src1, i128mem:$src3, i8imm:$src5), !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), - []>, OpSize; + []>, Sched<[WritePCmpEStrILd, ReadAfterLd]>; } let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], neverHasSideEffects = 1 in { @@ -7255,14 +7601,15 @@ class SS42I_crc32r<bits<8> opc, string asm, RegisterClass RCOut, RegisterClass RCIn, SDPatternOperator Int> : SS42FI<opc, MRMSrcReg, (outs RCOut:$dst), (ins RCOut:$src1, RCIn:$src2), !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), - [(set RCOut:$dst, (Int RCOut:$src1, RCIn:$src2))], IIC_CRC32_REG>; + [(set RCOut:$dst, (Int RCOut:$src1, RCIn:$src2))], IIC_CRC32_REG>, + Sched<[WriteFAdd]>; class SS42I_crc32m<bits<8> opc, string asm, RegisterClass RCOut, X86MemOperand x86memop, SDPatternOperator Int> : SS42FI<opc, MRMSrcMem, (outs RCOut:$dst), (ins RCOut:$src1, x86memop:$src2), !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), [(set RCOut:$dst, (Int RCOut:$src1, (load addr:$src2)))], - IIC_CRC32_MEM>; + IIC_CRC32_MEM>, Sched<[WriteFAddLd, ReadAfterLd]>; let Constraints = "$src1 = $dst" in { def CRC32r32m8 : SS42I_crc32m<0xF0, "crc32{b}", GR32, i8mem, @@ -7270,13 +7617,13 @@ let Constraints = "$src1 = $dst" in { def CRC32r32r8 : SS42I_crc32r<0xF0, "crc32{b}", GR32, GR8, int_x86_sse42_crc32_32_8>; def CRC32r32m16 : SS42I_crc32m<0xF1, "crc32{w}", GR32, i16mem, - int_x86_sse42_crc32_32_16>, OpSize; + int_x86_sse42_crc32_32_16>, OpSize16; def CRC32r32r16 : SS42I_crc32r<0xF1, "crc32{w}", GR32, GR16, - int_x86_sse42_crc32_32_16>, OpSize; + int_x86_sse42_crc32_32_16>, OpSize16; def CRC32r32m32 : SS42I_crc32m<0xF1, "crc32{l}", GR32, i32mem, - int_x86_sse42_crc32_32_32>; + int_x86_sse42_crc32_32_32>, OpSize32; def CRC32r32r32 : SS42I_crc32r<0xF1, "crc32{l}", GR32, GR32, - int_x86_sse42_crc32_32_32>; + int_x86_sse42_crc32_32_32>, OpSize32; def CRC32r64m64 : SS42I_crc32m<0xF1, "crc32{q}", GR64, i64mem, int_x86_sse42_crc32_64_64>, REX_W; def CRC32r64r64 : SS42I_crc32r<0xF1, "crc32{q}", GR64, GR64, @@ -7357,14 +7704,15 @@ multiclass AESI_binop_rm_int<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>, - OpSize; + Sched<[WriteAESDecEnc]>; def rm : AES8I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), !if(Is2Addr, !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), [(set VR128:$dst, - (IntId128 VR128:$src1, (memopv2i64 addr:$src2)))]>, OpSize; + (IntId128 VR128:$src1, (memopv2i64 addr:$src2)))]>, + Sched<[WriteAESDecEncLd, ReadAfterLd]>; } // Perform One Round of an AES Encryption/Decryption Flow @@ -7396,25 +7744,24 @@ let Predicates = [HasAVX, HasAES] in { (ins VR128:$src1), "vaesimc\t{$src1, $dst|$dst, $src1}", [(set VR128:$dst, - (int_x86_aesni_aesimc VR128:$src1))]>, - OpSize, VEX; + (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>, + VEX; def VAESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1), "vaesimc\t{$src1, $dst|$dst, $src1}", [(set VR128:$dst, (int_x86_aesni_aesimc (loadv2i64 addr:$src1)))]>, - OpSize, VEX; + Sched<[WriteAESIMCLd]>, VEX; } def AESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1), "aesimc\t{$src1, $dst|$dst, $src1}", [(set VR128:$dst, - (int_x86_aesni_aesimc VR128:$src1))]>, - OpSize; + (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>; def AESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1), "aesimc\t{$src1, $dst|$dst, $src1}", [(set VR128:$dst, (int_x86_aesni_aesimc (memopv2i64 addr:$src1)))]>, - OpSize; + Sched<[WriteAESIMCLd]>; // AES Round Key Generation Assist let Predicates = [HasAVX, HasAES] in { @@ -7423,26 +7770,26 @@ let Predicates = [HasAVX, HasAES] in { "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>, - OpSize, VEX; + Sched<[WriteAESKeyGen]>, VEX; def VAESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1, i8imm:$src2), "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist (loadv2i64 addr:$src1), imm:$src2))]>, - OpSize, VEX; + Sched<[WriteAESKeyGenLd]>, VEX; } def AESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, i8imm:$src2), "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>, - OpSize; + Sched<[WriteAESKeyGen]>; def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1, i8imm:$src2), "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist (memopv2i64 addr:$src1), imm:$src2))]>, - OpSize; + Sched<[WriteAESKeyGenLd]>; //===----------------------------------------------------------------------===// // PCLMUL Instructions @@ -7453,13 +7800,15 @@ def VPCLMULQDQrr : AVXPCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i8imm:$src3), "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR128:$dst, - (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))]>; + (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))]>, + Sched<[WriteCLMul]>; def VPCLMULQDQrm : AVXPCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2, i8imm:$src3), "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, - (loadv2i64 addr:$src2), imm:$src3))]>; + (loadv2i64 addr:$src2), imm:$src3))]>, + Sched<[WriteCLMulLd, ReadAfterLd]>; // Carry-less Multiplication instructions let Constraints = "$src1 = $dst" in { @@ -7468,14 +7817,15 @@ def PCLMULQDQrr : PCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst), "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))], - IIC_SSE_PCLMULQDQ_RR>; + IIC_SSE_PCLMULQDQ_RR>, Sched<[WriteCLMul]>; def PCLMULQDQrm : PCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2, i8imm:$src3), "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, (memopv2i64 addr:$src2), imm:$src3))], - IIC_SSE_PCLMULQDQ_RM>; + IIC_SSE_PCLMULQDQ_RM>, + Sched<[WriteCLMulLd, ReadAfterLd]>; } // Constraints = "$src1 = $dst" @@ -7506,16 +7856,16 @@ defm : pclmul_alias<"lqlq", 0x00>; let Predicates = [HasSSE4A] in { let Constraints = "$src = $dst" in { -def EXTRQI : Ii8<0x78, MRM0r, (outs VR128:$dst), +def EXTRQI : Ii8<0x78, MRMXr, (outs VR128:$dst), (ins VR128:$src, i8imm:$len, i8imm:$idx), "extrq\t{$idx, $len, $src|$src, $len, $idx}", [(set VR128:$dst, (int_x86_sse4a_extrqi VR128:$src, imm:$len, - imm:$idx))]>, TB, OpSize; + imm:$idx))]>, PD; def EXTRQ : I<0x79, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src, VR128:$mask), "extrq\t{$mask, $src|$src, $mask}", [(set VR128:$dst, (int_x86_sse4a_extrq VR128:$src, - VR128:$mask))]>, TB, OpSize; + VR128:$mask))]>, PD; def INSERTQI : Ii8<0x78, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src, VR128:$src2, i8imm:$len, i8imm:$idx), @@ -7547,43 +7897,50 @@ def MOVNTSD : I<0x2B, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), // destination operand // class avx_broadcast<bits<8> opc, string OpcodeStr, RegisterClass RC, - X86MemOperand x86memop, Intrinsic Int> : + X86MemOperand x86memop, Intrinsic Int, SchedWrite Sched> : AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set RC:$dst, (Int addr:$src))]>, VEX; + [(set RC:$dst, (Int addr:$src))]>, Sched<[Sched]>, VEX; // AVX2 adds register forms class avx2_broadcast_reg<bits<8> opc, string OpcodeStr, RegisterClass RC, - Intrinsic Int> : + Intrinsic Int, SchedWrite Sched> : AVX28I<opc, MRMSrcReg, (outs RC:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set RC:$dst, (Int VR128:$src))]>, VEX; + [(set RC:$dst, (Int VR128:$src))]>, Sched<[Sched]>, VEX; let ExeDomain = SSEPackedSingle in { def VBROADCASTSSrm : avx_broadcast<0x18, "vbroadcastss", VR128, f32mem, - int_x86_avx_vbroadcast_ss>; + int_x86_avx_vbroadcast_ss, WriteLoad>; def VBROADCASTSSYrm : avx_broadcast<0x18, "vbroadcastss", VR256, f32mem, - int_x86_avx_vbroadcast_ss_256>, VEX_L; + int_x86_avx_vbroadcast_ss_256, + WriteFShuffleLd>, VEX_L; } let ExeDomain = SSEPackedDouble in def VBROADCASTSDYrm : avx_broadcast<0x19, "vbroadcastsd", VR256, f64mem, - int_x86_avx_vbroadcast_sd_256>, VEX_L; + int_x86_avx_vbroadcast_sd_256, + WriteFShuffleLd>, VEX_L; def VBROADCASTF128 : avx_broadcast<0x1A, "vbroadcastf128", VR256, f128mem, - int_x86_avx_vbroadcastf128_pd_256>, VEX_L; + int_x86_avx_vbroadcastf128_pd_256, + WriteFShuffleLd>, VEX_L; let ExeDomain = SSEPackedSingle in { def VBROADCASTSSrr : avx2_broadcast_reg<0x18, "vbroadcastss", VR128, - int_x86_avx2_vbroadcast_ss_ps>; + int_x86_avx2_vbroadcast_ss_ps, + WriteFShuffle>; def VBROADCASTSSYrr : avx2_broadcast_reg<0x18, "vbroadcastss", VR256, - int_x86_avx2_vbroadcast_ss_ps_256>, VEX_L; + int_x86_avx2_vbroadcast_ss_ps_256, + WriteFShuffle256>, VEX_L; } let ExeDomain = SSEPackedDouble in def VBROADCASTSDYrr : avx2_broadcast_reg<0x19, "vbroadcastsd", VR256, - int_x86_avx2_vbroadcast_sd_pd_256>, VEX_L; + int_x86_avx2_vbroadcast_sd_pd_256, + WriteFShuffle256>, VEX_L; let Predicates = [HasAVX2] in def VBROADCASTI128 : avx_broadcast<0x5A, "vbroadcasti128", VR256, i128mem, - int_x86_avx2_vbroadcasti128>, VEX_L; + int_x86_avx2_vbroadcasti128, WriteLoad>, + VEX_L; let Predicates = [HasAVX] in def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src), @@ -7597,12 +7954,12 @@ let neverHasSideEffects = 1, ExeDomain = SSEPackedSingle in { def VINSERTF128rr : AVXAIi8<0x18, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR128:$src2, i8imm:$src3), "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - []>, VEX_4V, VEX_L; + []>, Sched<[WriteFShuffle]>, VEX_4V, VEX_L; let mayLoad = 1 in def VINSERTF128rm : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, f128mem:$src2, i8imm:$src3), "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - []>, VEX_4V, VEX_L; + []>, Sched<[WriteFShuffleLd, ReadAfterLd]>, VEX_4V, VEX_L; } let Predicates = [HasAVX] in { @@ -7671,12 +8028,12 @@ let neverHasSideEffects = 1, ExeDomain = SSEPackedSingle in { def VEXTRACTF128rr : AVXAIi8<0x19, MRMDestReg, (outs VR128:$dst), (ins VR256:$src1, i8imm:$src2), "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}", - []>, VEX, VEX_L; + []>, Sched<[WriteFShuffle]>, VEX, VEX_L; let mayStore = 1 in def VEXTRACTF128mr : AVXAIi8<0x19, MRMDestMem, (outs), (ins f128mem:$dst, VR256:$src1, i8imm:$src2), "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}", - []>, VEX, VEX_L; + []>, Sched<[WriteStore]>, VEX, VEX_L; } // AVX1 patterns @@ -7785,22 +8142,26 @@ multiclass avx_permil<bits<8> opc_rm, bits<8> opc_rmi, string OpcodeStr, def rr : AVX8I<opc_rm, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, (IntVar RC:$src1, RC:$src2))]>, VEX_4V; + [(set RC:$dst, (IntVar RC:$src1, RC:$src2))]>, VEX_4V, + Sched<[WriteFShuffle]>; def rm : AVX8I<opc_rm, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop_i:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, (IntVar RC:$src1, - (bitconvert (i_frag addr:$src2))))]>, VEX_4V; + (bitconvert (i_frag addr:$src2))))]>, VEX_4V, + Sched<[WriteFShuffleLd, ReadAfterLd]>; def ri : AVXAIi8<opc_rmi, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, (vt (X86VPermilp RC:$src1, (i8 imm:$src2))))]>, VEX; + [(set RC:$dst, (vt (X86VPermilp RC:$src1, (i8 imm:$src2))))]>, VEX, + Sched<[WriteFShuffle]>; def mi : AVXAIi8<opc_rmi, MRMSrcMem, (outs RC:$dst), (ins x86memop_f:$src1, i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set RC:$dst, - (vt (X86VPermilp (memop addr:$src1), (i8 imm:$src2))))]>, VEX; + (vt (X86VPermilp (memop addr:$src1), (i8 imm:$src2))))]>, VEX, + Sched<[WriteFShuffleLd]>; } let ExeDomain = SSEPackedSingle in { @@ -7841,12 +8202,14 @@ def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2, i8imm:$src3), "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR256:$dst, (v8f32 (X86VPerm2x128 VR256:$src1, VR256:$src2, - (i8 imm:$src3))))]>, VEX_4V, VEX_L; + (i8 imm:$src3))))]>, VEX_4V, VEX_L, + Sched<[WriteFShuffle]>; def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, f256mem:$src2, i8imm:$src3), "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv8f32 addr:$src2), - (i8 imm:$src3)))]>, VEX_4V, VEX_L; + (i8 imm:$src3)))]>, VEX_4V, VEX_L, + Sched<[WriteFShuffleLd, ReadAfterLd]>; } let Predicates = [HasAVX] in { @@ -7888,11 +8251,11 @@ let Defs = [YMM0, YMM1, YMM2, YMM3, YMM4, YMM5, YMM6, YMM7, YMM8, YMM9, YMM10, YMM11, YMM12, YMM13, YMM14, YMM15] in { // Zero All YMM registers def VZEROALL : I<0x77, RawFrm, (outs), (ins), "vzeroall", - [(int_x86_avx_vzeroall)]>, TB, VEX, VEX_L, Requires<[HasAVX]>; + [(int_x86_avx_vzeroall)]>, PS, VEX, VEX_L, Requires<[HasAVX]>; // Zero Upper bits of YMM registers def VZEROUPPER : I<0x77, RawFrm, (outs), (ins), "vzeroupper", - [(int_x86_avx_vzeroupper)]>, TB, VEX, Requires<[HasAVX]>; + [(int_x86_avx_vzeroupper)]>, PS, VEX, Requires<[HasAVX]>; } //===----------------------------------------------------------------------===// @@ -7902,10 +8265,11 @@ multiclass f16c_ph2ps<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { def rr : I<0x13, MRMSrcReg, (outs RC:$dst), (ins VR128:$src), "vcvtph2ps\t{$src, $dst|$dst, $src}", [(set RC:$dst, (Int VR128:$src))]>, - T8, OpSize, VEX; + T8PD, VEX, Sched<[WriteCvtF2F]>; let neverHasSideEffects = 1, mayLoad = 1 in def rm : I<0x13, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), - "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8, OpSize, VEX; + "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8PD, VEX, + Sched<[WriteCvtF2FLd]>; } multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { @@ -7913,12 +8277,13 @@ multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { (ins RC:$src1, i32i8imm:$src2), "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (Int RC:$src1, imm:$src2))]>, - TA, OpSize, VEX; - let neverHasSideEffects = 1, mayStore = 1 in + TAPD, VEX, Sched<[WriteCvtF2F]>; + let neverHasSideEffects = 1, mayStore = 1, + SchedRW = [WriteCvtF2FLd, WriteRMW] in def mr : Ii8<0x1D, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src1, i32i8imm:$src2), "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, - TA, OpSize, VEX; + TAPD, VEX; } let Predicates = [HasF16C] in { @@ -7942,7 +8307,7 @@ multiclass AVX2_binop_rmi_int<bits<8> opc, string OpcodeStr, !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set RC:$dst, (IntId RC:$src1, RC:$src2, imm:$src3))]>, - VEX_4V; + Sched<[WriteBlend]>, VEX_4V; def rmi : AVX2AIi8<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u32u8imm:$src3), !strconcat(OpcodeStr, @@ -7950,7 +8315,7 @@ multiclass AVX2_binop_rmi_int<bits<8> opc, string OpcodeStr, [(set RC:$dst, (IntId RC:$src1, (bitconvert (memop_frag addr:$src2)), imm:$src3))]>, - VEX_4V; + Sched<[WriteBlendLd, ReadAfterLd]>, VEX_4V; } let isCommutable = 0 in { @@ -7976,19 +8341,22 @@ multiclass avx2_broadcast<bits<8> opc, string OpcodeStr, Intrinsic Int128, Intrinsic Int256> { def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int128 VR128:$src))]>, VEX; + [(set VR128:$dst, (Int128 VR128:$src))]>, + Sched<[WriteShuffle]>, VEX; def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR128:$dst, - (Int128 (scalar_to_vector (ld_frag addr:$src))))]>, VEX; + (Int128 (scalar_to_vector (ld_frag addr:$src))))]>, + Sched<[WriteLoad]>, VEX; def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR256:$dst, (Int256 VR128:$src))]>, VEX, VEX_L; + [(set VR256:$dst, (Int256 VR128:$src))]>, + Sched<[WriteShuffle256]>, VEX, VEX_L; def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins x86memop:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [(set VR256:$dst, (Int256 (scalar_to_vector (ld_frag addr:$src))))]>, - VEX, VEX_L; + Sched<[WriteLoad]>, VEX, VEX_L; } defm VPBROADCASTB : avx2_broadcast<0x78, "vpbroadcastb", i8mem, loadi8, @@ -8063,6 +8431,31 @@ let Predicates = [HasAVX2] in { (VBROADCASTSSYrr (COPY_TO_REGCLASS GR32:$src, VR128))>; def : Pat<(v4i64 (X86VBroadcast GR64:$src)), (VBROADCASTSDYrr (COPY_TO_REGCLASS GR64:$src, VR128))>; + + def : Pat<(v16i8 (X86VBroadcast GR8:$src)), + (VPBROADCASTBrr (COPY_TO_REGCLASS + (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), + VR128))>; + def : Pat<(v32i8 (X86VBroadcast GR8:$src)), + (VPBROADCASTBYrr (COPY_TO_REGCLASS + (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), + VR128))>; + + def : Pat<(v8i16 (X86VBroadcast GR16:$src)), + (VPBROADCASTWrr (COPY_TO_REGCLASS + (i32 (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit)), + VR128))>; + def : Pat<(v16i16 (X86VBroadcast GR16:$src)), + (VPBROADCASTWYrr (COPY_TO_REGCLASS + (i32 (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit)), + VR128))>; + + // The patterns for VPBROADCASTD are not needed because they would match + // the exact same thing as VBROADCASTSS patterns. + + def : Pat<(v2i64 (X86VBroadcast GR64:$src)), + (VPBROADCASTQrr (COPY_TO_REGCLASS GR64:$src, VR128))>; + // The v4i64 pattern is not needed because VBROADCASTSDYrr already match. } } @@ -8124,7 +8517,7 @@ multiclass avx2_perm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, (OpVT (X86VPermv VR256:$src1, VR256:$src2)))]>, - VEX_4V, VEX_L; + Sched<[WriteFShuffle256]>, VEX_4V, VEX_L; def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), !strconcat(OpcodeStr, @@ -8132,7 +8525,7 @@ multiclass avx2_perm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, [(set VR256:$dst, (OpVT (X86VPermv VR256:$src1, (bitconvert (mem_frag addr:$src2)))))]>, - VEX_4V, VEX_L; + Sched<[WriteFShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; } defm VPERMD : avx2_perm<0x36, "vpermd", loadv4i64, v8i32>; @@ -8147,14 +8540,15 @@ multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, (OpVT (X86VPermi VR256:$src1, (i8 imm:$src2))))]>, - VEX, VEX_L; + Sched<[WriteShuffle256]>, VEX, VEX_L; def Ymi : AVX2AIi8<opc, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src1, i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, (OpVT (X86VPermi (mem_frag addr:$src1), - (i8 imm:$src2))))]>, VEX, VEX_L; + (i8 imm:$src2))))]>, + Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX, VEX_L; } defm VPERMQ : avx2_perm_imm<0x00, "vpermq", loadv4i64, v4i64>, VEX_W; @@ -8168,12 +8562,14 @@ def VPERM2I128rr : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2, i8imm:$src3), "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR256:$dst, (v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, - (i8 imm:$src3))))]>, VEX_4V, VEX_L; + (i8 imm:$src3))))]>, Sched<[WriteShuffle256]>, + VEX_4V, VEX_L; def VPERM2I128rm : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, f256mem:$src2, i8imm:$src3), "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4i64 addr:$src2), - (i8 imm:$src3)))]>, VEX_4V, VEX_L; + (i8 imm:$src3)))]>, + Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; let Predicates = [HasAVX2] in { def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), @@ -8202,12 +8598,12 @@ let neverHasSideEffects = 1 in { def VINSERTI128rr : AVX2AIi8<0x38, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR128:$src2, i8imm:$src3), "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - []>, VEX_4V, VEX_L; + []>, Sched<[WriteShuffle256]>, VEX_4V, VEX_L; let mayLoad = 1 in def VINSERTI128rm : AVX2AIi8<0x38, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, i128mem:$src2, i8imm:$src3), "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", - []>, VEX_4V, VEX_L; + []>, Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; } let Predicates = [HasAVX2] in { @@ -8257,12 +8653,12 @@ def VEXTRACTI128rr : AVX2AIi8<0x39, MRMDestReg, (outs VR128:$dst), "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_avx2_vextracti128 VR256:$src1, imm:$src2))]>, - VEX, VEX_L; + Sched<[WriteShuffle256]>, VEX, VEX_L; let neverHasSideEffects = 1, mayStore = 1 in def VEXTRACTI128mr : AVX2AIi8<0x39, MRMDestMem, (outs), (ins i128mem:$dst, VR256:$src1, i8imm:$src2), "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, - VEX, VEX_L; + Sched<[WriteStore]>, VEX, VEX_L; let Predicates = [HasAVX2] in { def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), @@ -8347,27 +8743,27 @@ multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode, !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128:$dst, (vt128 (OpNode VR128:$src1, (vt128 VR128:$src2))))]>, - VEX_4V; + VEX_4V, Sched<[WriteVarVecShift]>; def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128:$dst, (vt128 (OpNode VR128:$src1, (vt128 (bitconvert (loadv2i64 addr:$src2))))))]>, - VEX_4V; + VEX_4V, Sched<[WriteVarVecShiftLd, ReadAfterLd]>; def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, (vt256 (OpNode VR256:$src1, (vt256 VR256:$src2))))]>, - VEX_4V, VEX_L; + VEX_4V, VEX_L, Sched<[WriteVarVecShift]>; def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR256:$dst, (vt256 (OpNode VR256:$src1, (vt256 (bitconvert (loadv4i64 addr:$src2))))))]>, - VEX_4V, VEX_L; + VEX_4V, VEX_L, Sched<[WriteVarVecShiftLd, ReadAfterLd]>; } defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", shl, v4i32, v8i32>; @@ -8395,12 +8791,18 @@ multiclass avx2_gather<bits<8> opc, string OpcodeStr, RegisterClass RC256, let mayLoad = 1, Constraints = "@earlyclobber $dst,@earlyclobber $mask_wb, $src1 = $dst, $mask = $mask_wb" in { - defm VGATHERDPD : avx2_gather<0x92, "vgatherdpd", VR256, vx64mem, vx64mem>, VEX_W; - defm VGATHERQPD : avx2_gather<0x93, "vgatherqpd", VR256, vx64mem, vy64mem>, VEX_W; - defm VGATHERDPS : avx2_gather<0x92, "vgatherdps", VR256, vx32mem, vy32mem>; - defm VGATHERQPS : avx2_gather<0x93, "vgatherqps", VR128, vx32mem, vy32mem>; defm VPGATHERDQ : avx2_gather<0x90, "vpgatherdq", VR256, vx64mem, vx64mem>, VEX_W; defm VPGATHERQQ : avx2_gather<0x91, "vpgatherqq", VR256, vx64mem, vy64mem>, VEX_W; defm VPGATHERDD : avx2_gather<0x90, "vpgatherdd", VR256, vx32mem, vy32mem>; defm VPGATHERQD : avx2_gather<0x91, "vpgatherqd", VR128, vx32mem, vy32mem>; + + let ExeDomain = SSEPackedDouble in { + defm VGATHERDPD : avx2_gather<0x92, "vgatherdpd", VR256, vx64mem, vx64mem>, VEX_W; + defm VGATHERQPD : avx2_gather<0x93, "vgatherqpd", VR256, vx64mem, vy64mem>, VEX_W; + } + + let ExeDomain = SSEPackedSingle in { + defm VGATHERDPS : avx2_gather<0x92, "vgatherdps", VR256, vx32mem, vy32mem>; + defm VGATHERQPS : avx2_gather<0x93, "vgatherqps", VR128, vx32mem, vy32mem>; + } } diff --git a/lib/Target/X86/X86InstrSVM.td b/lib/Target/X86/X86InstrSVM.td index 0191c01..c847be7 100644 --- a/lib/Target/X86/X86InstrSVM.td +++ b/lib/Target/X86/X86InstrSVM.td @@ -31,7 +31,7 @@ def SKINIT : I<0x01, MRM_DE, (outs), (ins), "skinit\t{%eax|eax}", []>, TB; // 0F 01 D8 let Uses = [EAX] in def VMRUN32 : I<0x01, MRM_D8, (outs), (ins), - "vmrun\t{%eax|eax}", []>, TB, Requires<[In32BitMode]>; + "vmrun\t{%eax|eax}", []>, TB, Requires<[Not64BitMode]>; let Uses = [RAX] in def VMRUN64 : I<0x01, MRM_D8, (outs), (ins), "vmrun\t{%rax|rax}", []>, TB, Requires<[In64BitMode]>; @@ -39,7 +39,7 @@ def VMRUN64 : I<0x01, MRM_D8, (outs), (ins), // 0F 01 DA let Uses = [EAX] in def VMLOAD32 : I<0x01, MRM_DA, (outs), (ins), - "vmload\t{%eax|eax}", []>, TB, Requires<[In32BitMode]>; + "vmload\t{%eax|eax}", []>, TB, Requires<[Not64BitMode]>; let Uses = [RAX] in def VMLOAD64 : I<0x01, MRM_DA, (outs), (ins), "vmload\t{%rax|rax}", []>, TB, Requires<[In64BitMode]>; @@ -47,7 +47,7 @@ def VMLOAD64 : I<0x01, MRM_DA, (outs), (ins), // 0F 01 DB let Uses = [EAX] in def VMSAVE32 : I<0x01, MRM_DB, (outs), (ins), - "vmsave\t{%eax|eax}", []>, TB, Requires<[In32BitMode]>; + "vmsave\t{%eax|eax}", []>, TB, Requires<[Not64BitMode]>; let Uses = [RAX] in def VMSAVE64 : I<0x01, MRM_DB, (outs), (ins), "vmsave\t{%rax|rax}", []>, TB, Requires<[In64BitMode]>; @@ -55,7 +55,7 @@ def VMSAVE64 : I<0x01, MRM_DB, (outs), (ins), // 0F 01 DF let Uses = [EAX, ECX] in def INVLPGA32 : I<0x01, MRM_DF, (outs), (ins), - "invlpga\t{%ecx, %eax|eax, ecx}", []>, TB, Requires<[In32BitMode]>; + "invlpga\t{%ecx, %eax|eax, ecx}", []>, TB, Requires<[Not64BitMode]>; let Uses = [RAX, ECX] in def INVLPGA64 : I<0x01, MRM_DF, (outs), (ins), "invlpga\t{%ecx, %rax|rax, ecx}", []>, TB, Requires<[In64BitMode]>; diff --git a/lib/Target/X86/X86InstrShiftRotate.td b/lib/Target/X86/X86InstrShiftRotate.td index 1937770..d0bb523 100644 --- a/lib/Target/X86/X86InstrShiftRotate.td +++ b/lib/Target/X86/X86InstrShiftRotate.td @@ -22,10 +22,10 @@ def SHL8rCL : I<0xD2, MRM4r, (outs GR8 :$dst), (ins GR8 :$src1), [(set GR8:$dst, (shl GR8:$src1, CL))], IIC_SR>; def SHL16rCL : I<0xD3, MRM4r, (outs GR16:$dst), (ins GR16:$src1), "shl{w}\t{%cl, $dst|$dst, cl}", - [(set GR16:$dst, (shl GR16:$src1, CL))], IIC_SR>, OpSize; + [(set GR16:$dst, (shl GR16:$src1, CL))], IIC_SR>, OpSize16; def SHL32rCL : I<0xD3, MRM4r, (outs GR32:$dst), (ins GR32:$src1), "shl{l}\t{%cl, $dst|$dst, cl}", - [(set GR32:$dst, (shl GR32:$src1, CL))], IIC_SR>; + [(set GR32:$dst, (shl GR32:$src1, CL))], IIC_SR>, OpSize32; def SHL64rCL : RI<0xD3, MRM4r, (outs GR64:$dst), (ins GR64:$src1), "shl{q}\t{%cl, $dst|$dst, cl}", [(set GR64:$dst, (shl GR64:$src1, CL))], IIC_SR>; @@ -39,10 +39,11 @@ let isConvertibleToThreeAddress = 1 in { // Can transform into LEA. def SHL16ri : Ii8<0xC1, MRM4r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$src2), "shl{w}\t{$src2, $dst|$dst, $src2}", [(set GR16:$dst, (shl GR16:$src1, (i8 imm:$src2)))], IIC_SR>, - OpSize; + OpSize16; def SHL32ri : Ii8<0xC1, MRM4r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$src2), "shl{l}\t{$src2, $dst|$dst, $src2}", - [(set GR32:$dst, (shl GR32:$src1, (i8 imm:$src2)))], IIC_SR>; + [(set GR32:$dst, (shl GR32:$src1, (i8 imm:$src2)))], IIC_SR>, + OpSize32; def SHL64ri : RIi8<0xC1, MRM4r, (outs GR64:$dst), (ins GR64:$src1, i8imm:$src2), "shl{q}\t{$src2, $dst|$dst, $src2}", @@ -55,9 +56,9 @@ let hasSideEffects = 0 in { def SHL8r1 : I<0xD0, MRM4r, (outs GR8:$dst), (ins GR8:$src1), "shl{b}\t$dst", [], IIC_SR>; def SHL16r1 : I<0xD1, MRM4r, (outs GR16:$dst), (ins GR16:$src1), - "shl{w}\t$dst", [], IIC_SR>, OpSize; + "shl{w}\t$dst", [], IIC_SR>, OpSize16; def SHL32r1 : I<0xD1, MRM4r, (outs GR32:$dst), (ins GR32:$src1), - "shl{l}\t$dst", [], IIC_SR>; + "shl{l}\t$dst", [], IIC_SR>, OpSize32; def SHL64r1 : RI<0xD1, MRM4r, (outs GR64:$dst), (ins GR64:$src1), "shl{q}\t$dst", [], IIC_SR>; } // hasSideEffects = 0 @@ -75,10 +76,11 @@ def SHL8mCL : I<0xD2, MRM4m, (outs), (ins i8mem :$dst), def SHL16mCL : I<0xD3, MRM4m, (outs), (ins i16mem:$dst), "shl{w}\t{%cl, $dst|$dst, cl}", [(store (shl (loadi16 addr:$dst), CL), addr:$dst)], IIC_SR>, - OpSize; + OpSize16; def SHL32mCL : I<0xD3, MRM4m, (outs), (ins i32mem:$dst), "shl{l}\t{%cl, $dst|$dst, cl}", - [(store (shl (loadi32 addr:$dst), CL), addr:$dst)], IIC_SR>; + [(store (shl (loadi32 addr:$dst), CL), addr:$dst)], IIC_SR>, + OpSize32; def SHL64mCL : RI<0xD3, MRM4m, (outs), (ins i64mem:$dst), "shl{q}\t{%cl, $dst|$dst, cl}", [(store (shl (loadi64 addr:$dst), CL), addr:$dst)], IIC_SR>; @@ -90,12 +92,11 @@ def SHL8mi : Ii8<0xC0, MRM4m, (outs), (ins i8mem :$dst, i8imm:$src), def SHL16mi : Ii8<0xC1, MRM4m, (outs), (ins i16mem:$dst, i8imm:$src), "shl{w}\t{$src, $dst|$dst, $src}", [(store (shl (loadi16 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def SHL32mi : Ii8<0xC1, MRM4m, (outs), (ins i32mem:$dst, i8imm:$src), "shl{l}\t{$src, $dst|$dst, $src}", [(store (shl (loadi32 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SHL64mi : RIi8<0xC1, MRM4m, (outs), (ins i64mem:$dst, i8imm:$src), "shl{q}\t{$src, $dst|$dst, $src}", [(store (shl (loadi64 addr:$dst), (i8 imm:$src)), addr:$dst)], @@ -109,12 +110,11 @@ def SHL8m1 : I<0xD0, MRM4m, (outs), (ins i8mem :$dst), def SHL16m1 : I<0xD1, MRM4m, (outs), (ins i16mem:$dst), "shl{w}\t$dst", [(store (shl (loadi16 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def SHL32m1 : I<0xD1, MRM4m, (outs), (ins i32mem:$dst), "shl{l}\t$dst", [(store (shl (loadi32 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SHL64m1 : RI<0xD1, MRM4m, (outs), (ins i64mem:$dst), "shl{q}\t$dst", [(store (shl (loadi64 addr:$dst), (i8 1)), addr:$dst)], @@ -128,10 +128,10 @@ def SHR8rCL : I<0xD2, MRM5r, (outs GR8 :$dst), (ins GR8 :$src1), [(set GR8:$dst, (srl GR8:$src1, CL))], IIC_SR>; def SHR16rCL : I<0xD3, MRM5r, (outs GR16:$dst), (ins GR16:$src1), "shr{w}\t{%cl, $dst|$dst, cl}", - [(set GR16:$dst, (srl GR16:$src1, CL))], IIC_SR>, OpSize; + [(set GR16:$dst, (srl GR16:$src1, CL))], IIC_SR>, OpSize16; def SHR32rCL : I<0xD3, MRM5r, (outs GR32:$dst), (ins GR32:$src1), "shr{l}\t{%cl, $dst|$dst, cl}", - [(set GR32:$dst, (srl GR32:$src1, CL))], IIC_SR>; + [(set GR32:$dst, (srl GR32:$src1, CL))], IIC_SR>, OpSize32; def SHR64rCL : RI<0xD3, MRM5r, (outs GR64:$dst), (ins GR64:$src1), "shr{q}\t{%cl, $dst|$dst, cl}", [(set GR64:$dst, (srl GR64:$src1, CL))], IIC_SR>; @@ -143,11 +143,11 @@ def SHR8ri : Ii8<0xC0, MRM5r, (outs GR8:$dst), (ins GR8:$src1, i8imm:$src2), def SHR16ri : Ii8<0xC1, MRM5r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$src2), "shr{w}\t{$src2, $dst|$dst, $src2}", [(set GR16:$dst, (srl GR16:$src1, (i8 imm:$src2)))], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def SHR32ri : Ii8<0xC1, MRM5r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$src2), "shr{l}\t{$src2, $dst|$dst, $src2}", [(set GR32:$dst, (srl GR32:$src1, (i8 imm:$src2)))], - IIC_SR>; + IIC_SR>, OpSize32; def SHR64ri : RIi8<0xC1, MRM5r, (outs GR64:$dst), (ins GR64:$src1, i8imm:$src2), "shr{q}\t{$src2, $dst|$dst, $src2}", [(set GR64:$dst, (srl GR64:$src1, (i8 imm:$src2)))], IIC_SR>; @@ -158,10 +158,10 @@ def SHR8r1 : I<0xD0, MRM5r, (outs GR8:$dst), (ins GR8:$src1), [(set GR8:$dst, (srl GR8:$src1, (i8 1)))], IIC_SR>; def SHR16r1 : I<0xD1, MRM5r, (outs GR16:$dst), (ins GR16:$src1), "shr{w}\t$dst", - [(set GR16:$dst, (srl GR16:$src1, (i8 1)))], IIC_SR>, OpSize; + [(set GR16:$dst, (srl GR16:$src1, (i8 1)))], IIC_SR>, OpSize16; def SHR32r1 : I<0xD1, MRM5r, (outs GR32:$dst), (ins GR32:$src1), "shr{l}\t$dst", - [(set GR32:$dst, (srl GR32:$src1, (i8 1)))], IIC_SR>; + [(set GR32:$dst, (srl GR32:$src1, (i8 1)))], IIC_SR>, OpSize32; def SHR64r1 : RI<0xD1, MRM5r, (outs GR64:$dst), (ins GR64:$src1), "shr{q}\t$dst", [(set GR64:$dst, (srl GR64:$src1, (i8 1)))], IIC_SR>; @@ -176,10 +176,11 @@ def SHR8mCL : I<0xD2, MRM5m, (outs), (ins i8mem :$dst), def SHR16mCL : I<0xD3, MRM5m, (outs), (ins i16mem:$dst), "shr{w}\t{%cl, $dst|$dst, cl}", [(store (srl (loadi16 addr:$dst), CL), addr:$dst)], IIC_SR>, - OpSize; + OpSize16; def SHR32mCL : I<0xD3, MRM5m, (outs), (ins i32mem:$dst), "shr{l}\t{%cl, $dst|$dst, cl}", - [(store (srl (loadi32 addr:$dst), CL), addr:$dst)], IIC_SR>; + [(store (srl (loadi32 addr:$dst), CL), addr:$dst)], IIC_SR>, + OpSize32; def SHR64mCL : RI<0xD3, MRM5m, (outs), (ins i64mem:$dst), "shr{q}\t{%cl, $dst|$dst, cl}", [(store (srl (loadi64 addr:$dst), CL), addr:$dst)], IIC_SR>; @@ -191,12 +192,11 @@ def SHR8mi : Ii8<0xC0, MRM5m, (outs), (ins i8mem :$dst, i8imm:$src), def SHR16mi : Ii8<0xC1, MRM5m, (outs), (ins i16mem:$dst, i8imm:$src), "shr{w}\t{$src, $dst|$dst, $src}", [(store (srl (loadi16 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def SHR32mi : Ii8<0xC1, MRM5m, (outs), (ins i32mem:$dst, i8imm:$src), "shr{l}\t{$src, $dst|$dst, $src}", [(store (srl (loadi32 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SHR64mi : RIi8<0xC1, MRM5m, (outs), (ins i64mem:$dst, i8imm:$src), "shr{q}\t{$src, $dst|$dst, $src}", [(store (srl (loadi64 addr:$dst), (i8 imm:$src)), addr:$dst)], @@ -210,11 +210,11 @@ def SHR8m1 : I<0xD0, MRM5m, (outs), (ins i8mem :$dst), def SHR16m1 : I<0xD1, MRM5m, (outs), (ins i16mem:$dst), "shr{w}\t$dst", [(store (srl (loadi16 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>,OpSize; + IIC_SR>, OpSize16; def SHR32m1 : I<0xD1, MRM5m, (outs), (ins i32mem:$dst), "shr{l}\t$dst", [(store (srl (loadi32 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SHR64m1 : RI<0xD1, MRM5m, (outs), (ins i64mem:$dst), "shr{q}\t$dst", [(store (srl (loadi64 addr:$dst), (i8 1)), addr:$dst)], @@ -230,11 +230,11 @@ def SAR8rCL : I<0xD2, MRM7r, (outs GR8 :$dst), (ins GR8 :$src1), def SAR16rCL : I<0xD3, MRM7r, (outs GR16:$dst), (ins GR16:$src1), "sar{w}\t{%cl, $dst|$dst, cl}", [(set GR16:$dst, (sra GR16:$src1, CL))], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def SAR32rCL : I<0xD3, MRM7r, (outs GR32:$dst), (ins GR32:$src1), "sar{l}\t{%cl, $dst|$dst, cl}", [(set GR32:$dst, (sra GR32:$src1, CL))], - IIC_SR>; + IIC_SR>, OpSize32; def SAR64rCL : RI<0xD3, MRM7r, (outs GR64:$dst), (ins GR64:$src1), "sar{q}\t{%cl, $dst|$dst, cl}", [(set GR64:$dst, (sra GR64:$src1, CL))], @@ -248,12 +248,11 @@ def SAR8ri : Ii8<0xC0, MRM7r, (outs GR8 :$dst), (ins GR8 :$src1, i8imm:$src2), def SAR16ri : Ii8<0xC1, MRM7r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$src2), "sar{w}\t{$src2, $dst|$dst, $src2}", [(set GR16:$dst, (sra GR16:$src1, (i8 imm:$src2)))], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def SAR32ri : Ii8<0xC1, MRM7r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$src2), "sar{l}\t{$src2, $dst|$dst, $src2}", [(set GR32:$dst, (sra GR32:$src1, (i8 imm:$src2)))], - IIC_SR>; + IIC_SR>, OpSize32; def SAR64ri : RIi8<0xC1, MRM7r, (outs GR64:$dst), (ins GR64:$src1, i8imm:$src2), "sar{q}\t{$src2, $dst|$dst, $src2}", @@ -268,11 +267,11 @@ def SAR8r1 : I<0xD0, MRM7r, (outs GR8 :$dst), (ins GR8 :$src1), def SAR16r1 : I<0xD1, MRM7r, (outs GR16:$dst), (ins GR16:$src1), "sar{w}\t$dst", [(set GR16:$dst, (sra GR16:$src1, (i8 1)))], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def SAR32r1 : I<0xD1, MRM7r, (outs GR32:$dst), (ins GR32:$src1), "sar{l}\t$dst", [(set GR32:$dst, (sra GR32:$src1, (i8 1)))], - IIC_SR>; + IIC_SR>, OpSize32; def SAR64r1 : RI<0xD1, MRM7r, (outs GR64:$dst), (ins GR64:$src1), "sar{q}\t$dst", [(set GR64:$dst, (sra GR64:$src1, (i8 1)))], @@ -289,11 +288,11 @@ def SAR8mCL : I<0xD2, MRM7m, (outs), (ins i8mem :$dst), def SAR16mCL : I<0xD3, MRM7m, (outs), (ins i16mem:$dst), "sar{w}\t{%cl, $dst|$dst, cl}", [(store (sra (loadi16 addr:$dst), CL), addr:$dst)], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def SAR32mCL : I<0xD3, MRM7m, (outs), (ins i32mem:$dst), "sar{l}\t{%cl, $dst|$dst, cl}", [(store (sra (loadi32 addr:$dst), CL), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SAR64mCL : RI<0xD3, MRM7m, (outs), (ins i64mem:$dst), "sar{q}\t{%cl, $dst|$dst, cl}", [(store (sra (loadi64 addr:$dst), CL), addr:$dst)], @@ -306,12 +305,11 @@ def SAR8mi : Ii8<0xC0, MRM7m, (outs), (ins i8mem :$dst, i8imm:$src), def SAR16mi : Ii8<0xC1, MRM7m, (outs), (ins i16mem:$dst, i8imm:$src), "sar{w}\t{$src, $dst|$dst, $src}", [(store (sra (loadi16 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def SAR32mi : Ii8<0xC1, MRM7m, (outs), (ins i32mem:$dst, i8imm:$src), "sar{l}\t{$src, $dst|$dst, $src}", [(store (sra (loadi32 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SAR64mi : RIi8<0xC1, MRM7m, (outs), (ins i64mem:$dst, i8imm:$src), "sar{q}\t{$src, $dst|$dst, $src}", [(store (sra (loadi64 addr:$dst), (i8 imm:$src)), addr:$dst)], @@ -325,12 +323,11 @@ def SAR8m1 : I<0xD0, MRM7m, (outs), (ins i8mem :$dst), def SAR16m1 : I<0xD1, MRM7m, (outs), (ins i16mem:$dst), "sar{w}\t$dst", [(store (sra (loadi16 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def SAR32m1 : I<0xD1, MRM7m, (outs), (ins i32mem:$dst), "sar{l}\t$dst", [(store (sra (loadi32 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def SAR64m1 : RI<0xD1, MRM7m, (outs), (ins i64mem:$dst), "sar{q}\t$dst", [(store (sra (loadi64 addr:$dst), (i8 1)), addr:$dst)], @@ -352,20 +349,20 @@ def RCL8rCL : I<0xD2, MRM2r, (outs GR8:$dst), (ins GR8:$src1), "rcl{b}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; def RCL16r1 : I<0xD1, MRM2r, (outs GR16:$dst), (ins GR16:$src1), - "rcl{w}\t$dst", [], IIC_SR>, OpSize; + "rcl{w}\t$dst", [], IIC_SR>, OpSize16; def RCL16ri : Ii8<0xC1, MRM2r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$cnt), - "rcl{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize; + "rcl{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize16; let Uses = [CL] in def RCL16rCL : I<0xD3, MRM2r, (outs GR16:$dst), (ins GR16:$src1), - "rcl{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize; + "rcl{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize16; def RCL32r1 : I<0xD1, MRM2r, (outs GR32:$dst), (ins GR32:$src1), - "rcl{l}\t$dst", [], IIC_SR>; + "rcl{l}\t$dst", [], IIC_SR>, OpSize32; def RCL32ri : Ii8<0xC1, MRM2r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$cnt), - "rcl{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>; + "rcl{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize32; let Uses = [CL] in def RCL32rCL : I<0xD3, MRM2r, (outs GR32:$dst), (ins GR32:$src1), - "rcl{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; + "rcl{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize32; def RCL64r1 : RI<0xD1, MRM2r, (outs GR64:$dst), (ins GR64:$src1), @@ -386,20 +383,20 @@ def RCR8rCL : I<0xD2, MRM3r, (outs GR8:$dst), (ins GR8:$src1), "rcr{b}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; def RCR16r1 : I<0xD1, MRM3r, (outs GR16:$dst), (ins GR16:$src1), - "rcr{w}\t$dst", [], IIC_SR>, OpSize; + "rcr{w}\t$dst", [], IIC_SR>, OpSize16; def RCR16ri : Ii8<0xC1, MRM3r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$cnt), - "rcr{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize; + "rcr{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize16; let Uses = [CL] in def RCR16rCL : I<0xD3, MRM3r, (outs GR16:$dst), (ins GR16:$src1), - "rcr{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize; + "rcr{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize16; def RCR32r1 : I<0xD1, MRM3r, (outs GR32:$dst), (ins GR32:$src1), - "rcr{l}\t$dst", [], IIC_SR>; + "rcr{l}\t$dst", [], IIC_SR>, OpSize32; def RCR32ri : Ii8<0xC1, MRM3r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$cnt), - "rcr{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>; + "rcr{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize32; let Uses = [CL] in def RCR32rCL : I<0xD3, MRM3r, (outs GR32:$dst), (ins GR32:$src1), - "rcr{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; + "rcr{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize32; def RCR64r1 : RI<0xD1, MRM3r, (outs GR64:$dst), (ins GR64:$src1), "rcr{q}\t$dst", [], IIC_SR>; @@ -417,13 +414,13 @@ def RCL8m1 : I<0xD0, MRM2m, (outs), (ins i8mem:$dst), def RCL8mi : Ii8<0xC0, MRM2m, (outs), (ins i8mem:$dst, i8imm:$cnt), "rcl{b}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>; def RCL16m1 : I<0xD1, MRM2m, (outs), (ins i16mem:$dst), - "rcl{w}\t$dst", [], IIC_SR>, OpSize; + "rcl{w}\t$dst", [], IIC_SR>, OpSize16; def RCL16mi : Ii8<0xC1, MRM2m, (outs), (ins i16mem:$dst, i8imm:$cnt), - "rcl{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize; + "rcl{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize16; def RCL32m1 : I<0xD1, MRM2m, (outs), (ins i32mem:$dst), - "rcl{l}\t$dst", [], IIC_SR>; + "rcl{l}\t$dst", [], IIC_SR>, OpSize32; def RCL32mi : Ii8<0xC1, MRM2m, (outs), (ins i32mem:$dst, i8imm:$cnt), - "rcl{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>; + "rcl{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize32; def RCL64m1 : RI<0xD1, MRM2m, (outs), (ins i64mem:$dst), "rcl{q}\t$dst", [], IIC_SR>; def RCL64mi : RIi8<0xC1, MRM2m, (outs), (ins i64mem:$dst, i8imm:$cnt), @@ -434,13 +431,13 @@ def RCR8m1 : I<0xD0, MRM3m, (outs), (ins i8mem:$dst), def RCR8mi : Ii8<0xC0, MRM3m, (outs), (ins i8mem:$dst, i8imm:$cnt), "rcr{b}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>; def RCR16m1 : I<0xD1, MRM3m, (outs), (ins i16mem:$dst), - "rcr{w}\t$dst", [], IIC_SR>, OpSize; + "rcr{w}\t$dst", [], IIC_SR>, OpSize16; def RCR16mi : Ii8<0xC1, MRM3m, (outs), (ins i16mem:$dst, i8imm:$cnt), - "rcr{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize; + "rcr{w}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize16; def RCR32m1 : I<0xD1, MRM3m, (outs), (ins i32mem:$dst), - "rcr{l}\t$dst", [], IIC_SR>; + "rcr{l}\t$dst", [], IIC_SR>, OpSize32; def RCR32mi : Ii8<0xC1, MRM3m, (outs), (ins i32mem:$dst, i8imm:$cnt), - "rcr{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>; + "rcr{l}\t{$cnt, $dst|$dst, $cnt}", [], IIC_SR>, OpSize32; def RCR64m1 : RI<0xD1, MRM3m, (outs), (ins i64mem:$dst), "rcr{q}\t$dst", [], IIC_SR>; def RCR64mi : RIi8<0xC1, MRM3m, (outs), (ins i64mem:$dst, i8imm:$cnt), @@ -450,18 +447,18 @@ let Uses = [CL] in { def RCL8mCL : I<0xD2, MRM2m, (outs), (ins i8mem:$dst), "rcl{b}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; def RCL16mCL : I<0xD3, MRM2m, (outs), (ins i16mem:$dst), - "rcl{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize; + "rcl{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize16; def RCL32mCL : I<0xD3, MRM2m, (outs), (ins i32mem:$dst), - "rcl{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; + "rcl{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize32; def RCL64mCL : RI<0xD3, MRM2m, (outs), (ins i64mem:$dst), "rcl{q}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; def RCR8mCL : I<0xD2, MRM3m, (outs), (ins i8mem:$dst), "rcr{b}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; def RCR16mCL : I<0xD3, MRM3m, (outs), (ins i16mem:$dst), - "rcr{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize; + "rcr{w}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize16; def RCR32mCL : I<0xD3, MRM3m, (outs), (ins i32mem:$dst), - "rcr{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; + "rcr{l}\t{%cl, $dst|$dst, cl}", [], IIC_SR>, OpSize32; def RCR64mCL : RI<0xD3, MRM3m, (outs), (ins i64mem:$dst), "rcr{q}\t{%cl, $dst|$dst, cl}", [], IIC_SR>; } @@ -476,10 +473,10 @@ def ROL8rCL : I<0xD2, MRM0r, (outs GR8 :$dst), (ins GR8 :$src1), [(set GR8:$dst, (rotl GR8:$src1, CL))], IIC_SR>; def ROL16rCL : I<0xD3, MRM0r, (outs GR16:$dst), (ins GR16:$src1), "rol{w}\t{%cl, $dst|$dst, cl}", - [(set GR16:$dst, (rotl GR16:$src1, CL))], IIC_SR>, OpSize; + [(set GR16:$dst, (rotl GR16:$src1, CL))], IIC_SR>, OpSize16; def ROL32rCL : I<0xD3, MRM0r, (outs GR32:$dst), (ins GR32:$src1), "rol{l}\t{%cl, $dst|$dst, cl}", - [(set GR32:$dst, (rotl GR32:$src1, CL))], IIC_SR>; + [(set GR32:$dst, (rotl GR32:$src1, CL))], IIC_SR>, OpSize32; def ROL64rCL : RI<0xD3, MRM0r, (outs GR64:$dst), (ins GR64:$src1), "rol{q}\t{%cl, $dst|$dst, cl}", [(set GR64:$dst, (rotl GR64:$src1, CL))], IIC_SR>; @@ -491,12 +488,11 @@ def ROL8ri : Ii8<0xC0, MRM0r, (outs GR8 :$dst), (ins GR8 :$src1, i8imm:$src2), def ROL16ri : Ii8<0xC1, MRM0r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$src2), "rol{w}\t{$src2, $dst|$dst, $src2}", [(set GR16:$dst, (rotl GR16:$src1, (i8 imm:$src2)))], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def ROL32ri : Ii8<0xC1, MRM0r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$src2), "rol{l}\t{$src2, $dst|$dst, $src2}", [(set GR32:$dst, (rotl GR32:$src1, (i8 imm:$src2)))], - IIC_SR>; + IIC_SR>, OpSize32; def ROL64ri : RIi8<0xC1, MRM0r, (outs GR64:$dst), (ins GR64:$src1, i8imm:$src2), "rol{q}\t{$src2, $dst|$dst, $src2}", @@ -511,11 +507,11 @@ def ROL8r1 : I<0xD0, MRM0r, (outs GR8 :$dst), (ins GR8 :$src1), def ROL16r1 : I<0xD1, MRM0r, (outs GR16:$dst), (ins GR16:$src1), "rol{w}\t$dst", [(set GR16:$dst, (rotl GR16:$src1, (i8 1)))], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def ROL32r1 : I<0xD1, MRM0r, (outs GR32:$dst), (ins GR32:$src1), "rol{l}\t$dst", [(set GR32:$dst, (rotl GR32:$src1, (i8 1)))], - IIC_SR>; + IIC_SR>, OpSize32; def ROL64r1 : RI<0xD1, MRM0r, (outs GR64:$dst), (ins GR64:$src1), "rol{q}\t$dst", [(set GR64:$dst, (rotl GR64:$src1, (i8 1)))], @@ -531,11 +527,11 @@ def ROL8mCL : I<0xD2, MRM0m, (outs), (ins i8mem :$dst), def ROL16mCL : I<0xD3, MRM0m, (outs), (ins i16mem:$dst), "rol{w}\t{%cl, $dst|$dst, cl}", [(store (rotl (loadi16 addr:$dst), CL), addr:$dst)], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def ROL32mCL : I<0xD3, MRM0m, (outs), (ins i32mem:$dst), "rol{l}\t{%cl, $dst|$dst, cl}", [(store (rotl (loadi32 addr:$dst), CL), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def ROL64mCL : RI<0xD3, MRM0m, (outs), (ins i64mem:$dst), "rol{q}\t{%cl, $dst|$dst, cl}", [(store (rotl (loadi64 addr:$dst), CL), addr:$dst)], @@ -548,12 +544,11 @@ def ROL8mi : Ii8<0xC0, MRM0m, (outs), (ins i8mem :$dst, i8imm:$src1), def ROL16mi : Ii8<0xC1, MRM0m, (outs), (ins i16mem:$dst, i8imm:$src1), "rol{w}\t{$src1, $dst|$dst, $src1}", [(store (rotl (loadi16 addr:$dst), (i8 imm:$src1)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def ROL32mi : Ii8<0xC1, MRM0m, (outs), (ins i32mem:$dst, i8imm:$src1), "rol{l}\t{$src1, $dst|$dst, $src1}", [(store (rotl (loadi32 addr:$dst), (i8 imm:$src1)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def ROL64mi : RIi8<0xC1, MRM0m, (outs), (ins i64mem:$dst, i8imm:$src1), "rol{q}\t{$src1, $dst|$dst, $src1}", [(store (rotl (loadi64 addr:$dst), (i8 imm:$src1)), addr:$dst)], @@ -567,12 +562,11 @@ def ROL8m1 : I<0xD0, MRM0m, (outs), (ins i8mem :$dst), def ROL16m1 : I<0xD1, MRM0m, (outs), (ins i16mem:$dst), "rol{w}\t$dst", [(store (rotl (loadi16 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def ROL32m1 : I<0xD1, MRM0m, (outs), (ins i32mem:$dst), "rol{l}\t$dst", [(store (rotl (loadi32 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def ROL64m1 : RI<0xD1, MRM0m, (outs), (ins i64mem:$dst), "rol{q}\t$dst", [(store (rotl (loadi64 addr:$dst), (i8 1)), addr:$dst)], @@ -586,10 +580,10 @@ def ROR8rCL : I<0xD2, MRM1r, (outs GR8 :$dst), (ins GR8 :$src1), [(set GR8:$dst, (rotr GR8:$src1, CL))], IIC_SR>; def ROR16rCL : I<0xD3, MRM1r, (outs GR16:$dst), (ins GR16:$src1), "ror{w}\t{%cl, $dst|$dst, cl}", - [(set GR16:$dst, (rotr GR16:$src1, CL))], IIC_SR>, OpSize; + [(set GR16:$dst, (rotr GR16:$src1, CL))], IIC_SR>, OpSize16; def ROR32rCL : I<0xD3, MRM1r, (outs GR32:$dst), (ins GR32:$src1), "ror{l}\t{%cl, $dst|$dst, cl}", - [(set GR32:$dst, (rotr GR32:$src1, CL))], IIC_SR>; + [(set GR32:$dst, (rotr GR32:$src1, CL))], IIC_SR>, OpSize32; def ROR64rCL : RI<0xD3, MRM1r, (outs GR64:$dst), (ins GR64:$src1), "ror{q}\t{%cl, $dst|$dst, cl}", [(set GR64:$dst, (rotr GR64:$src1, CL))], IIC_SR>; @@ -601,12 +595,11 @@ def ROR8ri : Ii8<0xC0, MRM1r, (outs GR8 :$dst), (ins GR8 :$src1, i8imm:$src2), def ROR16ri : Ii8<0xC1, MRM1r, (outs GR16:$dst), (ins GR16:$src1, i8imm:$src2), "ror{w}\t{$src2, $dst|$dst, $src2}", [(set GR16:$dst, (rotr GR16:$src1, (i8 imm:$src2)))], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def ROR32ri : Ii8<0xC1, MRM1r, (outs GR32:$dst), (ins GR32:$src1, i8imm:$src2), "ror{l}\t{$src2, $dst|$dst, $src2}", [(set GR32:$dst, (rotr GR32:$src1, (i8 imm:$src2)))], - IIC_SR>; + IIC_SR>, OpSize32; def ROR64ri : RIi8<0xC1, MRM1r, (outs GR64:$dst), (ins GR64:$src1, i8imm:$src2), "ror{q}\t{$src2, $dst|$dst, $src2}", @@ -621,11 +614,11 @@ def ROR8r1 : I<0xD0, MRM1r, (outs GR8 :$dst), (ins GR8 :$src1), def ROR16r1 : I<0xD1, MRM1r, (outs GR16:$dst), (ins GR16:$src1), "ror{w}\t$dst", [(set GR16:$dst, (rotr GR16:$src1, (i8 1)))], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def ROR32r1 : I<0xD1, MRM1r, (outs GR32:$dst), (ins GR32:$src1), "ror{l}\t$dst", [(set GR32:$dst, (rotr GR32:$src1, (i8 1)))], - IIC_SR>; + IIC_SR>, OpSize32; def ROR64r1 : RI<0xD1, MRM1r, (outs GR64:$dst), (ins GR64:$src1), "ror{q}\t$dst", [(set GR64:$dst, (rotr GR64:$src1, (i8 1)))], @@ -641,11 +634,11 @@ def ROR8mCL : I<0xD2, MRM1m, (outs), (ins i8mem :$dst), def ROR16mCL : I<0xD3, MRM1m, (outs), (ins i16mem:$dst), "ror{w}\t{%cl, $dst|$dst, cl}", [(store (rotr (loadi16 addr:$dst), CL), addr:$dst)], - IIC_SR>, OpSize; + IIC_SR>, OpSize16; def ROR32mCL : I<0xD3, MRM1m, (outs), (ins i32mem:$dst), "ror{l}\t{%cl, $dst|$dst, cl}", [(store (rotr (loadi32 addr:$dst), CL), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def ROR64mCL : RI<0xD3, MRM1m, (outs), (ins i64mem:$dst), "ror{q}\t{%cl, $dst|$dst, cl}", [(store (rotr (loadi64 addr:$dst), CL), addr:$dst)], @@ -658,12 +651,11 @@ def ROR8mi : Ii8<0xC0, MRM1m, (outs), (ins i8mem :$dst, i8imm:$src), def ROR16mi : Ii8<0xC1, MRM1m, (outs), (ins i16mem:$dst, i8imm:$src), "ror{w}\t{$src, $dst|$dst, $src}", [(store (rotr (loadi16 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def ROR32mi : Ii8<0xC1, MRM1m, (outs), (ins i32mem:$dst, i8imm:$src), "ror{l}\t{$src, $dst|$dst, $src}", [(store (rotr (loadi32 addr:$dst), (i8 imm:$src)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def ROR64mi : RIi8<0xC1, MRM1m, (outs), (ins i64mem:$dst, i8imm:$src), "ror{q}\t{$src, $dst|$dst, $src}", [(store (rotr (loadi64 addr:$dst), (i8 imm:$src)), addr:$dst)], @@ -677,12 +669,11 @@ def ROR8m1 : I<0xD0, MRM1m, (outs), (ins i8mem :$dst), def ROR16m1 : I<0xD1, MRM1m, (outs), (ins i16mem:$dst), "ror{w}\t$dst", [(store (rotr (loadi16 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>, - OpSize; + IIC_SR>, OpSize16; def ROR32m1 : I<0xD1, MRM1m, (outs), (ins i32mem:$dst), "ror{l}\t$dst", [(store (rotr (loadi32 addr:$dst), (i8 1)), addr:$dst)], - IIC_SR>; + IIC_SR>, OpSize32; def ROR64m1 : RI<0xD1, MRM1m, (outs), (ins i64mem:$dst), "ror{q}\t$dst", [(store (rotr (loadi64 addr:$dst), (i8 1)), addr:$dst)], @@ -702,23 +693,23 @@ def SHLD16rrCL : I<0xA5, MRMDestReg, (outs GR16:$dst), "shld{w}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(set GR16:$dst, (X86shld GR16:$src1, GR16:$src2, CL))], IIC_SHD16_REG_CL>, - TB, OpSize; + TB, OpSize16; def SHRD16rrCL : I<0xAD, MRMDestReg, (outs GR16:$dst), (ins GR16:$src1, GR16:$src2), "shrd{w}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(set GR16:$dst, (X86shrd GR16:$src1, GR16:$src2, CL))], IIC_SHD16_REG_CL>, - TB, OpSize; + TB, OpSize16; def SHLD32rrCL : I<0xA5, MRMDestReg, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2), "shld{l}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(set GR32:$dst, (X86shld GR32:$src1, GR32:$src2, CL))], - IIC_SHD32_REG_CL>, TB; + IIC_SHD32_REG_CL>, TB, OpSize32; def SHRD32rrCL : I<0xAD, MRMDestReg, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2), "shrd{l}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(set GR32:$dst, (X86shrd GR32:$src1, GR32:$src2, CL))], - IIC_SHD32_REG_CL>, TB; + IIC_SHD32_REG_CL>, TB, OpSize32; def SHLD64rrCL : RI<0xA5, MRMDestReg, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2), "shld{q}\t{%cl, $src2, $dst|$dst, $src2, cl}", @@ -740,28 +731,28 @@ def SHLD16rri8 : Ii8<0xA4, MRMDestReg, "shld{w}\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(set GR16:$dst, (X86shld GR16:$src1, GR16:$src2, (i8 imm:$src3)))], IIC_SHD16_REG_IM>, - TB, OpSize; + TB, OpSize16; def SHRD16rri8 : Ii8<0xAC, MRMDestReg, (outs GR16:$dst), (ins GR16:$src1, GR16:$src2, i8imm:$src3), "shrd{w}\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(set GR16:$dst, (X86shrd GR16:$src1, GR16:$src2, (i8 imm:$src3)))], IIC_SHD16_REG_IM>, - TB, OpSize; + TB, OpSize16; def SHLD32rri8 : Ii8<0xA4, MRMDestReg, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2, i8imm:$src3), "shld{l}\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(set GR32:$dst, (X86shld GR32:$src1, GR32:$src2, (i8 imm:$src3)))], IIC_SHD32_REG_IM>, - TB; + TB, OpSize32; def SHRD32rri8 : Ii8<0xAC, MRMDestReg, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2, i8imm:$src3), "shrd{l}\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(set GR32:$dst, (X86shrd GR32:$src1, GR32:$src2, (i8 imm:$src3)))], IIC_SHD32_REG_IM>, - TB; + TB, OpSize32; def SHLD64rri8 : RIi8<0xA4, MRMDestReg, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2, i8imm:$src3), @@ -784,20 +775,20 @@ let Uses = [CL] in { def SHLD16mrCL : I<0xA5, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2), "shld{w}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(store (X86shld (loadi16 addr:$dst), GR16:$src2, CL), - addr:$dst)], IIC_SHD16_MEM_CL>, TB, OpSize; + addr:$dst)], IIC_SHD16_MEM_CL>, TB, OpSize16; def SHRD16mrCL : I<0xAD, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2), "shrd{w}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(store (X86shrd (loadi16 addr:$dst), GR16:$src2, CL), - addr:$dst)], IIC_SHD16_MEM_CL>, TB, OpSize; + addr:$dst)], IIC_SHD16_MEM_CL>, TB, OpSize16; def SHLD32mrCL : I<0xA5, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2), "shld{l}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(store (X86shld (loadi32 addr:$dst), GR32:$src2, CL), - addr:$dst)], IIC_SHD32_MEM_CL>, TB; + addr:$dst)], IIC_SHD32_MEM_CL>, TB, OpSize32; def SHRD32mrCL : I<0xAD, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2), "shrd{l}\t{%cl, $src2, $dst|$dst, $src2, cl}", [(store (X86shrd (loadi32 addr:$dst), GR32:$src2, CL), - addr:$dst)], IIC_SHD32_MEM_CL>, TB; + addr:$dst)], IIC_SHD32_MEM_CL>, TB, OpSize32; def SHLD64mrCL : RI<0xA5, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src2), "shld{q}\t{%cl, $src2, $dst|$dst, $src2, cl}", @@ -815,14 +806,14 @@ def SHLD16mri8 : Ii8<0xA4, MRMDestMem, [(store (X86shld (loadi16 addr:$dst), GR16:$src2, (i8 imm:$src3)), addr:$dst)], IIC_SHD16_MEM_IM>, - TB, OpSize; + TB, OpSize16; def SHRD16mri8 : Ii8<0xAC, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2, i8imm:$src3), "shrd{w}\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(store (X86shrd (loadi16 addr:$dst), GR16:$src2, (i8 imm:$src3)), addr:$dst)], IIC_SHD16_MEM_IM>, - TB, OpSize; + TB, OpSize16; def SHLD32mri8 : Ii8<0xA4, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2, i8imm:$src3), @@ -830,14 +821,14 @@ def SHLD32mri8 : Ii8<0xA4, MRMDestMem, [(store (X86shld (loadi32 addr:$dst), GR32:$src2, (i8 imm:$src3)), addr:$dst)], IIC_SHD32_MEM_IM>, - TB; + TB, OpSize32; def SHRD32mri8 : Ii8<0xAC, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2, i8imm:$src3), "shrd{l}\t{$src3, $src2, $dst|$dst, $src2, $src3}", [(store (X86shrd (loadi32 addr:$dst), GR32:$src2, (i8 imm:$src3)), addr:$dst)], IIC_SHD32_MEM_IM>, - TB; + TB, OpSize32; def SHLD64mri8 : RIi8<0xA4, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src2, i8imm:$src3), @@ -905,8 +896,8 @@ let Predicates = [HasBMI2] in { defm SARX64 : bmi_shift<"sarx{q}", GR64, i64mem>, T8XS, VEX_W; defm SHRX32 : bmi_shift<"shrx{l}", GR32, i32mem>, T8XD; defm SHRX64 : bmi_shift<"shrx{q}", GR64, i64mem>, T8XD, VEX_W; - defm SHLX32 : bmi_shift<"shlx{l}", GR32, i32mem>, T8, OpSize; - defm SHLX64 : bmi_shift<"shlx{q}", GR64, i64mem>, T8, OpSize, VEX_W; + defm SHLX32 : bmi_shift<"shlx{l}", GR32, i32mem>, T8PD; + defm SHLX64 : bmi_shift<"shlx{q}", GR64, i64mem>, T8PD, VEX_W; // Prefer RORX which is non-destructive and doesn't update EFLAGS. let AddedComplexity = 10 in { diff --git a/lib/Target/X86/X86InstrSystem.td b/lib/Target/X86/X86InstrSystem.td index 2196dc3..9d3aa1c 100644 --- a/lib/Target/X86/X86InstrSystem.td +++ b/lib/Target/X86/X86InstrSystem.td @@ -61,11 +61,12 @@ def SYSENTER : I<0x34, RawFrm, (outs), (ins), "sysenter", [], def SYSEXIT : I<0x35, RawFrm, (outs), (ins), "sysexit{l}", [], IIC_SYS_ENTER_EXIT>, TB; -def SYSEXIT64 :RI<0x35, RawFrm, (outs), (ins), "sysexit{q}", []>, TB, - Requires<[In64BitMode]>; +def SYSEXIT64 :RI<0x35, RawFrm, (outs), (ins), "sysexit{q}", [], + IIC_SYS_ENTER_EXIT>, TB, Requires<[In64BitMode]>; -def IRET16 : I<0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>, OpSize; -def IRET32 : I<0xcf, RawFrm, (outs), (ins), "iret{l|d}", [], IIC_IRET>; +def IRET16 : I<0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>, OpSize16; +def IRET32 : I<0xcf, RawFrm, (outs), (ins), "iret{l|d}", [], IIC_IRET>, + OpSize32; def IRET64 : RI<0xcf, RawFrm, (outs), (ins), "iretq", [], IIC_IRET>, Requires<[In64BitMode]>; } // SchedRW @@ -80,44 +81,41 @@ def IN8rr : I<0xEC, RawFrm, (outs), (ins), "in{b}\t{%dx, %al|al, dx}", [], IIC_IN_RR>; let Defs = [AX], Uses = [DX] in def IN16rr : I<0xED, RawFrm, (outs), (ins), - "in{w}\t{%dx, %ax|ax, dx}", [], IIC_IN_RR>, OpSize; + "in{w}\t{%dx, %ax|ax, dx}", [], IIC_IN_RR>, OpSize16; let Defs = [EAX], Uses = [DX] in def IN32rr : I<0xED, RawFrm, (outs), (ins), - "in{l}\t{%dx, %eax|eax, dx}", [], IIC_IN_RR>; + "in{l}\t{%dx, %eax|eax, dx}", [], IIC_IN_RR>, OpSize32; let Defs = [AL] in def IN8ri : Ii8<0xE4, RawFrm, (outs), (ins i8imm:$port), "in{b}\t{$port, %al|al, $port}", [], IIC_IN_RI>; let Defs = [AX] in def IN16ri : Ii8<0xE5, RawFrm, (outs), (ins i8imm:$port), - "in{w}\t{$port, %ax|ax, $port}", [], IIC_IN_RI>, OpSize; + "in{w}\t{$port, %ax|ax, $port}", [], IIC_IN_RI>, OpSize16; let Defs = [EAX] in def IN32ri : Ii8<0xE5, RawFrm, (outs), (ins i8imm:$port), - "in{l}\t{$port, %eax|eax, $port}", [], IIC_IN_RI>; + "in{l}\t{$port, %eax|eax, $port}", [], IIC_IN_RI>, OpSize32; let Uses = [DX, AL] in def OUT8rr : I<0xEE, RawFrm, (outs), (ins), "out{b}\t{%al, %dx|dx, al}", [], IIC_OUT_RR>; let Uses = [DX, AX] in def OUT16rr : I<0xEF, RawFrm, (outs), (ins), - "out{w}\t{%ax, %dx|dx, ax}", [], IIC_OUT_RR>, OpSize; + "out{w}\t{%ax, %dx|dx, ax}", [], IIC_OUT_RR>, OpSize16; let Uses = [DX, EAX] in def OUT32rr : I<0xEF, RawFrm, (outs), (ins), - "out{l}\t{%eax, %dx|dx, eax}", [], IIC_OUT_RR>; + "out{l}\t{%eax, %dx|dx, eax}", [], IIC_OUT_RR>, OpSize32; let Uses = [AL] in def OUT8ir : Ii8<0xE6, RawFrm, (outs), (ins i8imm:$port), "out{b}\t{%al, $port|$port, al}", [], IIC_OUT_IR>; let Uses = [AX] in def OUT16ir : Ii8<0xE7, RawFrm, (outs), (ins i8imm:$port), - "out{w}\t{%ax, $port|$port, ax}", [], IIC_OUT_IR>, OpSize; + "out{w}\t{%ax, $port|$port, ax}", [], IIC_OUT_IR>, OpSize16; let Uses = [EAX] in def OUT32ir : Ii8<0xE7, RawFrm, (outs), (ins i8imm:$port), - "out{l}\t{%eax, $port|$port, eax}", [], IIC_OUT_IR>; + "out{l}\t{%eax, $port|$port, eax}", [], IIC_OUT_IR>, OpSize32; -def IN8 : I<0x6C, RawFrm, (outs), (ins), "ins{b}", [], IIC_INS>; -def IN16 : I<0x6D, RawFrm, (outs), (ins), "ins{w}", [], IIC_INS>, OpSize; -def IN32 : I<0x6D, RawFrm, (outs), (ins), "ins{l}", [], IIC_INS>; } // SchedRW //===----------------------------------------------------------------------===// @@ -125,14 +123,18 @@ def IN32 : I<0x6D, RawFrm, (outs), (ins), "ins{l}", [], IIC_INS>; let SchedRW = [WriteSystem] in { def MOV32rd : I<0x21, MRMDestReg, (outs GR32:$dst), (ins DEBUG_REG:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_DR>, TB; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_DR>, TB, + Requires<[Not64BitMode]>; def MOV64rd : I<0x21, MRMDestReg, (outs GR64:$dst), (ins DEBUG_REG:$src), - "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_DR>, TB; - + "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_DR>, TB, + Requires<[In64BitMode]>; + def MOV32dr : I<0x23, MRMSrcReg, (outs DEBUG_REG:$dst), (ins GR32:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_DR_REG>, TB; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_DR_REG>, TB, + Requires<[Not64BitMode]>; def MOV64dr : I<0x23, MRMSrcReg, (outs DEBUG_REG:$dst), (ins GR64:$src), - "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_DR_REG>, TB; + "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_DR_REG>, TB, + Requires<[In64BitMode]>; } // SchedRW //===----------------------------------------------------------------------===// @@ -140,14 +142,18 @@ def MOV64dr : I<0x23, MRMSrcReg, (outs DEBUG_REG:$dst), (ins GR64:$src), let SchedRW = [WriteSystem] in { def MOV32rc : I<0x20, MRMDestReg, (outs GR32:$dst), (ins CONTROL_REG:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_CR>, TB; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_CR>, TB, + Requires<[Not64BitMode]>; def MOV64rc : I<0x20, MRMDestReg, (outs GR64:$dst), (ins CONTROL_REG:$src), - "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_CR>, TB; - + "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_CR>, TB, + Requires<[In64BitMode]>; + def MOV32cr : I<0x22, MRMSrcReg, (outs CONTROL_REG:$dst), (ins GR32:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_CR_REG>, TB; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_CR_REG>, TB, + Requires<[Not64BitMode]>; def MOV64cr : I<0x22, MRMSrcReg, (outs CONTROL_REG:$dst), (ins GR64:$src), - "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_CR_REG>, TB; + "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_CR_REG>, TB, + Requires<[In64BitMode]>; } // SchedRW //===----------------------------------------------------------------------===// @@ -167,30 +173,30 @@ def GS_PREFIX : I<0x65, RawFrm, (outs), (ins), "gs", []>; let SchedRW = [WriteMove] in { def MOV16rs : I<0x8C, MRMDestReg, (outs GR16:$dst), (ins SEGMENT_REG:$src), - "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_SR>, OpSize; + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_SR>, OpSize16; def MOV32rs : I<0x8C, MRMDestReg, (outs GR32:$dst), (ins SEGMENT_REG:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_SR>; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_SR>, OpSize32; def MOV64rs : RI<0x8C, MRMDestReg, (outs GR64:$dst), (ins SEGMENT_REG:$src), "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_REG_SR>; def MOV16ms : I<0x8C, MRMDestMem, (outs i16mem:$dst), (ins SEGMENT_REG:$src), - "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_MEM_SR>, OpSize; + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_MEM_SR>, OpSize16; def MOV32ms : I<0x8C, MRMDestMem, (outs i32mem:$dst), (ins SEGMENT_REG:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_MEM_SR>; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_MEM_SR>, OpSize32; def MOV64ms : RI<0x8C, MRMDestMem, (outs i64mem:$dst), (ins SEGMENT_REG:$src), "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_MEM_SR>; def MOV16sr : I<0x8E, MRMSrcReg, (outs SEGMENT_REG:$dst), (ins GR16:$src), - "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_REG>, OpSize; + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_REG>, OpSize16; def MOV32sr : I<0x8E, MRMSrcReg, (outs SEGMENT_REG:$dst), (ins GR32:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_REG>; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_REG>, OpSize32; def MOV64sr : RI<0x8E, MRMSrcReg, (outs SEGMENT_REG:$dst), (ins GR64:$src), "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_REG>; def MOV16sm : I<0x8E, MRMSrcMem, (outs SEGMENT_REG:$dst), (ins i16mem:$src), - "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_MEM>, OpSize; + "mov{w}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_MEM>, OpSize16; def MOV32sm : I<0x8E, MRMSrcMem, (outs SEGMENT_REG:$dst), (ins i32mem:$src), - "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_MEM>; + "mov{l}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_MEM>, OpSize32; def MOV64sm : RI<0x8E, MRMSrcMem, (outs SEGMENT_REG:$dst), (ins i64mem:$src), "mov{q}\t{$src, $dst|$dst, $src}", [], IIC_MOV_SR_MEM>; } // SchedRW @@ -202,15 +208,19 @@ let SchedRW = [WriteSystem] in { def SWAPGS : I<0x01, MRM_F8, (outs), (ins), "swapgs", [], IIC_SWAPGS>, TB; def LAR16rm : I<0x02, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), - "lar{w}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RM>, TB, OpSize; + "lar{w}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RM>, TB, + OpSize16; def LAR16rr : I<0x02, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), - "lar{w}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RR>, TB, OpSize; + "lar{w}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RR>, TB, + OpSize16; // i16mem operand in LAR32rm and GR32 operand in LAR32rr is not a typo. def LAR32rm : I<0x02, MRMSrcMem, (outs GR32:$dst), (ins i16mem:$src), - "lar{l}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RM>, TB; + "lar{l}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RM>, TB, + OpSize32; def LAR32rr : I<0x02, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), - "lar{l}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RR>, TB; + "lar{l}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RR>, TB, + OpSize32; // i16mem operand in LAR64rm and GR32 operand in LAR32rr is not a typo. def LAR64rm : RI<0x02, MRMSrcMem, (outs GR64:$dst), (ins i16mem:$src), "lar{q}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RM>, TB; @@ -218,13 +228,17 @@ def LAR64rr : RI<0x02, MRMSrcReg, (outs GR64:$dst), (ins GR32:$src), "lar{q}\t{$src, $dst|$dst, $src}", [], IIC_LAR_RR>, TB; def LSL16rm : I<0x03, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), - "lsl{w}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RM>, TB, OpSize; + "lsl{w}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RM>, TB, + OpSize16; def LSL16rr : I<0x03, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), - "lsl{w}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RR>, TB, OpSize; + "lsl{w}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RR>, TB, + OpSize16; def LSL32rm : I<0x03, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), - "lsl{l}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RM>, TB; + "lsl{l}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RM>, TB, + OpSize32; def LSL32rr : I<0x03, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), - "lsl{l}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RR>, TB; + "lsl{l}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RR>, TB, + OpSize32; def LSL64rm : RI<0x03, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), "lsl{q}\t{$src, $dst|$dst, $src}", [], IIC_LSL_RM>, TB; def LSL64rr : RI<0x03, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), @@ -234,9 +248,9 @@ def INVLPG : I<0x01, MRM7m, (outs), (ins i8mem:$addr), "invlpg\t$addr", [], IIC_INVLPG>, TB; def STR16r : I<0x00, MRM1r, (outs GR16:$dst), (ins), - "str{w}\t$dst", [], IIC_STR>, TB, OpSize; + "str{w}\t$dst", [], IIC_STR>, TB, OpSize16; def STR32r : I<0x00, MRM1r, (outs GR32:$dst), (ins), - "str{l}\t$dst", [], IIC_STR>, TB; + "str{l}\t$dst", [], IIC_STR>, TB, OpSize32; def STR64r : RI<0x00, MRM1r, (outs GR64:$dst), (ins), "str{q}\t$dst", [], IIC_STR>, TB; def STRm : I<0x00, MRM1m, (outs i16mem:$dst), (ins), @@ -248,105 +262,115 @@ def LTRm : I<0x00, MRM3m, (outs), (ins i16mem:$src), "ltr{w}\t$src", [], IIC_LTR>, TB; def PUSHCS16 : I<0x0E, RawFrm, (outs), (ins), - "push{w}\t{%cs|cs}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>, - OpSize; + "push{w}\t{%cs|cs}", [], IIC_PUSH_SR>, + OpSize16, Requires<[Not64BitMode]>; def PUSHCS32 : I<0x0E, RawFrm, (outs), (ins), - "push{l}\t{%cs|cs}", [], IIC_PUSH_CS>, Requires<[In32BitMode]>; + "push{l}\t{%cs|cs}", [], IIC_PUSH_CS>, + OpSize32, Requires<[Not64BitMode]>; def PUSHSS16 : I<0x16, RawFrm, (outs), (ins), - "push{w}\t{%ss|ss}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>, - OpSize; + "push{w}\t{%ss|ss}", [], IIC_PUSH_SR>, + OpSize16, Requires<[Not64BitMode]>; def PUSHSS32 : I<0x16, RawFrm, (outs), (ins), - "push{l}\t{%ss|ss}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>; + "push{l}\t{%ss|ss}", [], IIC_PUSH_SR>, + OpSize32, Requires<[Not64BitMode]>; def PUSHDS16 : I<0x1E, RawFrm, (outs), (ins), - "push{w}\t{%ds|ds}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>, - OpSize; + "push{w}\t{%ds|ds}", [], IIC_PUSH_SR>, + OpSize16, Requires<[Not64BitMode]>; def PUSHDS32 : I<0x1E, RawFrm, (outs), (ins), - "push{l}\t{%ds|ds}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>; + "push{l}\t{%ds|ds}", [], IIC_PUSH_SR>, + OpSize32, Requires<[Not64BitMode]>; def PUSHES16 : I<0x06, RawFrm, (outs), (ins), - "push{w}\t{%es|es}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>, - OpSize; + "push{w}\t{%es|es}", [], IIC_PUSH_SR>, + OpSize16, Requires<[Not64BitMode]>; def PUSHES32 : I<0x06, RawFrm, (outs), (ins), - "push{l}\t{%es|es}", [], IIC_PUSH_SR>, Requires<[In32BitMode]>; - + "push{l}\t{%es|es}", [], IIC_PUSH_SR>, + OpSize32, Requires<[Not64BitMode]>; def PUSHFS16 : I<0xa0, RawFrm, (outs), (ins), - "push{w}\t{%fs|fs}", [], IIC_PUSH_SR>, OpSize, TB; + "push{w}\t{%fs|fs}", [], IIC_PUSH_SR>, OpSize16, TB; def PUSHFS32 : I<0xa0, RawFrm, (outs), (ins), - "push{l}\t{%fs|fs}", [], IIC_PUSH_SR>, TB, Requires<[In32BitMode]>; + "push{l}\t{%fs|fs}", [], IIC_PUSH_SR>, TB, + OpSize32, Requires<[Not64BitMode]>; def PUSHGS16 : I<0xa8, RawFrm, (outs), (ins), - "push{w}\t{%gs|gs}", [], IIC_PUSH_SR>, OpSize, TB; + "push{w}\t{%gs|gs}", [], IIC_PUSH_SR>, OpSize16, TB; def PUSHGS32 : I<0xa8, RawFrm, (outs), (ins), - "push{l}\t{%gs|gs}", [], IIC_PUSH_SR>, TB, Requires<[In32BitMode]>; - + "push{l}\t{%gs|gs}", [], IIC_PUSH_SR>, TB, + OpSize32, Requires<[Not64BitMode]>; def PUSHFS64 : I<0xa0, RawFrm, (outs), (ins), - "push{q}\t{%fs|fs}", [], IIC_PUSH_SR>, TB; + "push{q}\t{%fs|fs}", [], IIC_PUSH_SR>, TB, + OpSize32, Requires<[In64BitMode]>; def PUSHGS64 : I<0xa8, RawFrm, (outs), (ins), - "push{q}\t{%gs|gs}", [], IIC_PUSH_SR>, TB; + "push{q}\t{%gs|gs}", [], IIC_PUSH_SR>, TB, + OpSize32, Requires<[In64BitMode]>; // No "pop cs" instruction. def POPSS16 : I<0x17, RawFrm, (outs), (ins), "pop{w}\t{%ss|ss}", [], IIC_POP_SR_SS>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def POPSS32 : I<0x17, RawFrm, (outs), (ins), "pop{l}\t{%ss|ss}", [], IIC_POP_SR_SS>, - Requires<[In32BitMode]>; - + OpSize32, Requires<[Not64BitMode]>; + def POPDS16 : I<0x1F, RawFrm, (outs), (ins), "pop{w}\t{%ds|ds}", [], IIC_POP_SR>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def POPDS32 : I<0x1F, RawFrm, (outs), (ins), "pop{l}\t{%ds|ds}", [], IIC_POP_SR>, - Requires<[In32BitMode]>; - + OpSize32, Requires<[Not64BitMode]>; + def POPES16 : I<0x07, RawFrm, (outs), (ins), "pop{w}\t{%es|es}", [], IIC_POP_SR>, - OpSize, Requires<[In32BitMode]>; + OpSize16, Requires<[Not64BitMode]>; def POPES32 : I<0x07, RawFrm, (outs), (ins), "pop{l}\t{%es|es}", [], IIC_POP_SR>, - Requires<[In32BitMode]>; - + OpSize32, Requires<[Not64BitMode]>; + def POPFS16 : I<0xa1, RawFrm, (outs), (ins), - "pop{w}\t{%fs|fs}", [], IIC_POP_SR>, OpSize, TB; + "pop{w}\t{%fs|fs}", [], IIC_POP_SR>, OpSize16, TB; def POPFS32 : I<0xa1, RawFrm, (outs), (ins), - "pop{l}\t{%fs|fs}", [], IIC_POP_SR>, TB, Requires<[In32BitMode]>; + "pop{l}\t{%fs|fs}", [], IIC_POP_SR>, TB, + OpSize32, Requires<[Not64BitMode]>; def POPFS64 : I<0xa1, RawFrm, (outs), (ins), - "pop{q}\t{%fs|fs}", [], IIC_POP_SR>, TB; - + "pop{q}\t{%fs|fs}", [], IIC_POP_SR>, TB, + OpSize32, Requires<[In64BitMode]>; + def POPGS16 : I<0xa9, RawFrm, (outs), (ins), - "pop{w}\t{%gs|gs}", [], IIC_POP_SR>, OpSize, TB; + "pop{w}\t{%gs|gs}", [], IIC_POP_SR>, OpSize16, TB; def POPGS32 : I<0xa9, RawFrm, (outs), (ins), - "pop{l}\t{%gs|gs}", [], IIC_POP_SR>, TB, Requires<[In32BitMode]>; + "pop{l}\t{%gs|gs}", [], IIC_POP_SR>, TB, + OpSize32, Requires<[Not64BitMode]>; def POPGS64 : I<0xa9, RawFrm, (outs), (ins), - "pop{q}\t{%gs|gs}", [], IIC_POP_SR>, TB; - + "pop{q}\t{%gs|gs}", [], IIC_POP_SR>, TB, + OpSize32, Requires<[In64BitMode]>; + def LDS16rm : I<0xc5, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src), - "lds{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, OpSize; + "lds{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, OpSize16; def LDS32rm : I<0xc5, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src), - "lds{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>; + "lds{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, OpSize32; def LSS16rm : I<0xb2, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src), - "lss{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize; + "lss{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize16; def LSS32rm : I<0xb2, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src), - "lss{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB; + "lss{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize32; def LSS64rm : RI<0xb2, MRMSrcMem, (outs GR64:$dst), (ins opaque80mem:$src), "lss{q}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB; def LES16rm : I<0xc4, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src), - "les{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, OpSize; + "les{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, OpSize16; def LES32rm : I<0xc4, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src), - "les{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>; + "les{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, OpSize32; def LFS16rm : I<0xb4, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src), - "lfs{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize; + "lfs{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize16; def LFS32rm : I<0xb4, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src), - "lfs{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB; + "lfs{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize32; def LFS64rm : RI<0xb4, MRMSrcMem, (outs GR64:$dst), (ins opaque80mem:$src), "lfs{q}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB; def LGS16rm : I<0xb5, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src), - "lgs{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize; + "lgs{w}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize16; def LGS32rm : I<0xb5, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src), - "lgs{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB; + "lgs{l}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB, OpSize32; def LGS64rm : RI<0xb5, MRMSrcMem, (outs GR64:$dst), (ins opaque80mem:$src), "lgs{q}\t{$src, $dst|$dst, $src}", [], IIC_LXS>, TB; @@ -367,19 +391,23 @@ def VERWm : I<0x00, MRM5m, (outs), (ins i16mem:$seg), let SchedRW = [WriteSystem] in { def SGDT16m : I<0x01, MRM0m, (outs opaque48mem:$dst), (ins), - "sgdt{w}\t$dst", [], IIC_SGDT>, TB, OpSize, Requires<[In32BitMode]>; -def SGDTm : I<0x01, MRM0m, (outs opaque48mem:$dst), (ins), - "sgdt\t$dst", [], IIC_SGDT>, TB; + "sgdt{w}\t$dst", [], IIC_SGDT>, TB, OpSize16, Requires<[Not64BitMode]>; +def SGDT32m : I<0x01, MRM0m, (outs opaque48mem:$dst), (ins), + "sgdt{l}\t$dst", [], IIC_SGDT>, OpSize32, TB, Requires <[Not64BitMode]>; +def SGDT64m : I<0x01, MRM0m, (outs opaque80mem:$dst), (ins), + "sgdt{q}\t$dst", [], IIC_SGDT>, TB, Requires <[In64BitMode]>; def SIDT16m : I<0x01, MRM1m, (outs opaque48mem:$dst), (ins), - "sidt{w}\t$dst", [], IIC_SIDT>, TB, OpSize, Requires<[In32BitMode]>; -def SIDTm : I<0x01, MRM1m, (outs opaque48mem:$dst), (ins), - "sidt\t$dst", []>, TB; + "sidt{w}\t$dst", [], IIC_SIDT>, TB, OpSize16, Requires<[Not64BitMode]>; +def SIDT32m : I<0x01, MRM1m, (outs opaque48mem:$dst), (ins), + "sidt{l}\t$dst", []>, OpSize32, TB, Requires <[Not64BitMode]>; +def SIDT64m : I<0x01, MRM1m, (outs opaque80mem:$dst), (ins), + "sidt{q}\t$dst", []>, TB, Requires <[In64BitMode]>; def SLDT16r : I<0x00, MRM0r, (outs GR16:$dst), (ins), - "sldt{w}\t$dst", [], IIC_SLDT>, TB, OpSize; + "sldt{w}\t$dst", [], IIC_SLDT>, TB, OpSize16; def SLDT16m : I<0x00, MRM0m, (outs i16mem:$dst), (ins), "sldt{w}\t$dst", [], IIC_SLDT>, TB; def SLDT32r : I<0x00, MRM0r, (outs GR32:$dst), (ins), - "sldt{l}\t$dst", [], IIC_SLDT>, TB; + "sldt{l}\t$dst", [], IIC_SLDT>, OpSize32, TB; // LLDT is not interpreted specially in 64-bit mode because there is no sign // extension. @@ -389,13 +417,17 @@ def SLDT64m : RI<0x00, MRM0m, (outs i16mem:$dst), (ins), "sldt{q}\t$dst", [], IIC_SLDT>, TB; def LGDT16m : I<0x01, MRM2m, (outs), (ins opaque48mem:$src), - "lgdt{w}\t$src", [], IIC_LGDT>, TB, OpSize, Requires<[In32BitMode]>; -def LGDTm : I<0x01, MRM2m, (outs), (ins opaque48mem:$src), - "lgdt\t$src", [], IIC_LGDT>, TB; + "lgdt{w}\t$src", [], IIC_LGDT>, TB, OpSize16, Requires<[Not64BitMode]>; +def LGDT32m : I<0x01, MRM2m, (outs), (ins opaque48mem:$src), + "lgdt{l}\t$src", [], IIC_LGDT>, OpSize32, TB, Requires<[Not64BitMode]>; +def LGDT64m : I<0x01, MRM2m, (outs), (ins opaque80mem:$src), + "lgdt{q}\t$src", [], IIC_LGDT>, TB, Requires<[In64BitMode]>; def LIDT16m : I<0x01, MRM3m, (outs), (ins opaque48mem:$src), - "lidt{w}\t$src", [], IIC_LIDT>, TB, OpSize, Requires<[In32BitMode]>; -def LIDTm : I<0x01, MRM3m, (outs), (ins opaque48mem:$src), - "lidt\t$src", [], IIC_LIDT>, TB; + "lidt{w}\t$src", [], IIC_LIDT>, TB, OpSize16, Requires<[Not64BitMode]>; +def LIDT32m : I<0x01, MRM3m, (outs), (ins opaque48mem:$src), + "lidt{l}\t$src", [], IIC_LIDT>, OpSize32, TB, Requires<[Not64BitMode]>; +def LIDT64m : I<0x01, MRM3m, (outs), (ins opaque80mem:$src), + "lidt{q}\t$src", [], IIC_LIDT>, TB, Requires<[In64BitMode]>; def LLDT16r : I<0x00, MRM2r, (outs), (ins GR16:$src), "lldt{w}\t$src", [], IIC_LLDT_REG>, TB; def LLDT16m : I<0x00, MRM2m, (outs), (ins i16mem:$src), @@ -410,9 +442,9 @@ def RDMSR : I<0x32, RawFrm, (outs), (ins), "rdmsr", [], IIC_RDMSR>, TB; def RDPMC : I<0x33, RawFrm, (outs), (ins), "rdpmc", [], IIC_RDPMC>, TB; def SMSW16r : I<0x01, MRM4r, (outs GR16:$dst), (ins), - "smsw{w}\t$dst", [], IIC_SMSW>, OpSize, TB; + "smsw{w}\t$dst", [], IIC_SMSW>, OpSize16, TB; def SMSW32r : I<0x01, MRM4r, (outs GR32:$dst), (ins), - "smsw{l}\t$dst", [], IIC_SMSW>, TB; + "smsw{l}\t$dst", [], IIC_SMSW>, OpSize32, TB; // no m form encodable; use SMSW16m def SMSW64r : RI<0x01, MRM4r, (outs GR64:$dst), (ins), "smsw{q}\t$dst", [], IIC_SMSW>, TB; @@ -425,8 +457,13 @@ def LMSW16r : I<0x01, MRM6r, (outs), (ins GR16:$src), "lmsw{w}\t$src", [], IIC_LMSW_MEM>, TB; def LMSW16m : I<0x01, MRM6m, (outs), (ins i16mem:$src), "lmsw{w}\t$src", [], IIC_LMSW_REG>, TB; - -def CPUID : I<0xA2, RawFrm, (outs), (ins), "cpuid", [], IIC_CPUID>, TB; + +let Defs = [EAX, EBX, ECX, EDX], Uses = [EAX, ECX] in + def CPUID32 : I<0xA2, RawFrm, (outs), (ins), "cpuid", [], IIC_CPUID>, TB, + Requires<[Not64BitMode]>; +let Defs = [RAX, RBX, RCX, RDX], Uses = [RAX, RCX] in + def CPUID64 : I<0xA2, RawFrm, (outs), (ins), "cpuid", [], IIC_CPUID>, TB, + Requires<[In64BitMode]>; } // SchedRW //===----------------------------------------------------------------------===// @@ -448,77 +485,77 @@ let Uses = [RDX, RAX, RCX] in let Uses = [RDX, RAX] in { def XSAVE : I<0xAE, MRM4m, (outs opaque512mem:$dst), (ins), "xsave\t$dst", []>, TB; - def XSAVE64 : I<0xAE, MRM4m, (outs opaque512mem:$dst), (ins), - "xsave{q|64}\t$dst", []>, TB, REX_W, Requires<[In64BitMode]>; + def XSAVE64 : RI<0xAE, MRM4m, (outs opaque512mem:$dst), (ins), + "xsave{q|64}\t$dst", []>, TB, Requires<[In64BitMode]>; def XRSTOR : I<0xAE, MRM5m, (outs), (ins opaque512mem:$dst), "xrstor\t$dst", []>, TB; - def XRSTOR64 : I<0xAE, MRM5m, (outs), (ins opaque512mem:$dst), - "xrstor{q|64}\t$dst", []>, TB, REX_W, Requires<[In64BitMode]>; + def XRSTOR64 : RI<0xAE, MRM5m, (outs), (ins opaque512mem:$dst), + "xrstor{q|64}\t$dst", []>, TB, Requires<[In64BitMode]>; def XSAVEOPT : I<0xAE, MRM6m, (outs opaque512mem:$dst), (ins), "xsaveopt\t$dst", []>, TB; - def XSAVEOPT64 : I<0xAE, MRM6m, (outs opaque512mem:$dst), (ins), - "xsaveopt{q|64}\t$dst", []>, TB, REX_W, Requires<[In64BitMode]>; + def XSAVEOPT64 : RI<0xAE, MRM6m, (outs opaque512mem:$dst), (ins), + "xsaveopt{q|64}\t$dst", []>, TB, Requires<[In64BitMode]>; } } // SchedRW //===----------------------------------------------------------------------===// // VIA PadLock crypto instructions let Defs = [RAX, RDI], Uses = [RDX, RDI] in - def XSTORE : I<0xc0, RawFrm, (outs), (ins), "xstore", []>, A7; + def XSTORE : I<0xa7, MRM_C0, (outs), (ins), "xstore", []>, TB; def : InstAlias<"xstorerng", (XSTORE)>; let Defs = [RSI, RDI], Uses = [RBX, RDX, RSI, RDI] in { - def XCRYPTECB : I<0xc8, RawFrm, (outs), (ins), "xcryptecb", []>, A7; - def XCRYPTCBC : I<0xd0, RawFrm, (outs), (ins), "xcryptcbc", []>, A7; - def XCRYPTCTR : I<0xd8, RawFrm, (outs), (ins), "xcryptctr", []>, A7; - def XCRYPTCFB : I<0xe0, RawFrm, (outs), (ins), "xcryptcfb", []>, A7; - def XCRYPTOFB : I<0xe8, RawFrm, (outs), (ins), "xcryptofb", []>, A7; + def XCRYPTECB : I<0xa7, MRM_C8, (outs), (ins), "xcryptecb", []>, TB; + def XCRYPTCBC : I<0xa7, MRM_D0, (outs), (ins), "xcryptcbc", []>, TB; + def XCRYPTCTR : I<0xa7, MRM_D8, (outs), (ins), "xcryptctr", []>, TB; + def XCRYPTCFB : I<0xa7, MRM_E0, (outs), (ins), "xcryptcfb", []>, TB; + def XCRYPTOFB : I<0xa7, MRM_E8, (outs), (ins), "xcryptofb", []>, TB; } let Defs = [RAX, RSI, RDI], Uses = [RAX, RSI, RDI] in { - def XSHA1 : I<0xc8, RawFrm, (outs), (ins), "xsha1", []>, A6; - def XSHA256 : I<0xd0, RawFrm, (outs), (ins), "xsha256", []>, A6; + def XSHA1 : I<0xa6, MRM_C8, (outs), (ins), "xsha1", []>, TB; + def XSHA256 : I<0xa6, MRM_D0, (outs), (ins), "xsha256", []>, TB; } let Defs = [RAX, RDX, RSI], Uses = [RAX, RSI] in - def MONTMUL : I<0xc0, RawFrm, (outs), (ins), "montmul", []>, A6; + def MONTMUL : I<0xa6, MRM_C0, (outs), (ins), "montmul", []>, TB; //===----------------------------------------------------------------------===// // FS/GS Base Instructions let Predicates = [HasFSGSBase, In64BitMode] in { def RDFSBASE : I<0xAE, MRM0r, (outs GR32:$dst), (ins), "rdfsbase{l}\t$dst", - [(set GR32:$dst, (int_x86_rdfsbase_32))]>, TB, XS; + [(set GR32:$dst, (int_x86_rdfsbase_32))]>, XS; def RDFSBASE64 : RI<0xAE, MRM0r, (outs GR64:$dst), (ins), "rdfsbase{q}\t$dst", - [(set GR64:$dst, (int_x86_rdfsbase_64))]>, TB, XS; + [(set GR64:$dst, (int_x86_rdfsbase_64))]>, XS; def RDGSBASE : I<0xAE, MRM1r, (outs GR32:$dst), (ins), "rdgsbase{l}\t$dst", - [(set GR32:$dst, (int_x86_rdgsbase_32))]>, TB, XS; + [(set GR32:$dst, (int_x86_rdgsbase_32))]>, XS; def RDGSBASE64 : RI<0xAE, MRM1r, (outs GR64:$dst), (ins), "rdgsbase{q}\t$dst", - [(set GR64:$dst, (int_x86_rdgsbase_64))]>, TB, XS; + [(set GR64:$dst, (int_x86_rdgsbase_64))]>, XS; def WRFSBASE : I<0xAE, MRM2r, (outs), (ins GR32:$src), "wrfsbase{l}\t$src", - [(int_x86_wrfsbase_32 GR32:$src)]>, TB, XS; + [(int_x86_wrfsbase_32 GR32:$src)]>, XS; def WRFSBASE64 : RI<0xAE, MRM2r, (outs), (ins GR64:$src), "wrfsbase{q}\t$src", - [(int_x86_wrfsbase_64 GR64:$src)]>, TB, XS; + [(int_x86_wrfsbase_64 GR64:$src)]>, XS; def WRGSBASE : I<0xAE, MRM3r, (outs), (ins GR32:$src), "wrgsbase{l}\t$src", - [(int_x86_wrgsbase_32 GR32:$src)]>, TB, XS; + [(int_x86_wrgsbase_32 GR32:$src)]>, XS; def WRGSBASE64 : RI<0xAE, MRM3r, (outs), (ins GR64:$src), "wrgsbase{q}\t$src", - [(int_x86_wrgsbase_64 GR64:$src)]>, TB, XS; + [(int_x86_wrgsbase_64 GR64:$src)]>, XS; } //===----------------------------------------------------------------------===// // INVPCID Instruction def INVPCID32 : I<0x82, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), - "invpcid\t{$src2, $src1|$src1, $src2}", []>, OpSize, T8, - Requires<[In32BitMode]>; + "invpcid\t{$src2, $src1|$src1, $src2}", []>, T8PD, + Requires<[Not64BitMode]>; def INVPCID64 : I<0x82, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), - "invpcid\t{$src2, $src1|$src1, $src2}", []>, OpSize, T8, + "invpcid\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; //===----------------------------------------------------------------------===// diff --git a/lib/Target/X86/X86InstrTSX.td b/lib/Target/X86/X86InstrTSX.td index 59a6f1e..4940efc 100644 --- a/lib/Target/X86/X86InstrTSX.td +++ b/lib/Target/X86/X86InstrTSX.td @@ -40,7 +40,8 @@ def XABORT : Ii8<0xc6, MRM_F8, (outs), (ins i8imm:$imm), // HLE prefixes +let isAsmParserOnly = 1 in { def XACQUIRE_PREFIX : I<0xF2, RawFrm, (outs), (ins), "xacquire", []>, Requires<[HasHLE]>; - def XRELEASE_PREFIX : I<0xF3, RawFrm, (outs), (ins), "xrelease", []>, Requires<[HasHLE]>; +} diff --git a/lib/Target/X86/X86InstrVMX.td b/lib/Target/X86/X86InstrVMX.td index 6d3548f..79afe9a 100644 --- a/lib/Target/X86/X86InstrVMX.td +++ b/lib/Target/X86/X86InstrVMX.td @@ -17,22 +17,22 @@ // 66 0F 38 80 def INVEPT32 : I<0x80, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), - "invept\t{$src2, $src1|$src1, $src2}", []>, OpSize, T8, - Requires<[In32BitMode]>; + "invept\t{$src2, $src1|$src1, $src2}", []>, T8PD, + Requires<[Not64BitMode]>; def INVEPT64 : I<0x80, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), - "invept\t{$src2, $src1|$src1, $src2}", []>, OpSize, T8, + "invept\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; // 66 0F 38 81 def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), - "invvpid\t{$src2, $src1|$src1, $src2}", []>, OpSize, T8, - Requires<[In32BitMode]>; + "invvpid\t{$src2, $src1|$src1, $src2}", []>, T8PD, + Requires<[Not64BitMode]>; def INVVPID64 : I<0x81, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), - "invvpid\t{$src2, $src1|$src1, $src2}", []>, OpSize, T8, + "invvpid\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; // 0F 01 C1 def VMCALL : I<0x01, MRM_C1, (outs), (ins), "vmcall", []>, TB; def VMCLEARm : I<0xC7, MRM6m, (outs), (ins i64mem:$vmcs), - "vmclear\t$vmcs", []>, OpSize, TB; + "vmclear\t$vmcs", []>, PD; // OF 01 D4 def VMFUNC : I<0x01, MRM_D4, (outs), (ins), "vmfunc", []>, TB; // 0F 01 C2 @@ -40,25 +40,25 @@ def VMLAUNCH : I<0x01, MRM_C2, (outs), (ins), "vmlaunch", []>, TB; // 0F 01 C3 def VMRESUME : I<0x01, MRM_C3, (outs), (ins), "vmresume", []>, TB; def VMPTRLDm : I<0xC7, MRM6m, (outs), (ins i64mem:$vmcs), - "vmptrld\t$vmcs", []>, TB; + "vmptrld\t$vmcs", []>, PS; def VMPTRSTm : I<0xC7, MRM7m, (outs i64mem:$vmcs), (ins), "vmptrst\t$vmcs", []>, TB; def VMREAD64rm : I<0x78, MRMDestMem, (outs i64mem:$dst), (ins GR64:$src), - "vmread{q}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In64BitMode]>; + "vmread{q}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[In64BitMode]>; def VMREAD64rr : I<0x78, MRMDestReg, (outs GR64:$dst), (ins GR64:$src), - "vmread{q}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In64BitMode]>; + "vmread{q}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[In64BitMode]>; def VMREAD32rm : I<0x78, MRMDestMem, (outs i32mem:$dst), (ins GR32:$src), - "vmread{l}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In32BitMode]>; + "vmread{l}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[Not64BitMode]>; def VMREAD32rr : I<0x78, MRMDestReg, (outs GR32:$dst), (ins GR32:$src), - "vmread{l}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In32BitMode]>; + "vmread{l}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[Not64BitMode]>; def VMWRITE64rm : I<0x79, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), - "vmwrite{q}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In64BitMode]>; + "vmwrite{q}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[In64BitMode]>; def VMWRITE64rr : I<0x79, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), - "vmwrite{q}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In64BitMode]>; + "vmwrite{q}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[In64BitMode]>; def VMWRITE32rm : I<0x79, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), - "vmwrite{l}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In32BitMode]>; + "vmwrite{l}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[Not64BitMode]>; def VMWRITE32rr : I<0x79, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), - "vmwrite{l}\t{$src, $dst|$dst, $src}", []>, TB, Requires<[In32BitMode]>; + "vmwrite{l}\t{$src, $dst|$dst, $src}", []>, PS, Requires<[Not64BitMode]>; // 0F 01 C4 def VMXOFF : I<0x01, MRM_C4, (outs), (ins), "vmxoff", []>, TB; def VMXON : I<0xC7, MRM6m, (outs), (ins i64mem:$vmxon), diff --git a/lib/Target/X86/X86InstrXOP.td b/lib/Target/X86/X86InstrXOP.td index 2b6ee5c..45e2ff0 100644 --- a/lib/Target/X86/X86InstrXOP.td +++ b/lib/Target/X86/X86InstrXOP.td @@ -14,10 +14,10 @@ multiclass xop2op<bits<8> opc, string OpcodeStr, Intrinsic Int, PatFrag memop> { def rr : IXOP<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int VR128:$src))]>, VEX; + [(set VR128:$dst, (Int VR128:$src))]>, XOP; def rm : IXOP<opc, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int (bitconvert (memop addr:$src))))]>, VEX; + [(set VR128:$dst, (Int (bitconvert (memop addr:$src))))]>, XOP; } defm VPHSUBWD : xop2op<0xE2, "vphsubwd", int_x86_xop_vphsubwd, memopv2i64>; @@ -41,10 +41,10 @@ multiclass xop2opsld<bits<8> opc, string OpcodeStr, Intrinsic Int, Operand memop, ComplexPattern mem_cpat> { def rr : IXOP<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int VR128:$src))]>, VEX; + [(set VR128:$dst, (Int VR128:$src))]>, XOP; def rm : IXOP<opc, MRMSrcMem, (outs VR128:$dst), (ins memop:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int (bitconvert mem_cpat:$src)))]>, VEX; + [(set VR128:$dst, (Int (bitconvert mem_cpat:$src)))]>, XOP; } defm VFRCZSS : xop2opsld<0x82, "vfrczss", int_x86_xop_vfrcz_ss, @@ -56,10 +56,10 @@ multiclass xop2op128<bits<8> opc, string OpcodeStr, Intrinsic Int, PatFrag memop> { def rr : IXOP<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int VR128:$src))]>, VEX; + [(set VR128:$dst, (Int VR128:$src))]>, XOP; def rm : IXOP<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR128:$dst, (Int (bitconvert (memop addr:$src))))]>, VEX; + [(set VR128:$dst, (Int (bitconvert (memop addr:$src))))]>, XOP; } defm VFRCZPS : xop2op128<0x80, "vfrczps", int_x86_xop_vfrcz_ps, memopv4f32>; @@ -69,10 +69,10 @@ multiclass xop2op256<bits<8> opc, string OpcodeStr, Intrinsic Int, PatFrag memop> { def rrY : IXOP<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR256:$dst, (Int VR256:$src))]>, VEX, VEX_L; + [(set VR256:$dst, (Int VR256:$src))]>, XOP, VEX_L; def rmY : IXOP<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), - [(set VR256:$dst, (Int (bitconvert (memop addr:$src))))]>, VEX, VEX_L; + [(set VR256:$dst, (Int (bitconvert (memop addr:$src))))]>, XOP, VEX_L; } defm VFRCZPS : xop2op256<0x80, "vfrczps", int_x86_xop_vfrcz_ps_256, memopv8f32>; @@ -82,19 +82,19 @@ multiclass xop3op<bits<8> opc, string OpcodeStr, Intrinsic Int> { def rr : IXOP<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set VR128:$dst, (Int VR128:$src1, VR128:$src2))]>, VEX_4VOp3; + [(set VR128:$dst, (Int VR128:$src1, VR128:$src2))]>, XOP_4VOp3; def rm : IXOP<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128:$dst, (Int VR128:$src1, (bitconvert (memopv2i64 addr:$src2))))]>, - VEX_4V, VEX_W; + XOP_4V, VEX_W; def mr : IXOP<opc, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1, VR128:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128:$dst, (Int (bitconvert (memopv2i64 addr:$src1)), VR128:$src2))]>, - VEX_4VOp3; + XOP_4VOp3; } defm VPSHLW : xop3op<0x95, "vpshlw", int_x86_xop_vpshlw>; @@ -114,12 +114,12 @@ multiclass xop3opimm<bits<8> opc, string OpcodeStr, Intrinsic Int> { def ri : IXOPi8<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set VR128:$dst, (Int VR128:$src1, imm:$src2))]>, VEX; + [(set VR128:$dst, (Int VR128:$src1, imm:$src2))]>, XOP; def mi : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1, i8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set VR128:$dst, - (Int (bitconvert (memopv2i64 addr:$src1)), imm:$src2))]>, VEX; + (Int (bitconvert (memopv2i64 addr:$src1)), imm:$src2))]>, XOP; } defm VPROTW : xop3opimm<0xC1, "vprotw", int_x86_xop_vprotwi>; @@ -134,14 +134,14 @@ multiclass xop4opm2<bits<8> opc, string OpcodeStr, Intrinsic Int> { !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, - (Int VR128:$src1, VR128:$src2, VR128:$src3))]>, VEX_4V, VEX_I8IMM; + (Int VR128:$src1, VR128:$src2, VR128:$src3))]>, XOP_4V, VEX_I8IMM; def rm : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2, VR128:$src3), !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, (Int VR128:$src1, (bitconvert (memopv2i64 addr:$src2)), - VR128:$src3))]>, VEX_4V, VEX_I8IMM; + VR128:$src3))]>, XOP_4V, VEX_I8IMM; } defm VPMADCSWD : xop4opm2<0xB6, "vpmadcswd", int_x86_xop_vpmadcswd>; @@ -164,14 +164,14 @@ multiclass xop4opimm<bits<8> opc, string OpcodeStr, Intrinsic Int> { !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, (Int VR128:$src1, VR128:$src2, imm:$src3))]>, - VEX_4V; + XOP_4V; def mi : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2, i8imm:$src3), !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, (Int VR128:$src1, (bitconvert (memopv2i64 addr:$src2)), - imm:$src3))]>, VEX_4V; + imm:$src3))]>, XOP_4V; } defm VPCOMB : xop4opimm<0xCC, "vpcomb", int_x86_xop_vpcomb>; @@ -190,7 +190,7 @@ multiclass xop4op<bits<8> opc, string OpcodeStr, Intrinsic Int> { !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, (Int VR128:$src1, VR128:$src2, VR128:$src3))]>, - VEX_4V, VEX_I8IMM; + XOP_4V, VEX_I8IMM; def rm : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i128mem:$src3), !strconcat(OpcodeStr, @@ -198,7 +198,7 @@ multiclass xop4op<bits<8> opc, string OpcodeStr, Intrinsic Int> { [(set VR128:$dst, (Int VR128:$src1, VR128:$src2, (bitconvert (memopv2i64 addr:$src3))))]>, - VEX_4V, VEX_I8IMM, VEX_W, MemOp4; + XOP_4V, VEX_I8IMM, VEX_W, MemOp4; def mr : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2, VR128:$src3), !strconcat(OpcodeStr, @@ -206,7 +206,7 @@ multiclass xop4op<bits<8> opc, string OpcodeStr, Intrinsic Int> { [(set VR128:$dst, (Int VR128:$src1, (bitconvert (memopv2i64 addr:$src2)), VR128:$src3))]>, - VEX_4V, VEX_I8IMM; + XOP_4V, VEX_I8IMM; } defm VPPERM : xop4op<0xA3, "vpperm", int_x86_xop_vpperm>; @@ -218,7 +218,7 @@ multiclass xop4op256<bits<8> opc, string OpcodeStr, Intrinsic Int> { !strconcat(OpcodeStr, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR256:$dst, (Int VR256:$src1, VR256:$src2, VR256:$src3))]>, - VEX_4V, VEX_I8IMM, VEX_L; + XOP_4V, VEX_I8IMM, VEX_L; def rmY : IXOPi8<opc, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, VR256:$src2, i256mem:$src3), !strconcat(OpcodeStr, @@ -226,7 +226,7 @@ multiclass xop4op256<bits<8> opc, string OpcodeStr, Intrinsic Int> { [(set VR256:$dst, (Int VR256:$src1, VR256:$src2, (bitconvert (memopv4i64 addr:$src3))))]>, - VEX_4V, VEX_I8IMM, VEX_W, MemOp4, VEX_L; + XOP_4V, VEX_I8IMM, VEX_W, MemOp4, VEX_L; def mrY : IXOPi8<opc, MRMSrcMem, (outs VR256:$dst), (ins VR256:$src1, f256mem:$src2, VR256:$src3), !strconcat(OpcodeStr, @@ -234,7 +234,7 @@ multiclass xop4op256<bits<8> opc, string OpcodeStr, Intrinsic Int> { [(set VR256:$dst, (Int VR256:$src1, (bitconvert (memopv4i64 addr:$src2)), VR256:$src3))]>, - VEX_4V, VEX_I8IMM, VEX_L; + XOP_4V, VEX_I8IMM, VEX_L; } defm VPCMOV : xop4op256<0xA2, "vpcmov", int_x86_xop_vpcmov_256>; diff --git a/lib/Target/X86/X86JITInfo.h b/lib/Target/X86/X86JITInfo.h index f916327..4d279de 100644 --- a/lib/Target/X86/X86JITInfo.h +++ b/lib/Target/X86/X86JITInfo.h @@ -35,41 +35,41 @@ namespace llvm { /// overwriting OLD with a branch to NEW. This is used for self-modifying /// code. /// - virtual void replaceMachineCodeForFunction(void *Old, void *New); + void replaceMachineCodeForFunction(void *Old, void *New) override; /// emitGlobalValueIndirectSym - Use the specified JITCodeEmitter object /// to emit an indirect symbol which contains the address of the specified /// ptr. - virtual void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr, - JITCodeEmitter &JCE); + void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr, + JITCodeEmitter &JCE) override; // getStubLayout - Returns the size and alignment of the largest call stub // on X86. - virtual StubLayout getStubLayout(); + StubLayout getStubLayout() override; /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a /// small native function that simply calls the function at the specified /// address. - virtual void *emitFunctionStub(const Function* F, void *Target, - JITCodeEmitter &JCE); + void *emitFunctionStub(const Function* F, void *Target, + JITCodeEmitter &JCE) override; /// getPICJumpTableEntry - Returns the value of the jumptable entry for the /// specific basic block. - virtual uintptr_t getPICJumpTableEntry(uintptr_t BB, uintptr_t JTBase); + uintptr_t getPICJumpTableEntry(uintptr_t BB, uintptr_t JTBase) override; /// getLazyResolverFunction - Expose the lazy resolver to the JIT. - virtual LazyResolverFn getLazyResolverFunction(JITCompilerFn); + LazyResolverFn getLazyResolverFunction(JITCompilerFn) override; /// relocate - Before the JIT can run a block of code that has been emitted, /// it must rewrite the code to contain the actual addresses of any /// referenced global symbols. - virtual void relocate(void *Function, MachineRelocation *MR, - unsigned NumRelocs, unsigned char* GOTBase); + void relocate(void *Function, MachineRelocation *MR, + unsigned NumRelocs, unsigned char* GOTBase) override; /// allocateThreadLocalMemory - Each target has its own way of /// handling thread local variables. This method returns a value only /// meaningful to the target. - virtual char* allocateThreadLocalMemory(size_t size); + char* allocateThreadLocalMemory(size_t size) override; /// setPICBase / getPICBase - Getter / setter of PICBase, used to compute /// PIC jumptable entry. diff --git a/lib/Target/X86/X86MCInstLower.cpp b/lib/Target/X86/X86MCInstLower.cpp index 6649c82..6d7f3cb 100644 --- a/lib/Target/X86/X86MCInstLower.cpp +++ b/lib/Target/X86/X86MCInstLower.cpp @@ -14,11 +14,14 @@ #include "X86AsmPrinter.h" #include "InstPrinter/X86ATTInstPrinter.h" -#include "X86COFFMachineModuleInfo.h" +#include "MCTargetDesc/X86BaseInfo.h" #include "llvm/ADT/SmallString.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/CodeGen/StackMaps.h" -#include "llvm/IR/Type.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/Mangler.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -26,8 +29,6 @@ #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Target/Mangler.h" using namespace llvm; namespace { @@ -70,42 +71,53 @@ MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const { /// operand to an MCSymbol. MCSymbol *X86MCInstLower:: GetSymbolFromOperand(const MachineOperand &MO) const { + const DataLayout *DL = TM.getDataLayout(); assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && "Isn't a symbol reference"); SmallString<128> Name; + StringRef Suffix; + + switch (MO.getTargetFlags()) { + case X86II::MO_DLLIMPORT: + // Handle dllimport linkage. + Name += "__imp_"; + break; + case X86II::MO_DARWIN_STUB: + Suffix = "$stub"; + break; + case X86II::MO_DARWIN_NONLAZY: + case X86II::MO_DARWIN_NONLAZY_PIC_BASE: + case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: + Suffix = "$non_lazy_ptr"; + break; + } + + if (!Suffix.empty()) + Name += DL->getPrivateGlobalPrefix(); + + unsigned PrefixLen = Name.size(); if (MO.isGlobal()) { const GlobalValue *GV = MO.getGlobal(); - bool isImplicitlyPrivate = false; - if (MO.getTargetFlags() == X86II::MO_DARWIN_STUB || - MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY || - MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY_PIC_BASE || - MO.getTargetFlags() == X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE) - isImplicitlyPrivate = true; - - getMang()->getNameWithPrefix(Name, GV, isImplicitlyPrivate); + AsmPrinter.getNameWithPrefix(Name, GV); } else if (MO.isSymbol()) { - Name += MAI.getGlobalPrefix(); - Name += MO.getSymbolName(); + getMang()->getNameWithPrefix(Name, MO.getSymbolName()); } else if (MO.isMBB()) { Name += MO.getMBB()->getSymbol()->getName(); } + unsigned OrigLen = Name.size() - PrefixLen; + + Name += Suffix; + MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name); + + StringRef OrigName = StringRef(Name).substr(PrefixLen, OrigLen); // If the target flags on the operand changes the name of the symbol, do that // before we return the symbol. switch (MO.getTargetFlags()) { default: break; - case X86II::MO_DLLIMPORT: { - // Handle dllimport linkage. - const char *Prefix = "__imp_"; - Name.insert(Name.begin(), Prefix, Prefix+strlen(Prefix)); - break; - } case X86II::MO_DARWIN_NONLAZY: case X86II::MO_DARWIN_NONLAZY_PIC_BASE: { - Name += "$non_lazy_ptr"; - MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name.str()); - MachineModuleInfoImpl::StubValueTy &StubSym = getMachOMMI().getGVStubEntry(Sym); if (StubSym.getPointer() == 0) { @@ -115,11 +127,9 @@ GetSymbolFromOperand(const MachineOperand &MO) const { StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), !MO.getGlobal()->hasInternalLinkage()); } - return Sym; + break; } case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: { - Name += "$non_lazy_ptr"; - MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name.str()); MachineModuleInfoImpl::StubValueTy &StubSym = getMachOMMI().getHiddenGVStubEntry(Sym); if (StubSym.getPointer() == 0) { @@ -129,11 +139,9 @@ GetSymbolFromOperand(const MachineOperand &MO) const { StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), !MO.getGlobal()->hasInternalLinkage()); } - return Sym; + break; } case X86II::MO_DARWIN_STUB: { - Name += "$stub"; - MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name.str()); MachineModuleInfoImpl::StubValueTy &StubSym = getMachOMMI().getFnStubEntry(Sym); if (StubSym.getPointer()) @@ -145,16 +153,15 @@ GetSymbolFromOperand(const MachineOperand &MO) const { StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), !MO.getGlobal()->hasInternalLinkage()); } else { - Name.erase(Name.end()-5, Name.end()); StubSym = MachineModuleInfoImpl:: - StubValueTy(Ctx.GetOrCreateSymbol(Name.str()), false); + StubValueTy(Ctx.GetOrCreateSymbol(OrigName), false); } - return Sym; + break; } } - return Ctx.GetOrCreateSymbol(Name.str()); + return Sym; } MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, @@ -227,13 +234,6 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, } -/// LowerUnaryToTwoAddr - R = setb -> R = sbb R, R -static void LowerUnaryToTwoAddr(MCInst &OutMI, unsigned NewOpc) { - OutMI.setOpcode(NewOpc); - OutMI.addOperand(OutMI.getOperand(0)); - OutMI.addOperand(OutMI.getOperand(0)); -} - /// \brief Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with /// a short fixed-register form. static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) { @@ -297,12 +297,12 @@ static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst, unsigned RegOp = IsStore ? 0 : 5; unsigned AddrOp = AddrBase + 3; assert(Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && - Inst.getOperand(AddrBase + 0).isReg() && // base - Inst.getOperand(AddrBase + 1).isImm() && // scale - Inst.getOperand(AddrBase + 2).isReg() && // index register - (Inst.getOperand(AddrOp).isExpr() || // address - Inst.getOperand(AddrOp).isImm())&& - Inst.getOperand(AddrBase + 4).isReg() && // segment + Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && + Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && + Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && + Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && + (Inst.getOperand(AddrOp).isExpr() || + Inst.getOperand(AddrOp).isImm()) && "Unexpected instruction!"); // Check whether the destination register can be fixed. @@ -322,17 +322,23 @@ static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst, } if (Absolute && - (Inst.getOperand(AddrBase + 0).getReg() != 0 || - Inst.getOperand(AddrBase + 2).getReg() != 0 || - Inst.getOperand(AddrBase + 4).getReg() != 0 || - Inst.getOperand(AddrBase + 1).getImm() != 1)) + (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 || + Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 || + Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0)) return; // If so, rewrite the instruction. MCOperand Saved = Inst.getOperand(AddrOp); + MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg); Inst = MCInst(); Inst.setOpcode(Opcode); Inst.addOperand(Saved); + Inst.addOperand(Seg); +} + +static unsigned getRetOpcode(const X86Subtarget &Subtarget) +{ + return Subtarget.is64Bit() ? X86::RETQ : X86::RETL; } void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { @@ -390,7 +396,6 @@ ReSimplify: assert(OutMI.getOperand(1+X86::AddrSegmentReg).getReg() == 0 && "LEA has segment specified!"); break; - case X86::MOV32r0: LowerUnaryToTwoAddr(OutMI, X86::XOR32rr); break; case X86::MOV32ri64: OutMI.setOpcode(X86::MOV32ri); @@ -464,7 +469,7 @@ ReSimplify: case X86::EH_RETURN: case X86::EH_RETURN64: { OutMI = MCInst(); - OutMI.setOpcode(X86::RET); + OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); break; } @@ -598,7 +603,8 @@ ReSimplify: static void LowerTlsAddr(MCStreamer &OutStreamer, X86MCInstLower &MCInstLowering, - const MachineInstr &MI) { + const MachineInstr &MI, + const MCSubtargetInfo& STI) { bool is64Bits = MI.getOpcode() == X86::TLS_addr64 || MI.getOpcode() == X86::TLS_base_addr64; @@ -608,7 +614,7 @@ static void LowerTlsAddr(MCStreamer &OutStreamer, MCContext &context = OutStreamer.getContext(); if (needsPadding) - OutStreamer.EmitInstruction(MCInstBuilder(X86::DATA16_PREFIX)); + OutStreamer.EmitInstruction(MCInstBuilder(X86::DATA16_PREFIX), STI); MCSymbolRefExpr::VariantKind SRVK; switch (MI.getOpcode()) { @@ -655,12 +661,12 @@ static void LowerTlsAddr(MCStreamer &OutStreamer, LEA.addOperand(MCOperand::CreateExpr(symRef)); // disp LEA.addOperand(MCOperand::CreateReg(0)); // seg } - OutStreamer.EmitInstruction(LEA); + OutStreamer.EmitInstruction(LEA, STI); if (needsPadding) { - OutStreamer.EmitInstruction(MCInstBuilder(X86::DATA16_PREFIX)); - OutStreamer.EmitInstruction(MCInstBuilder(X86::DATA16_PREFIX)); - OutStreamer.EmitInstruction(MCInstBuilder(X86::REX64_PREFIX)); + OutStreamer.EmitInstruction(MCInstBuilder(X86::DATA16_PREFIX), STI); + OutStreamer.EmitInstruction(MCInstBuilder(X86::DATA16_PREFIX), STI); + OutStreamer.EmitInstruction(MCInstBuilder(X86::REX64_PREFIX), STI); } StringRef name = is64Bits ? "__tls_get_addr" : "___tls_get_addr"; @@ -672,114 +678,79 @@ static void LowerTlsAddr(MCStreamer &OutStreamer, OutStreamer.EmitInstruction(MCInstBuilder(is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32) - .addExpr(tlsRef)); -} - -static std::pair<StackMaps::Location, MachineInstr::const_mop_iterator> -parseMemoryOperand(StackMaps::Location::LocationType LocTy, unsigned Size, - MachineInstr::const_mop_iterator MOI, - MachineInstr::const_mop_iterator MOE) { - - typedef StackMaps::Location Location; - - assert(std::distance(MOI, MOE) >= 5 && "Too few operands to encode mem op."); - - const MachineOperand &Base = *MOI; - const MachineOperand &Scale = *(++MOI); - const MachineOperand &Index = *(++MOI); - const MachineOperand &Disp = *(++MOI); - const MachineOperand &ZeroReg = *(++MOI); - - // Sanity check for supported operand format. - assert(Base.isReg() && - Scale.isImm() && Scale.getImm() == 1 && - Index.isReg() && Index.getReg() == 0 && - Disp.isImm() && ZeroReg.isReg() && (ZeroReg.getReg() == 0) && - "Unsupported x86 memory operand sequence."); - (void)Scale; - (void)Index; - (void)ZeroReg; - - return std::make_pair( - Location(LocTy, Size, Base.getReg(), Disp.getImm()), ++MOI); + .addExpr(tlsRef), STI); } -std::pair<StackMaps::Location, MachineInstr::const_mop_iterator> -X86AsmPrinter::stackmapOperandParser(MachineInstr::const_mop_iterator MOI, - MachineInstr::const_mop_iterator MOE, - const TargetMachine &TM) { - - typedef StackMaps::Location Location; - - const MachineOperand &MOP = *MOI; - assert(!MOP.isRegMask() && (!MOP.isReg() || !MOP.isImplicit()) && - "Register mask and implicit operands should not be processed."); - - if (MOP.isImm()) { - // Verify anyregcc - // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ... - - switch (MOP.getImm()) { - default: llvm_unreachable("Unrecognized operand type."); - case StackMaps::DirectMemRefOp: { - unsigned Size = TM.getDataLayout()->getPointerSizeInBits(); - assert((Size % 8) == 0 && "Need pointer size in bytes."); - Size /= 8; - return parseMemoryOperand(StackMaps::Location::Direct, Size, - llvm::next(MOI), MOE); +/// \brief Emit the optimal amount of multi-byte nops on X86. +static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSubtargetInfo &STI) { + // This works only for 64bit. For 32bit we have to do additional checking if + // the CPU supports multi-byte nops. + assert(Is64Bit && "EmitNops only supports X86-64"); + while (NumBytes) { + unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg; + Opc = IndexReg = Displacement = SegmentReg = 0; + BaseReg = X86::RAX; ScaleVal = 1; + switch (NumBytes) { + case 0: llvm_unreachable("Zero nops?"); break; + case 1: NumBytes -= 1; Opc = X86::NOOP; break; + case 2: NumBytes -= 2; Opc = X86::XCHG16ar; break; + case 3: NumBytes -= 3; Opc = X86::NOOPL; break; + case 4: NumBytes -= 4; Opc = X86::NOOPL; Displacement = 8; break; + case 5: NumBytes -= 5; Opc = X86::NOOPL; Displacement = 8; + IndexReg = X86::RAX; break; + case 6: NumBytes -= 6; Opc = X86::NOOPW; Displacement = 8; + IndexReg = X86::RAX; break; + case 7: NumBytes -= 7; Opc = X86::NOOPL; Displacement = 512; break; + case 8: NumBytes -= 8; Opc = X86::NOOPL; Displacement = 512; + IndexReg = X86::RAX; break; + case 9: NumBytes -= 9; Opc = X86::NOOPW; Displacement = 512; + IndexReg = X86::RAX; break; + default: NumBytes -= 10; Opc = X86::NOOPW; Displacement = 512; + IndexReg = X86::RAX; SegmentReg = X86::CS; break; } - case StackMaps::IndirectMemRefOp: { - ++MOI; - int64_t Size = MOI->getImm(); - assert(Size > 0 && "Need a valid size for indirect memory locations."); - return parseMemoryOperand(StackMaps::Location::Indirect, Size, - llvm::next(MOI), MOE); - } - case StackMaps::ConstantOp: { - ++MOI; - assert(MOI->isImm() && "Expected constant operand."); - int64_t Imm = MOI->getImm(); - return std::make_pair( - Location(Location::Constant, sizeof(int64_t), 0, Imm), ++MOI); - } - } - } - // Otherwise this is a reg operand. The physical register number will - // ultimately be encoded as a DWARF regno. The stack map also records the size - // of a spill slot that can hold the register content. (The runtime can - // track the actual size of the data type if it needs to.) - assert(MOP.isReg() && "Expected register operand here."); - assert(TargetRegisterInfo::isPhysicalRegister(MOP.getReg()) && - "Virtreg operands should have been rewritten before now."); - const TargetRegisterClass *RC = - TM.getRegisterInfo()->getMinimalPhysRegClass(MOP.getReg()); - assert(!MOP.getSubReg() && "Physical subreg still around."); - return std::make_pair( - Location(Location::Register, RC->getSize(), MOP.getReg(), 0), ++MOI); + unsigned NumPrefixes = std::min(NumBytes, 5U); + NumBytes -= NumPrefixes; + for (unsigned i = 0; i != NumPrefixes; ++i) + OS.EmitBytes("\x66"); + + switch (Opc) { + default: llvm_unreachable("Unexpected opcode"); break; + case X86::NOOP: + OS.EmitInstruction(MCInstBuilder(Opc), STI); + break; + case X86::XCHG16ar: + OS.EmitInstruction(MCInstBuilder(Opc).addReg(X86::AX), STI); + break; + case X86::NOOPL: + case X86::NOOPW: + OS.EmitInstruction(MCInstBuilder(Opc).addReg(BaseReg).addImm(ScaleVal) + .addReg(IndexReg) + .addImm(Displacement) + .addReg(SegmentReg), STI); + break; + } + } // while (NumBytes) } // Lower a stackmap of the form: // <id>, <shadowBytes>, ... -static void LowerSTACKMAP(MCStreamer &OutStreamer, - StackMaps &SM, - const MachineInstr &MI) -{ - unsigned NumNOPBytes = MI.getOperand(1).getImm(); +static void LowerSTACKMAP(MCStreamer &OS, StackMaps &SM, + const MachineInstr &MI, bool Is64Bit, const MCSubtargetInfo& STI) { + unsigned NumBytes = MI.getOperand(1).getImm(); SM.recordStackMap(MI); // Emit padding. // FIXME: These nops ensure that the stackmap's shadow is covered by // instructions from the same basic block, but the nops should not be // necessary if instructions from the same block follow the stackmap. - for (unsigned i = 0; i < NumNOPBytes; ++i) - OutStreamer.EmitInstruction(MCInstBuilder(X86::NOOP)); + EmitNops(OS, NumBytes, Is64Bit, STI); } // Lower a patchpoint of the form: // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ... -static void LowerPATCHPOINT(MCStreamer &OutStreamer, - StackMaps &SM, - const MachineInstr &MI) { +static void LowerPATCHPOINT(MCStreamer &OS, StackMaps &SM, + const MachineInstr &MI, bool Is64Bit, const MCSubtargetInfo& STI) { + assert(Is64Bit && "Patchpoint currently only supports X86-64"); SM.recordPatchPoint(MI); PatchPointOpers opers(&MI); @@ -789,22 +760,21 @@ static void LowerPATCHPOINT(MCStreamer &OutStreamer, if (CallTarget) { // Emit MOV to materialize the target address and the CALL to target. // This is encoded with 12-13 bytes, depending on which register is used. - // We conservatively assume that it is 12 bytes and emit in worst case one - // extra NOP byte. - EncodedBytes = 12; - OutStreamer.EmitInstruction(MCInstBuilder(X86::MOV64ri) - .addReg(MI.getOperand(ScratchIdx).getReg()) - .addImm(CallTarget)); - OutStreamer.EmitInstruction(MCInstBuilder(X86::CALL64r) - .addReg(MI.getOperand(ScratchIdx).getReg())); + unsigned ScratchReg = MI.getOperand(ScratchIdx).getReg(); + if (X86II::isX86_64ExtendedReg(ScratchReg)) + EncodedBytes = 13; + else + EncodedBytes = 12; + OS.EmitInstruction(MCInstBuilder(X86::MOV64ri).addReg(ScratchReg) + .addImm(CallTarget), STI); + OS.EmitInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg), STI); } // Emit padding. unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm(); assert(NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call."); - for (unsigned i = EncodedBytes; i < NumBytes; ++i) - OutStreamer.EmitInstruction(MCInstBuilder(X86::NOOP)); + EmitNops(OS, NumBytes - EncodedBytes, Is64Bit, STI); } void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { @@ -815,8 +785,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { // Emit nothing here but a comment if we can. case X86::Int_MemBarrier: - if (OutStreamer.hasRawTextSupport()) - OutStreamer.EmitRawText(StringRef("\t#MEMBARRIER")); + OutStreamer.emitRawComment("MEMBARRIER"); return; @@ -839,7 +808,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case X86::TLS_addr64: case X86::TLS_base_addr32: case X86::TLS_base_addr64: - return LowerTlsAddr(OutStreamer, MCInstLowering, *MI); + return LowerTlsAddr(OutStreamer, MCInstLowering, *MI, getSubtargetInfo()); case X86::MOVPC32r: { // This is a pseudo op for a two instruction sequence with a label, which @@ -852,14 +821,14 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { MCSymbol *PICBase = MF->getPICBaseSymbol(); // FIXME: We would like an efficient form for this, so we don't have to do a // lot of extra uniquing. - OutStreamer.EmitInstruction(MCInstBuilder(X86::CALLpcrel32) + EmitToStreamer(OutStreamer, MCInstBuilder(X86::CALLpcrel32) .addExpr(MCSymbolRefExpr::Create(PICBase, OutContext))); // Emit the label. OutStreamer.EmitLabel(PICBase); // popl $reg - OutStreamer.EmitInstruction(MCInstBuilder(X86::POP32r) + EmitToStreamer(OutStreamer, MCInstBuilder(X86::POP32r) .addReg(MI->getOperand(0).getReg())); return; } @@ -890,7 +859,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { DotExpr = MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(OpSym,OutContext), DotExpr, OutContext); - OutStreamer.EmitInstruction(MCInstBuilder(X86::ADD32ri) + EmitToStreamer(OutStreamer, MCInstBuilder(X86::ADD32ri) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(DotExpr)); @@ -898,19 +867,19 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { } case TargetOpcode::STACKMAP: - return LowerSTACKMAP(OutStreamer, SM, *MI); + return LowerSTACKMAP(OutStreamer, SM, *MI, Subtarget->is64Bit(), getSubtargetInfo()); case TargetOpcode::PATCHPOINT: - return LowerPATCHPOINT(OutStreamer, SM, *MI); + return LowerPATCHPOINT(OutStreamer, SM, *MI, Subtarget->is64Bit(), getSubtargetInfo()); case X86::MORESTACK_RET: - OutStreamer.EmitInstruction(MCInstBuilder(X86::RET)); + EmitToStreamer(OutStreamer, MCInstBuilder(getRetOpcode(*Subtarget))); return; case X86::MORESTACK_RET_RESTORE_R10: // Return, then restore R10. - OutStreamer.EmitInstruction(MCInstBuilder(X86::RET)); - OutStreamer.EmitInstruction(MCInstBuilder(X86::MOV64rr) + EmitToStreamer(OutStreamer, MCInstBuilder(getRetOpcode(*Subtarget))); + EmitToStreamer(OutStreamer, MCInstBuilder(X86::MOV64rr) .addReg(X86::R10) .addReg(X86::RAX)); return; @@ -918,5 +887,5 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { MCInst TmpInst; MCInstLowering.Lower(MI, TmpInst); - OutStreamer.EmitInstruction(TmpInst); + EmitToStreamer(OutStreamer, TmpInst); } diff --git a/lib/Target/X86/X86PadShortFunction.cpp b/lib/Target/X86/X86PadShortFunction.cpp index 83e75ea..746d0d6 100644 --- a/lib/Target/X86/X86PadShortFunction.cpp +++ b/lib/Target/X86/X86PadShortFunction.cpp @@ -51,9 +51,9 @@ namespace { PadShortFunc() : MachineFunctionPass(ID) , Threshold(4), TM(0), TII(0) {} - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; - virtual const char *getPassName() const { + const char *getPassName() const override { return "X86 Atom pad short functions"; } diff --git a/lib/Target/X86/X86RegisterInfo.cpp b/lib/Target/X86/X86RegisterInfo.cpp index dbda556..85aa9b5 100644 --- a/lib/Target/X86/X86RegisterInfo.cpp +++ b/lib/Target/X86/X86RegisterInfo.cpp @@ -14,7 +14,6 @@ //===----------------------------------------------------------------------===// #include "X86RegisterInfo.h" -#include "X86.h" #include "X86InstrBuilder.h" #include "X86MachineFunctionInfo.h" #include "X86Subtarget.h" @@ -27,7 +26,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/CodeGen/MachineValueType.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/Type.h" @@ -234,19 +233,25 @@ X86RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, const uint16_t * X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { + bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX(); + bool HasAVX512 = TM.getSubtarget<X86Subtarget>().hasAVX512(); + + assert(MF && "MachineFunction required"); switch (MF->getFunction()->getCallingConv()) { case CallingConv::GHC: case CallingConv::HiPE: return CSR_NoRegs_SaveList; - - case CallingConv::WebKit_JS: - return CSR_64_SaveList; case CallingConv::AnyReg: - return CSR_MostRegs_64_SaveList; - + if (HasAVX) + return CSR_64_AllRegs_AVX_SaveList; + return CSR_64_AllRegs_SaveList; + case CallingConv::PreserveMost: + return CSR_64_RT_MostRegs_SaveList; + case CallingConv::PreserveAll: + if (HasAVX) + return CSR_64_RT_AllRegs_AVX_SaveList; + return CSR_64_RT_AllRegs_SaveList; case CallingConv::Intel_OCL_BI: { - bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX(); - bool HasAVX512 = TM.getSubtarget<X86Subtarget>().hasAVX512(); if (HasAVX512 && IsWin64) return CSR_Win64_Intel_OCL_BI_AVX512_SaveList; if (HasAVX512 && Is64Bit) @@ -259,12 +264,10 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { return CSR_64_Intel_OCL_BI_SaveList; break; } - case CallingConv::Cold: if (Is64Bit) - return CSR_MostRegs_64_SaveList; + return CSR_64_MostRegs_SaveList; break; - default: break; } @@ -287,29 +290,49 @@ X86RegisterInfo::getCallPreservedMask(CallingConv::ID CC) const { bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX(); bool HasAVX512 = TM.getSubtarget<X86Subtarget>().hasAVX512(); - if (CC == CallingConv::Intel_OCL_BI) { - if (IsWin64 && HasAVX512) + switch (CC) { + case CallingConv::GHC: + case CallingConv::HiPE: + return CSR_NoRegs_RegMask; + case CallingConv::AnyReg: + if (HasAVX) + return CSR_64_AllRegs_AVX_RegMask; + return CSR_64_AllRegs_RegMask; + case CallingConv::PreserveMost: + return CSR_64_RT_MostRegs_RegMask; + case CallingConv::PreserveAll: + if (HasAVX) + return CSR_64_RT_AllRegs_AVX_RegMask; + return CSR_64_RT_AllRegs_RegMask; + case CallingConv::Intel_OCL_BI: { + if (HasAVX512 && IsWin64) return CSR_Win64_Intel_OCL_BI_AVX512_RegMask; - if (Is64Bit && HasAVX512) + if (HasAVX512 && Is64Bit) return CSR_64_Intel_OCL_BI_AVX512_RegMask; - if (IsWin64 && HasAVX) + if (HasAVX && IsWin64) return CSR_Win64_Intel_OCL_BI_AVX_RegMask; - if (Is64Bit && HasAVX) + if (HasAVX && Is64Bit) return CSR_64_Intel_OCL_BI_AVX_RegMask; if (!HasAVX && !IsWin64 && Is64Bit) return CSR_64_Intel_OCL_BI_RegMask; + break; } - if (CC == CallingConv::GHC || CC == CallingConv::HiPE) - return CSR_NoRegs_RegMask; - if (CC == CallingConv::WebKit_JS || CC == CallingConv::AnyReg) - return CSR_MostRegs_64_RegMask; - if (!Is64Bit) - return CSR_32_RegMask; - if (CC == CallingConv::Cold) - return CSR_MostRegs_64_RegMask; - if (IsWin64) - return CSR_Win64_RegMask; - return CSR_64_RegMask; + case CallingConv::Cold: + if (Is64Bit) + return CSR_64_MostRegs_RegMask; + break; + default: + break; + } + + // Unlike getCalleeSavedRegs(), we don't have MMI so we can't check + // callsEHReturn(). + if (Is64Bit) { + if (IsWin64) + return CSR_Win64_RegMask; + return CSR_64_RegMask; + } + return CSR_32_RegMask; } const uint32_t* @@ -403,18 +426,15 @@ bool X86RegisterInfo::hasBasePointer(const MachineFunction &MF) const { if (!EnableBasePointer) return false; - // When we need stack realignment and there are dynamic allocas, we can't - // reference off of the stack pointer, so we reserve a base pointer. - // - // This is also true if the function contain MS-style inline assembly. We - // do this because if any stack changes occur in the inline assembly, e.g., - // "pusha", then any C local variable or C argument references in the - // inline assembly will be wrong because the SP is not properly tracked. - if ((needsStackRealignment(MF) && MFI->hasVarSizedObjects()) || - MF.hasMSInlineAsm()) - return true; - - return false; + // When we need stack realignment, we can't address the stack from the frame + // pointer. When we have dynamic allocas or stack-adjusting inline asm, we + // can't address variables from the stack pointer. MS inline asm can + // reference locals while also adjusting the stack pointer. When we can't + // use both the SP and the FP, we need a separate base pointer register. + bool CantUseFP = needsStackRealignment(MF); + bool CantUseSP = + MFI->hasVarSizedObjects() || MFI->hasInlineAsmWithSPAdjust(); + return CantUseFP && CantUseSP; } bool X86RegisterInfo::canRealignStack(const MachineFunction &MF) const { @@ -499,6 +519,15 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, } else FIOffset = TFI->getFrameIndexOffset(MF, FrameIndex); + // The frame index format for stackmaps and patchpoints is different from the + // X86 format. It only has a FI and an offset. + if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) { + assert(BasePtr == FramePtr && "Expected the FP as base register"); + int64_t Offset = MI.getOperand(FIOperandNum + 1).getImm() + FIOffset; + MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset); + return; + } + if (MI.getOperand(FIOperandNum+3).isImm()) { // Offset is a 32-bit integer. int Imm = (int)(MI.getOperand(FIOperandNum + 3).getImm()); diff --git a/lib/Target/X86/X86RegisterInfo.h b/lib/Target/X86/X86RegisterInfo.h index 22251b2..6a71113 100644 --- a/lib/Target/X86/X86RegisterInfo.h +++ b/lib/Target/X86/X86RegisterInfo.h @@ -24,7 +24,7 @@ namespace llvm { class TargetInstrInfo; class X86TargetMachine; -class X86RegisterInfo : public X86GenRegisterInfo { +class X86RegisterInfo final : public X86GenRegisterInfo { public: X86TargetMachine &TM; @@ -62,66 +62,70 @@ public: /// getCompactUnwindRegNum - This function maps the register to the number for /// compact unwind encoding. Return -1 if the register isn't valid. - int getCompactUnwindRegNum(unsigned RegNum, bool isEH) const; + int getCompactUnwindRegNum(unsigned RegNum, bool isEH) const override; /// Code Generation virtual methods... /// - virtual bool trackLivenessAfterRegAlloc(const MachineFunction &MF) const; + bool trackLivenessAfterRegAlloc(const MachineFunction &MF) const override; /// getMatchingSuperRegClass - Return a subclass of the specified register /// class A so that each register in it has a sub-register of the /// specified sub-register index which is in the specified register class B. - virtual const TargetRegisterClass * + const TargetRegisterClass * getMatchingSuperRegClass(const TargetRegisterClass *A, - const TargetRegisterClass *B, unsigned Idx) const; + const TargetRegisterClass *B, + unsigned Idx) const override; - virtual const TargetRegisterClass * - getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const; + const TargetRegisterClass * + getSubClassWithSubReg(const TargetRegisterClass *RC, + unsigned Idx) const override; const TargetRegisterClass* - getLargestLegalSuperClass(const TargetRegisterClass *RC) const; + getLargestLegalSuperClass(const TargetRegisterClass *RC) const override; /// getPointerRegClass - Returns a TargetRegisterClass used for pointer /// values. const TargetRegisterClass * - getPointerRegClass(const MachineFunction &MF, unsigned Kind = 0) const; + getPointerRegClass(const MachineFunction &MF, + unsigned Kind = 0) const override; /// getCrossCopyRegClass - Returns a legal register class to copy a register /// in the specified class to or from. Returns NULL if it is possible to copy /// between a two registers of the specified class. const TargetRegisterClass * - getCrossCopyRegClass(const TargetRegisterClass *RC) const; + getCrossCopyRegClass(const TargetRegisterClass *RC) const override; unsigned getRegPressureLimit(const TargetRegisterClass *RC, - MachineFunction &MF) const; + MachineFunction &MF) const override; /// getCalleeSavedRegs - Return a null-terminated list of all of the /// callee-save registers on this target. - const uint16_t *getCalleeSavedRegs(const MachineFunction* MF = 0) const; - const uint32_t *getCallPreservedMask(CallingConv::ID) const; + const uint16_t * + getCalleeSavedRegs(const MachineFunction* MF) const override; + const uint32_t *getCallPreservedMask(CallingConv::ID) const override; const uint32_t *getNoPreservedMask() const; /// getReservedRegs - Returns a bitset indexed by physical register number /// indicating if a register is a special register that has particular uses and /// should be considered unavailable at all times, e.g. SP, RA. This is used by /// register scavenger to determine what registers are free. - BitVector getReservedRegs(const MachineFunction &MF) const; + BitVector getReservedRegs(const MachineFunction &MF) const override; bool hasBasePointer(const MachineFunction &MF) const; bool canRealignStack(const MachineFunction &MF) const; - bool needsStackRealignment(const MachineFunction &MF) const; + bool needsStackRealignment(const MachineFunction &MF) const override; bool hasReservedSpillSlot(const MachineFunction &MF, unsigned Reg, - int &FrameIdx) const; + int &FrameIdx) const override; void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, - RegScavenger *RS = NULL) const; + RegScavenger *RS = NULL) const override; // Debug information queries. - unsigned getFrameRegister(const MachineFunction &MF) const; + unsigned getFrameRegister(const MachineFunction &MF) const override; unsigned getStackRegister() const { return StackPtr; } unsigned getBaseRegister() const { return BasePtr; } // FIXME: Move to FrameInfok diff --git a/lib/Target/X86/X86RegisterInfo.td b/lib/Target/X86/X86RegisterInfo.td index b802728..33c402b 100644 --- a/lib/Target/X86/X86RegisterInfo.td +++ b/lib/Target/X86/X86RegisterInfo.td @@ -463,9 +463,13 @@ def VR128X : RegisterClass<"X86", [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], def VR256X : RegisterClass<"X86", [v32i8, v16i16, v8i32, v4i64, v8f32, v4f64], 256, (sequence "YMM%u", 0, 31)>; -def VK8 : RegisterClass<"X86", [v8i1], 8, (sequence "K%u", 0, 7)>; -def VK16 : RegisterClass<"X86", [v16i1], 16, (add VK8)>; - -def VK8WM : RegisterClass<"X86", [v8i1], 8, (sub VK8, K0)>; +// The size of the all masked registers is 16 bit because we have only one +// KMOVW istruction that can store this register in memory, and it writes 2 bytes +def VK1 : RegisterClass<"X86", [i1], 16, (sequence "K%u", 0, 7)>; +def VK8 : RegisterClass<"X86", [v8i1], 16, (add VK1)> {let Size = 16;} +def VK16 : RegisterClass<"X86", [v16i1], 16, (add VK8)> {let Size = 16;} + +def VK1WM : RegisterClass<"X86", [i1], 16, (sub VK1, K0)> {let Size = 16;} +def VK8WM : RegisterClass<"X86", [v8i1], 16, (sub VK8, K0)> {let Size = 16;} def VK16WM : RegisterClass<"X86", [v16i1], 16, (add VK8WM)>; diff --git a/lib/Target/X86/X86SchedHaswell.td b/lib/Target/X86/X86SchedHaswell.td index 9748261..f5b51ee 100644 --- a/lib/Target/X86/X86SchedHaswell.td +++ b/lib/Target/X86/X86SchedHaswell.td @@ -29,7 +29,7 @@ let SchedModel = HaswellModel in { // Haswell can issue micro-ops to 8 different ports in one cycle. -// Ports 0, 1, 5, 6 and 7 handle all computation. +// Ports 0, 1, 5, and 6 handle all computation. // Port 4 gets the data half of stores. Store data can be available later than // the store address, but since we don't model the latency of stores, we can // ignore that. @@ -48,8 +48,9 @@ def HWPort7 : ProcResource<1>; def HWPort23 : ProcResGroup<[HWPort2, HWPort3]>; def HWPort237 : ProcResGroup<[HWPort2, HWPort3, HWPort7]>; def HWPort05 : ProcResGroup<[HWPort0, HWPort5]>; -def HWPort056 : ProcResGroup<[HWPort0, HWPort5, HWPort6]>; +def HWPort06 : ProcResGroup<[HWPort0, HWPort6]>; def HWPort15 : ProcResGroup<[HWPort1, HWPort5]>; +def HWPort16 : ProcResGroup<[HWPort1, HWPort6]>; def HWPort015 : ProcResGroup<[HWPort0, HWPort1, HWPort5]>; def HWPort0156: ProcResGroup<[HWPort0, HWPort1, HWPort5, HWPort6]>; @@ -88,6 +89,8 @@ multiclass HWWriteResPair<X86FoldableSchedWrite SchedRW, // need an extra port 2/3 cycle to recompute the address. def : WriteRes<WriteRMW, [HWPort4]>; +// Store_addr on 237. +// Store_data on 4. def : WriteRes<WriteStore, [HWPort237, HWPort4]>; def : WriteRes<WriteLoad, [HWPort23]> { let Latency = 4; } def : WriteRes<WriteMove, [HWPort0156]>; @@ -96,8 +99,8 @@ def : WriteRes<WriteZero, []>; defm : HWWriteResPair<WriteALU, HWPort0156, 1>; defm : HWWriteResPair<WriteIMul, HWPort1, 3>; def : WriteRes<WriteIMulH, []> { let Latency = 3; } -defm : HWWriteResPair<WriteShift, HWPort056, 1>; -defm : HWWriteResPair<WriteJump, HWPort5, 1>; +defm : HWWriteResPair<WriteShift, HWPort06, 1>; +defm : HWWriteResPair<WriteJump, HWPort06, 1>; // This is for simple LEAs with one or two input operands. // The complex ones can only execute on port 1, and they require two cycles on @@ -123,14 +126,136 @@ defm : HWWriteResPair<WriteFSqrt, HWPort0, 15>; defm : HWWriteResPair<WriteCvtF2I, HWPort1, 3>; defm : HWWriteResPair<WriteCvtI2F, HWPort1, 4>; defm : HWWriteResPair<WriteCvtF2F, HWPort1, 3>; +defm : HWWriteResPair<WriteFShuffle, HWPort5, 1>; +defm : HWWriteResPair<WriteFBlend, HWPort015, 1>; +defm : HWWriteResPair<WriteFShuffle256, HWPort5, 3>; + +def : WriteRes<WriteFVarBlend, [HWPort5]> { + let Latency = 2; + let ResourceCycles = [2]; +} +def : WriteRes<WriteFVarBlendLd, [HWPort5, HWPort23]> { + let Latency = 6; + let ResourceCycles = [2, 1]; +} // Vector integer operations. -defm : HWWriteResPair<WriteVecShift, HWPort05, 1>; +defm : HWWriteResPair<WriteVecShift, HWPort0, 1>; defm : HWWriteResPair<WriteVecLogic, HWPort015, 1>; defm : HWWriteResPair<WriteVecALU, HWPort15, 1>; defm : HWWriteResPair<WriteVecIMul, HWPort0, 5>; -defm : HWWriteResPair<WriteShuffle, HWPort15, 1>; +defm : HWWriteResPair<WriteShuffle, HWPort5, 1>; +defm : HWWriteResPair<WriteBlend, HWPort15, 1>; +defm : HWWriteResPair<WriteShuffle256, HWPort5, 3>; + +def : WriteRes<WriteVarBlend, [HWPort5]> { + let Latency = 2; + let ResourceCycles = [2]; +} +def : WriteRes<WriteVarBlendLd, [HWPort5, HWPort23]> { + let Latency = 6; + let ResourceCycles = [2, 1]; +} + +def : WriteRes<WriteVarVecShift, [HWPort0, HWPort5]> { + let Latency = 2; + let ResourceCycles = [2, 1]; +} +def : WriteRes<WriteVarVecShiftLd, [HWPort0, HWPort5, HWPort23]> { + let Latency = 6; + let ResourceCycles = [2, 1, 1]; +} + +def : WriteRes<WriteMPSAD, [HWPort0, HWPort5]> { + let Latency = 6; + let ResourceCycles = [1, 2]; +} +def : WriteRes<WriteMPSADLd, [HWPort23, HWPort0, HWPort5]> { + let Latency = 6; + let ResourceCycles = [1, 1, 2]; +} + +// String instructions. +// Packed Compare Implicit Length Strings, Return Mask +def : WriteRes<WritePCmpIStrM, [HWPort0]> { + let Latency = 10; + let ResourceCycles = [3]; +} +def : WriteRes<WritePCmpIStrMLd, [HWPort0, HWPort23]> { + let Latency = 10; + let ResourceCycles = [3, 1]; +} + +// Packed Compare Explicit Length Strings, Return Mask +def : WriteRes<WritePCmpEStrM, [HWPort0, HWPort16, HWPort5]> { + let Latency = 10; + let ResourceCycles = [3, 2, 4]; +} +def : WriteRes<WritePCmpEStrMLd, [HWPort05, HWPort16, HWPort23]> { + let Latency = 10; + let ResourceCycles = [6, 2, 1]; +} + +// Packed Compare Implicit Length Strings, Return Index +def : WriteRes<WritePCmpIStrI, [HWPort0]> { + let Latency = 11; + let ResourceCycles = [3]; +} +def : WriteRes<WritePCmpIStrILd, [HWPort0, HWPort23]> { + let Latency = 11; + let ResourceCycles = [3, 1]; +} + +// Packed Compare Explicit Length Strings, Return Index +def : WriteRes<WritePCmpEStrI, [HWPort05, HWPort16]> { + let Latency = 11; + let ResourceCycles = [6, 2]; +} +def : WriteRes<WritePCmpEStrILd, [HWPort0, HWPort16, HWPort5, HWPort23]> { + let Latency = 11; + let ResourceCycles = [3, 2, 2, 1]; +} + +// AES Instructions. +def : WriteRes<WriteAESDecEnc, [HWPort5]> { + let Latency = 7; + let ResourceCycles = [1]; +} +def : WriteRes<WriteAESDecEncLd, [HWPort5, HWPort23]> { + let Latency = 7; + let ResourceCycles = [1, 1]; +} + +def : WriteRes<WriteAESIMC, [HWPort5]> { + let Latency = 14; + let ResourceCycles = [2]; +} +def : WriteRes<WriteAESIMCLd, [HWPort5, HWPort23]> { + let Latency = 14; + let ResourceCycles = [2, 1]; +} + +def : WriteRes<WriteAESKeyGen, [HWPort0, HWPort5]> { + let Latency = 10; + let ResourceCycles = [2, 8]; +} +def : WriteRes<WriteAESKeyGenLd, [HWPort0, HWPort5, HWPort23]> { + let Latency = 10; + let ResourceCycles = [2, 7, 1]; +} + +// Carry-less multiplication instructions. +def : WriteRes<WriteCLMul, [HWPort0, HWPort5]> { + let Latency = 7; + let ResourceCycles = [2, 1]; +} +def : WriteRes<WriteCLMulLd, [HWPort0, HWPort5, HWPort23]> { + let Latency = 7; + let ResourceCycles = [2, 1, 1]; +} def : WriteRes<WriteSystem, [HWPort0156]> { let Latency = 100; } def : WriteRes<WriteMicrocoded, [HWPort0156]> { let Latency = 100; } +def : WriteRes<WriteFence, [HWPort23, HWPort4]>; +def : WriteRes<WriteNop, []>; } // SchedModel diff --git a/lib/Target/X86/X86SchedSandyBridge.td b/lib/Target/X86/X86SchedSandyBridge.td index 3011c6d..a58859a 100644 --- a/lib/Target/X86/X86SchedSandyBridge.td +++ b/lib/Target/X86/X86SchedSandyBridge.td @@ -118,6 +118,16 @@ defm : SBWriteResPair<WriteFSqrt, SBPort0, 15>; defm : SBWriteResPair<WriteCvtF2I, SBPort1, 3>; defm : SBWriteResPair<WriteCvtI2F, SBPort1, 4>; defm : SBWriteResPair<WriteCvtF2F, SBPort1, 3>; +defm : SBWriteResPair<WriteFShuffle, SBPort5, 1>; +defm : SBWriteResPair<WriteFBlend, SBPort05, 1>; +def : WriteRes<WriteFVarBlend, [SBPort0, SBPort5]> { + let Latency = 2; + let ResourceCycles = [1, 1]; +} +def : WriteRes<WriteFVarBlendLd, [SBPort0, SBPort5, SBPort23]> { + let Latency = 6; + let ResourceCycles = [1, 1, 1]; +} // Vector integer operations. defm : SBWriteResPair<WriteVecShift, SBPort05, 1>; @@ -125,7 +135,112 @@ defm : SBWriteResPair<WriteVecLogic, SBPort015, 1>; defm : SBWriteResPair<WriteVecALU, SBPort15, 1>; defm : SBWriteResPair<WriteVecIMul, SBPort0, 5>; defm : SBWriteResPair<WriteShuffle, SBPort15, 1>; +defm : SBWriteResPair<WriteBlend, SBPort15, 1>; +def : WriteRes<WriteVarBlend, [SBPort1, SBPort5]> { + let Latency = 2; + let ResourceCycles = [1, 1]; +} +def : WriteRes<WriteVarBlendLd, [SBPort1, SBPort5, SBPort23]> { + let Latency = 6; + let ResourceCycles = [1, 1, 1]; +} +def : WriteRes<WriteMPSAD, [SBPort0, SBPort1, SBPort5]> { + let Latency = 6; + let ResourceCycles = [1, 1, 1]; +} +def : WriteRes<WriteMPSADLd, [SBPort0, SBPort1, SBPort5, SBPort23]> { + let Latency = 6; + let ResourceCycles = [1, 1, 1, 1]; +} + +// String instructions. +// Packed Compare Implicit Length Strings, Return Mask +def : WriteRes<WritePCmpIStrM, [SBPort015]> { + let Latency = 11; + let ResourceCycles = [3]; +} +def : WriteRes<WritePCmpIStrMLd, [SBPort015, SBPort23]> { + let Latency = 11; + let ResourceCycles = [3, 1]; +} + +// Packed Compare Explicit Length Strings, Return Mask +def : WriteRes<WritePCmpEStrM, [SBPort015]> { + let Latency = 11; + let ResourceCycles = [8]; +} +def : WriteRes<WritePCmpEStrMLd, [SBPort015, SBPort23]> { + let Latency = 11; + let ResourceCycles = [7, 1]; +} + +// Packed Compare Implicit Length Strings, Return Index +def : WriteRes<WritePCmpIStrI, [SBPort015]> { + let Latency = 3; + let ResourceCycles = [3]; +} +def : WriteRes<WritePCmpIStrILd, [SBPort015, SBPort23]> { + let Latency = 3; + let ResourceCycles = [3, 1]; +} + +// Packed Compare Explicit Length Strings, Return Index +def : WriteRes<WritePCmpEStrI, [SBPort015]> { + let Latency = 4; + let ResourceCycles = [8]; +} +def : WriteRes<WritePCmpEStrILd, [SBPort015, SBPort23]> { + let Latency = 4; + let ResourceCycles = [7, 1]; +} + +// AES Instructions. +def : WriteRes<WriteAESDecEnc, [SBPort015]> { + let Latency = 8; + let ResourceCycles = [2]; +} +def : WriteRes<WriteAESDecEncLd, [SBPort015, SBPort23]> { + let Latency = 8; + let ResourceCycles = [2, 1]; +} + +def : WriteRes<WriteAESIMC, [SBPort015]> { + let Latency = 8; + let ResourceCycles = [2]; +} +def : WriteRes<WriteAESIMCLd, [SBPort015, SBPort23]> { + let Latency = 8; + let ResourceCycles = [2, 1]; +} + +def : WriteRes<WriteAESKeyGen, [SBPort015]> { + let Latency = 8; + let ResourceCycles = [11]; +} +def : WriteRes<WriteAESKeyGenLd, [SBPort015, SBPort23]> { + let Latency = 8; + let ResourceCycles = [10, 1]; +} + +// Carry-less multiplication instructions. +def : WriteRes<WriteCLMul, [SBPort015]> { + let Latency = 14; + let ResourceCycles = [18]; +} +def : WriteRes<WriteCLMulLd, [SBPort015, SBPort23]> { + let Latency = 14; + let ResourceCycles = [17, 1]; +} + def : WriteRes<WriteSystem, [SBPort015]> { let Latency = 100; } def : WriteRes<WriteMicrocoded, [SBPort015]> { let Latency = 100; } +def : WriteRes<WriteFence, [SBPort23, SBPort4]>; +def : WriteRes<WriteNop, []>; + +// AVX2 is not supported on that architecture, but we should define the basic +// scheduling resources anyway. +defm : SBWriteResPair<WriteFShuffle256, SBPort0, 1>; +defm : SBWriteResPair<WriteShuffle256, SBPort0, 1>; +defm : SBWriteResPair<WriteVarVecShift, SBPort0, 1>; } // SchedModel diff --git a/lib/Target/X86/X86Schedule.td b/lib/Target/X86/X86Schedule.td index 0556437..25c5a6b 100644 --- a/lib/Target/X86/X86Schedule.td +++ b/lib/Target/X86/X86Schedule.td @@ -69,6 +69,9 @@ defm WriteFDiv : X86SchedWritePair; // Floating point division. defm WriteFSqrt : X86SchedWritePair; // Floating point square root. defm WriteFRcp : X86SchedWritePair; // Floating point reciprocal. defm WriteFMA : X86SchedWritePair; // Fused Multiply Add. +defm WriteFShuffle : X86SchedWritePair; // Floating point vector shuffles. +defm WriteFBlend : X86SchedWritePair; // Floating point vector blends. +defm WriteFVarBlend : X86SchedWritePair; // Fp vector variable blends. // FMA Scheduling helper class. class FMASC { X86FoldableSchedWrite Sched = WriteFAdd; } @@ -77,23 +80,55 @@ class FMASC { X86FoldableSchedWrite Sched = WriteFAdd; } defm WriteVecALU : X86SchedWritePair; // Vector integer ALU op, no logicals. defm WriteVecShift : X86SchedWritePair; // Vector integer shifts. defm WriteVecIMul : X86SchedWritePair; // Vector integer multiply. +defm WriteShuffle : X86SchedWritePair; // Vector shuffles. +defm WriteBlend : X86SchedWritePair; // Vector blends. +defm WriteVarBlend : X86SchedWritePair; // Vector variable blends. +defm WriteMPSAD : X86SchedWritePair; // Vector MPSAD. // Vector bitwise operations. // These are often used on both floating point and integer vectors. defm WriteVecLogic : X86SchedWritePair; // Vector and/or/xor. -defm WriteShuffle : X86SchedWritePair; // Vector shuffles and blends. // Conversion between integer and float. defm WriteCvtF2I : X86SchedWritePair; // Float -> Integer. defm WriteCvtI2F : X86SchedWritePair; // Integer -> Float. defm WriteCvtF2F : X86SchedWritePair; // Float -> Float size conversion. +// Strings instructions. +// Packed Compare Implicit Length Strings, Return Mask +defm WritePCmpIStrM : X86SchedWritePair; +// Packed Compare Explicit Length Strings, Return Mask +defm WritePCmpEStrM : X86SchedWritePair; +// Packed Compare Implicit Length Strings, Return Index +defm WritePCmpIStrI : X86SchedWritePair; +// Packed Compare Explicit Length Strings, Return Index +defm WritePCmpEStrI : X86SchedWritePair; + +// AES instructions. +defm WriteAESDecEnc : X86SchedWritePair; // Decryption, encryption. +defm WriteAESIMC : X86SchedWritePair; // InvMixColumn. +defm WriteAESKeyGen : X86SchedWritePair; // Key Generation. + +// Carry-less multiplication instructions. +defm WriteCLMul : X86SchedWritePair; + // Catch-all for expensive system instructions. def WriteSystem : SchedWrite; +// AVX2. +defm WriteFShuffle256 : X86SchedWritePair; // Fp 256-bit width vector shuffles. +defm WriteShuffle256 : X86SchedWritePair; // 256-bit width vector shuffles. +defm WriteVarVecShift : X86SchedWritePair; // Variable vector shifts. + // Old microcoded instructions that nobody use. def WriteMicrocoded : SchedWrite; +// Fence instructions. +def WriteFence : SchedWrite; + +// Nop, not very useful expect it provides a model for nops! +def WriteNop : SchedWrite; + //===----------------------------------------------------------------------===// // Instruction Itinerary classes used for X86 def IIC_ALU_MEM : InstrItinClass; @@ -577,7 +612,7 @@ def IIC_NOP : InstrItinClass; //===----------------------------------------------------------------------===// // Processor instruction itineraries. -// IssueWidth is analagous to the number of decode units. Core and its +// IssueWidth is analogous to the number of decode units. Core and its // descendents, including Nehalem and SandyBridge have 4 decoders. // Resources beyond the decoder operate on micro-ops and are bufferred // so adjacent micro-ops don't directly compete. diff --git a/lib/Target/X86/X86SelectionDAGInfo.h b/lib/Target/X86/X86SelectionDAGInfo.h index d728af5..0d5dc38 100644 --- a/lib/Target/X86/X86SelectionDAGInfo.h +++ b/lib/Target/X86/X86SelectionDAGInfo.h @@ -33,22 +33,20 @@ public: explicit X86SelectionDAGInfo(const X86TargetMachine &TM); ~X86SelectionDAGInfo(); - virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVolatile, - MachinePointerInfo DstPtrInfo) const; + MachinePointerInfo DstPtrInfo) const override; - virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) const; + MachinePointerInfo SrcPtrInfo) const override; }; } diff --git a/lib/Target/X86/X86Subtarget.cpp b/lib/Target/X86/X86Subtarget.cpp index 01353b2..207d0ba 100644 --- a/lib/Target/X86/X86Subtarget.cpp +++ b/lib/Target/X86/X86Subtarget.cpp @@ -55,7 +55,7 @@ unsigned char X86Subtarget:: ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const { // DLLImport only exists on windows, it is implemented as a load from a // DLLIMPORT stub. - if (GV->hasDLLImportLinkage()) + if (GV->hasDLLImportStorageClass()) return X86II::MO_DLLIMPORT; // Determine whether this is a reference to a definition or a declaration. @@ -165,7 +165,10 @@ bool X86Subtarget::hasSinCos() const { /// IsLegalToCallImmediateAddr - Return true if the subtarget allows calls /// to immediate address. bool X86Subtarget::IsLegalToCallImmediateAddr(const TargetMachine &TM) const { - if (In64BitMode) + // FIXME: I386 PE/COFF supports PC relative calls using IMAGE_REL_I386_REL32 + // but WinCOFFObjectWriter::RecordRelocation cannot emit them. Once it does, + // the following check for Win32 should be removed. + if (In64BitMode || isTargetWin32()) return false; return isTargetELF() || TM.getRelocationModel() == Reloc::Static; } @@ -263,6 +266,15 @@ void X86Subtarget::AutoDetectSubtargetFeatures() { ToggleFeature(X86::FeatureSlowBTMem); } + // Determine if SHLD/SHRD instructions have higher latency then the + // equivalent series of shifts/or instructions. + // FIXME: Add Intel's processors that have SHLD instructions with very + // poor latency. + if (IsAMD) { + IsSHLDSlow = true; + ToggleFeature(X86::FeatureSlowSHLD); + } + // If it's an Intel chip since Nehalem and not an Atom chip, unaligned // memory access is fast. We hard code model numbers here because they // aren't strictly increasing for Intel chips it seems. @@ -473,6 +485,12 @@ void X86Subtarget::resetSubtargetFeatures(StringRef CPU, StringRef FS) { // target data structure which is shared with MC code emitter, etc. if (In64BitMode) ToggleFeature(X86::Mode64Bit); + else if (In32BitMode) + ToggleFeature(X86::Mode32Bit); + else if (In16BitMode) + ToggleFeature(X86::Mode16Bit); + else + llvm_unreachable("Not 16-bit, 32-bit or 64-bit mode!"); DEBUG(dbgs() << "Subtarget features: SSELevel " << X86SSELevel << ", 3DNowLevel " << X863DNowLevel @@ -519,6 +537,7 @@ void X86Subtarget::initializeEnvironment() { HasPRFCHW = false; HasRDSEED = false; IsBTMemSlow = false; + IsSHLDSlow = false; IsUAMemFast = false; HasVectorUAMem = false; HasCmpxchg16b = false; @@ -535,13 +554,17 @@ void X86Subtarget::initializeEnvironment() { X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU, const std::string &FS, - unsigned StackAlignOverride, bool is64Bit) + unsigned StackAlignOverride) : X86GenSubtargetInfo(TT, CPU, FS) , X86ProcFamily(Others) , PICStyle(PICStyles::None) , TargetTriple(TT) , StackAlignOverride(StackAlignOverride) - , In64BitMode(is64Bit) { + , In64BitMode(TargetTriple.getArch() == Triple::x86_64) + , In32BitMode(TargetTriple.getArch() == Triple::x86 && + TargetTriple.getEnvironment() != Triple::CODE16) + , In16BitMode(TargetTriple.getArch() == Triple::x86 && + TargetTriple.getEnvironment() == Triple::CODE16) { initializeEnvironment(); resetSubtargetFeatures(CPU, FS); } diff --git a/lib/Target/X86/X86Subtarget.h b/lib/Target/X86/X86Subtarget.h index dd8c081..52986b9 100644 --- a/lib/Target/X86/X86Subtarget.h +++ b/lib/Target/X86/X86Subtarget.h @@ -39,7 +39,7 @@ enum Style { }; } -class X86Subtarget : public X86GenSubtargetInfo { +class X86Subtarget final : public X86GenSubtargetInfo { protected: enum X86SSEEnum { NoMMXSSE, MMX, SSE1, SSE2, SSE3, SSSE3, SSE41, SSE42, AVX, AVX2, AVX512F @@ -142,6 +142,9 @@ protected: /// IsBTMemSlow - True if BT (bit test) of memory instructions are slow. bool IsBTMemSlow; + /// IsSHLDSlow - True if SHLD instructions are slow. + bool IsSHLDSlow; + /// IsUAMemFast - True if unaligned memory access is fast. bool IsUAMemFast; @@ -202,16 +205,22 @@ private: /// StackAlignOverride - Override the stack alignment. unsigned StackAlignOverride; - /// In64BitMode - True if compiling for 64-bit, false for 32-bit. + /// In64BitMode - True if compiling for 64-bit, false for 16-bit or 32-bit. bool In64BitMode; + /// In32BitMode - True if compiling for 32-bit, false for 16-bit or 64-bit. + bool In32BitMode; + + /// In16BitMode - True if compiling for 16-bit, false for 32-bit or 64-bit. + bool In16BitMode; + public: /// This constructor initializes the data members to match that /// of the specified triple. /// X86Subtarget(const std::string &TT, const std::string &CPU, const std::string &FS, - unsigned StackAlignOverride, bool is64Bit); + unsigned StackAlignOverride); /// getStackAlignment - Returns the minimum alignment known to hold of the /// stack frame on entry to the function and which must be maintained by every @@ -231,7 +240,7 @@ public: void AutoDetectSubtargetFeatures(); /// \brief Reset the features for the X86 target. - virtual void resetSubtargetFeatures(const MachineFunction *MF); + void resetSubtargetFeatures(const MachineFunction *MF) override; private: void initializeEnvironment(); void resetSubtargetFeatures(StringRef CPU, StringRef FS); @@ -241,9 +250,18 @@ public: return In64BitMode; } + bool is32Bit() const { + return In32BitMode; + } + + bool is16Bit() const { + return In16BitMode; + } + /// Is this x86_64 with the ILP32 programming model (x32 ABI)? bool isTarget64BitILP32() const { - return In64BitMode && (TargetTriple.getEnvironment() == Triple::GNUX32); + return In64BitMode && (TargetTriple.getEnvironment() == Triple::GNUX32 || + TargetTriple.getOS() == Triple::NaCl); } /// Is this x86_64 with the LP64 programming model (standard AMD64, no x32)? @@ -292,6 +310,7 @@ public: bool hasPRFCHW() const { return HasPRFCHW; } bool hasRDSEED() const { return HasRDSEED; } bool isBTMemSlow() const { return IsBTMemSlow; } + bool isSHLDSlow() const { return IsSHLDSlow; } bool isUnalignedMemAccessFast() const { return IsUAMemFast; } bool hasVectorUAMem() const { return HasVectorUAMem; } bool hasCmpxchg16b() const { return HasCmpxchg16b; } @@ -315,23 +334,33 @@ public: bool isTargetSolaris() const { return TargetTriple.getOS() == Triple::Solaris; } - bool isTargetELF() const { - return (TargetTriple.getEnvironment() == Triple::ELF || - TargetTriple.isOSBinFormatELF()); - } + + bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); } + bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); } + bool isTargetMacho() const { return TargetTriple.isOSBinFormatMachO(); } + bool isTargetLinux() const { return TargetTriple.isOSLinux(); } bool isTargetNaCl() const { return TargetTriple.isOSNaCl(); } bool isTargetNaCl32() const { return isTargetNaCl() && !is64Bit(); } bool isTargetNaCl64() const { return isTargetNaCl() && is64Bit(); } - bool isTargetWindows() const { return TargetTriple.getOS() == Triple::Win32; } - bool isTargetMingw() const { return TargetTriple.getOS() == Triple::MinGW32; } - bool isTargetCygwin() const { return TargetTriple.getOS() == Triple::Cygwin; } - bool isTargetCygMing() const { return TargetTriple.isOSCygMing(); } - bool isTargetCOFF() const { - return (TargetTriple.getEnvironment() != Triple::ELF && - TargetTriple.isOSBinFormatCOFF()); + + bool isTargetWindowsMSVC() const { + return TargetTriple.isWindowsMSVCEnvironment(); + } + + bool isTargetKnownWindowsMSVC() const { + return TargetTriple.isKnownWindowsMSVCEnvironment(); + } + + bool isTargetWindowsCygwin() const { + return TargetTriple.isWindowsCygwinEnvironment(); + } + + bool isTargetWindowsGNU() const { + return TargetTriple.isWindowsGNUEnvironment(); } - bool isTargetEnvMacho() const { return TargetTriple.isEnvironmentMachO(); } + + bool isTargetCygMing() const { return TargetTriple.isOSCygMing(); } bool isOSWindows() const { return TargetTriple.isOSWindows(); } @@ -340,7 +369,7 @@ public: } bool isTargetWin32() const { - return !In64BitMode && (isTargetCygMing() || isTargetWindows()); + return !In64BitMode && (isTargetCygMing() || isTargetKnownWindowsMSVC()); } bool isPICStyleSet() const { return PICStyle != PICStyles::None; } @@ -391,12 +420,12 @@ public: bool hasSinCos() const; /// Enable the MachineScheduler pass for all X86 subtargets. - bool enableMachineScheduler() const LLVM_OVERRIDE { return true; } + bool enableMachineScheduler() const override { return true; } /// enablePostRAScheduler - run for Atom optimization. bool enablePostRAScheduler(CodeGenOpt::Level OptLevel, TargetSubtargetInfo::AntiDepBreakMode& Mode, - RegClassVector& CriticalPathRCs) const; + RegClassVector& CriticalPathRCs) const override; bool postRAScheduler() const { return PostRAScheduler; } diff --git a/lib/Target/X86/X86TargetMachine.cpp b/lib/Target/X86/X86TargetMachine.cpp index ddf580f..6f09ccf 100644 --- a/lib/Target/X86/X86TargetMachine.cpp +++ b/lib/Target/X86/X86TargetMachine.cpp @@ -13,7 +13,6 @@ #include "X86TargetMachine.h" #include "X86.h" -#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/Passes.h" #include "llvm/PassManager.h" #include "llvm/Support/CommandLine.h" @@ -24,53 +23,49 @@ using namespace llvm; extern "C" void LLVMInitializeX86Target() { // Register the target. - RegisterTargetMachine<X86_32TargetMachine> X(TheX86_32Target); - RegisterTargetMachine<X86_64TargetMachine> Y(TheX86_64Target); + RegisterTargetMachine<X86TargetMachine> X(TheX86_32Target); + RegisterTargetMachine<X86TargetMachine> Y(TheX86_64Target); } -void X86_32TargetMachine::anchor() { } - -X86_32TargetMachine::X86_32TargetMachine(const Target &T, StringRef TT, - StringRef CPU, StringRef FS, - const TargetOptions &Options, - Reloc::Model RM, CodeModel::Model CM, - CodeGenOpt::Level OL) - : X86TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false), - DL(getSubtargetImpl()->isTargetDarwin() ? - "e-p:32:32-f64:32:64-i64:32:64-f80:128:128-f128:128:128-" - "n8:16:32-S128" : - (getSubtargetImpl()->isTargetCygMing() || - getSubtargetImpl()->isTargetWindows()) ? - "e-p:32:32-f64:64:64-i64:64:64-f80:32:32-f128:128:128-" - "n8:16:32-S32" : - "e-p:32:32-f64:32:64-i64:32:64-f80:32:32-f128:128:128-" - "n8:16:32-S128"), - InstrInfo(*this), - TLInfo(*this), - TSInfo(*this), - JITInfo(*this) { - initAsmInfo(); -} - -void X86_64TargetMachine::anchor() { } - -X86_64TargetMachine::X86_64TargetMachine(const Target &T, StringRef TT, - StringRef CPU, StringRef FS, - const TargetOptions &Options, - Reloc::Model RM, CodeModel::Model CM, - CodeGenOpt::Level OL) - : X86TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true), - // The x32 ABI dictates the ILP32 programming model for x64. - DL(getSubtargetImpl()->isTarget64BitILP32() ? - "e-p:32:32-s:64-f64:64:64-i64:64:64-f80:128:128-f128:128:128-" - "n8:16:32:64-S128" : - "e-p:64:64-s:64-f64:64:64-i64:64:64-f80:128:128-f128:128:128-" - "n8:16:32:64-S128"), - InstrInfo(*this), - TLInfo(*this), - TSInfo(*this), - JITInfo(*this) { - initAsmInfo(); +void X86TargetMachine::anchor() { } + +static std::string computeDataLayout(const X86Subtarget &ST) { + // X86 is little endian + std::string Ret = "e"; + + Ret += DataLayout::getManglingComponent(ST.getTargetTriple()); + // X86 and x32 have 32 bit pointers. + if (ST.isTarget64BitILP32() || !ST.is64Bit()) + Ret += "-p:32:32"; + + // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. + if (ST.is64Bit() || ST.isTargetCygMing() || ST.isTargetKnownWindowsMSVC() || + ST.isTargetNaCl()) + Ret += "-i64:64"; + else + Ret += "-f64:32:64"; + + // Some ABIs align long double to 128 bits, others to 32. + if (ST.isTargetNaCl()) + ; // No f80 + else if (ST.is64Bit() || ST.isTargetDarwin()) + Ret += "-f80:128"; + else + Ret += "-f80:32"; + + // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. + if (ST.is64Bit()) + Ret += "-n8:16:32:64"; + else + Ret += "-n8:16:32"; + + // The stack is aligned to 32 bits on some ABIs and 128 bits on others. + if (!ST.is64Bit() && (ST.isTargetCygMing() || ST.isTargetKnownWindowsMSVC())) + Ret += "-S32"; + else + Ret += "-S128"; + + return Ret; } /// X86TargetMachine ctor - Create an X86 target. @@ -79,12 +74,16 @@ X86TargetMachine::X86TargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS, const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM, - CodeGenOpt::Level OL, - bool is64Bit) + CodeGenOpt::Level OL) : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL), - Subtarget(TT, CPU, FS, Options.StackAlignmentOverride, is64Bit), + Subtarget(TT, CPU, FS, Options.StackAlignmentOverride), FrameLowering(*this, Subtarget), - InstrItins(Subtarget.getInstrItineraryData()){ + InstrItins(Subtarget.getInstrItineraryData()), + DL(computeDataLayout(*getSubtargetImpl())), + InstrInfo(*this), + TLInfo(*this), + TSInfo(*this), + JITInfo(*this) { // Determine the PICStyle based on the target selected. if (getRelocationModel() == Reloc::Static) { // Unless we're in PIC or DynamicNoPIC mode, set the PIC style to None. @@ -108,6 +107,8 @@ X86TargetMachine::X86TargetMachine(const Target &T, StringRef TT, // default to hard float ABI if (Options.FloatABIType == FloatABI::Default) this->Options.FloatABIType = FloatABI::Hard; + + initAsmInfo(); } //===----------------------------------------------------------------------===// @@ -156,11 +157,11 @@ public: return *getX86TargetMachine().getSubtargetImpl(); } - virtual bool addInstSelector(); - virtual bool addILPOpts(); - virtual bool addPreRegAlloc(); - virtual bool addPostRegAlloc(); - virtual bool addPreEmitPass(); + bool addInstSelector() override; + bool addILPOpts() override; + bool addPreRegAlloc() override; + bool addPostRegAlloc() override; + bool addPreEmitPass() override; }; } // namespace diff --git a/lib/Target/X86/X86TargetMachine.h b/lib/Target/X86/X86TargetMachine.h index 174d391..57e6eda 100644 --- a/lib/Target/X86/X86TargetMachine.h +++ b/lib/Target/X86/X86TargetMachine.h @@ -14,7 +14,6 @@ #ifndef X86TARGETMACHINE_H #define X86TARGETMACHINE_H -#include "X86.h" #include "X86FrameLowering.h" #include "X86ISelLowering.h" #include "X86InstrInfo.h" @@ -22,114 +21,60 @@ #include "X86SelectionDAGInfo.h" #include "X86Subtarget.h" #include "llvm/IR/DataLayout.h" -#include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetMachine.h" namespace llvm { class StringRef; -class X86TargetMachine : public LLVMTargetMachine { +class X86TargetMachine final : public LLVMTargetMachine { + virtual void anchor(); X86Subtarget Subtarget; X86FrameLowering FrameLowering; InstrItineraryData InstrItins; + const DataLayout DL; // Calculates type size & alignment + X86InstrInfo InstrInfo; + X86TargetLowering TLInfo; + X86SelectionDAGInfo TSInfo; + X86JITInfo JITInfo; public: X86TargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS, const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM, - CodeGenOpt::Level OL, - bool is64Bit); + CodeGenOpt::Level OL); - virtual const X86InstrInfo *getInstrInfo() const { - llvm_unreachable("getInstrInfo not implemented"); + const DataLayout *getDataLayout() const override { return &DL; } + const X86InstrInfo *getInstrInfo() const override { + return &InstrInfo; } - virtual const TargetFrameLowering *getFrameLowering() const { + const TargetFrameLowering *getFrameLowering() const override { return &FrameLowering; } - virtual X86JITInfo *getJITInfo() { - llvm_unreachable("getJITInfo not implemented"); + X86JITInfo *getJITInfo() override { + return &JITInfo; } - virtual const X86Subtarget *getSubtargetImpl() const{ return &Subtarget; } - virtual const X86TargetLowering *getTargetLowering() const { - llvm_unreachable("getTargetLowering not implemented"); + const X86Subtarget *getSubtargetImpl() const override { return &Subtarget; } + const X86TargetLowering *getTargetLowering() const override { + return &TLInfo; } - virtual const X86SelectionDAGInfo *getSelectionDAGInfo() const { - llvm_unreachable("getSelectionDAGInfo not implemented"); + const X86SelectionDAGInfo *getSelectionDAGInfo() const override { + return &TSInfo; } - virtual const X86RegisterInfo *getRegisterInfo() const { + const X86RegisterInfo *getRegisterInfo() const override { return &getInstrInfo()->getRegisterInfo(); } - virtual const InstrItineraryData *getInstrItineraryData() const { + const InstrItineraryData *getInstrItineraryData() const override { return &InstrItins; } /// \brief Register X86 analysis passes with a pass manager. - virtual void addAnalysisPasses(PassManagerBase &PM); + void addAnalysisPasses(PassManagerBase &PM) override; // Set up the pass pipeline. - virtual TargetPassConfig *createPassConfig(PassManagerBase &PM); - - virtual bool addCodeEmitter(PassManagerBase &PM, - JITCodeEmitter &JCE); -}; - -/// X86_32TargetMachine - X86 32-bit target machine. -/// -class X86_32TargetMachine : public X86TargetMachine { - virtual void anchor(); - const DataLayout DL; // Calculates type size & alignment - X86InstrInfo InstrInfo; - X86TargetLowering TLInfo; - X86SelectionDAGInfo TSInfo; - X86JITInfo JITInfo; -public: - X86_32TargetMachine(const Target &T, StringRef TT, - StringRef CPU, StringRef FS, const TargetOptions &Options, - Reloc::Model RM, CodeModel::Model CM, - CodeGenOpt::Level OL); - virtual const DataLayout *getDataLayout() const { return &DL; } - virtual const X86TargetLowering *getTargetLowering() const { - return &TLInfo; - } - virtual const X86SelectionDAGInfo *getSelectionDAGInfo() const { - return &TSInfo; - } - virtual const X86InstrInfo *getInstrInfo() const { - return &InstrInfo; - } - virtual X86JITInfo *getJITInfo() { - return &JITInfo; - } -}; + TargetPassConfig *createPassConfig(PassManagerBase &PM) override; -/// X86_64TargetMachine - X86 64-bit target machine. -/// -class X86_64TargetMachine : public X86TargetMachine { - virtual void anchor(); - const DataLayout DL; // Calculates type size & alignment - X86InstrInfo InstrInfo; - X86TargetLowering TLInfo; - X86SelectionDAGInfo TSInfo; - X86JITInfo JITInfo; -public: - X86_64TargetMachine(const Target &T, StringRef TT, - StringRef CPU, StringRef FS, const TargetOptions &Options, - Reloc::Model RM, CodeModel::Model CM, - CodeGenOpt::Level OL); - virtual const DataLayout *getDataLayout() const { return &DL; } - virtual const X86TargetLowering *getTargetLowering() const { - return &TLInfo; - } - virtual const X86SelectionDAGInfo *getSelectionDAGInfo() const { - return &TSInfo; - } - virtual const X86InstrInfo *getInstrInfo() const { - return &InstrInfo; - } - virtual X86JITInfo *getJITInfo() { - return &JITInfo; - } + bool addCodeEmitter(PassManagerBase &PM, JITCodeEmitter &JCE) override; }; } // End llvm namespace diff --git a/lib/Target/X86/X86TargetObjectFile.cpp b/lib/Target/X86/X86TargetObjectFile.cpp index 086cd4d..0a88e98 100644 --- a/lib/Target/X86/X86TargetObjectFile.cpp +++ b/lib/Target/X86/X86TargetObjectFile.cpp @@ -8,38 +8,40 @@ //===----------------------------------------------------------------------===// #include "X86TargetObjectFile.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Operator.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/Support/Dwarf.h" -#include "llvm/Target/Mangler.h" +#include "llvm/Target/TargetLowering.h" using namespace llvm; using namespace dwarf; -const MCExpr *X86_64MachoTargetObjectFile:: -getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI, unsigned Encoding, - MCStreamer &Streamer) const { +const MCExpr *X86_64MachoTargetObjectFile::getTTypeGlobalReference( + const GlobalValue *GV, unsigned Encoding, Mangler &Mang, + const TargetMachine &TM, MachineModuleInfo *MMI, + MCStreamer &Streamer) const { // On Darwin/X86-64, we can reference dwarf symbols with foo@GOTPCREL+4, which // is an indirect pc-relative reference. if (Encoding & (DW_EH_PE_indirect | DW_EH_PE_pcrel)) { - const MCSymbol *Sym = getSymbol(*Mang, GV); + const MCSymbol *Sym = TM.getSymbol(GV, Mang); const MCExpr *Res = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTPCREL, getContext()); const MCExpr *Four = MCConstantExpr::Create(4, getContext()); return MCBinaryExpr::CreateAdd(Res, Four, getContext()); } - return TargetLoweringObjectFileMachO:: - getTTypeGlobalReference(GV, Mang, MMI, Encoding, Streamer); + return TargetLoweringObjectFileMachO::getTTypeGlobalReference( + GV, Encoding, Mang, TM, MMI, Streamer); } -MCSymbol *X86_64MachoTargetObjectFile:: -getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI) const { - return getSymbol(*Mang, GV); +MCSymbol *X86_64MachoTargetObjectFile::getCFIPersonalitySymbol( + const GlobalValue *GV, Mangler &Mang, const TargetMachine &TM, + MachineModuleInfo *MMI) const { + return TM.getSymbol(GV, Mang); } void @@ -53,3 +55,54 @@ X86LinuxTargetObjectFile::getDebugThreadLocalSymbol( const MCSymbol *Sym) const { return MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_DTPOFF, getContext()); } + +const MCExpr *X86WindowsTargetObjectFile::getExecutableRelativeSymbol( + const ConstantExpr *CE, Mangler &Mang, const TargetMachine &TM) const { + // We are looking for the difference of two symbols, need a subtraction + // operation. + const SubOperator *Sub = dyn_cast<SubOperator>(CE); + if (!Sub) + return 0; + + // Symbols must first be numbers before we can subtract them, we need to see a + // ptrtoint on both subtraction operands. + const PtrToIntOperator *SubLHS = + dyn_cast<PtrToIntOperator>(Sub->getOperand(0)); + const PtrToIntOperator *SubRHS = + dyn_cast<PtrToIntOperator>(Sub->getOperand(1)); + if (!SubLHS || !SubRHS) + return 0; + + // Our symbols should exist in address space zero, cowardly no-op if + // otherwise. + if (SubLHS->getPointerAddressSpace() != 0 || + SubRHS->getPointerAddressSpace() != 0) + return 0; + + // Both ptrtoint instructions must wrap global variables: + // - Only global variables are eligible for image relative relocations. + // - The subtrahend refers to the special symbol __ImageBase, a global. + const GlobalVariable *GVLHS = + dyn_cast<GlobalVariable>(SubLHS->getPointerOperand()); + const GlobalVariable *GVRHS = + dyn_cast<GlobalVariable>(SubRHS->getPointerOperand()); + if (!GVLHS || !GVRHS) + return 0; + + // We expect __ImageBase to be a global variable without a section, externally + // defined. + // + // It should look something like this: @__ImageBase = external constant i8 + if (GVRHS->isThreadLocal() || GVRHS->getName() != "__ImageBase" || + !GVRHS->hasExternalLinkage() || GVRHS->hasInitializer() || + GVRHS->hasSection()) + return 0; + + // An image-relative, thread-local, symbol makes no sense. + if (GVLHS->isThreadLocal()) + return 0; + + return MCSymbolRefExpr::Create(TM.getSymbol(GVLHS, Mang), + MCSymbolRefExpr::VK_COFF_IMGREL32, + getContext()); +} diff --git a/lib/Target/X86/X86TargetObjectFile.h b/lib/Target/X86/X86TargetObjectFile.h index 79c861d..a08ed09 100644 --- a/lib/Target/X86/X86TargetObjectFile.h +++ b/lib/Target/X86/X86TargetObjectFile.h @@ -12,7 +12,6 @@ #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/Target/TargetLoweringObjectFile.h" -#include "llvm/Target/TargetMachine.h" namespace llvm { @@ -20,25 +19,33 @@ namespace llvm { /// x86-64. class X86_64MachoTargetObjectFile : public TargetLoweringObjectFileMachO { public: - virtual const MCExpr * - getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI, unsigned Encoding, - MCStreamer &Streamer) const; + const MCExpr * + getTTypeGlobalReference(const GlobalValue *GV, unsigned Encoding, + Mangler &Mang, const TargetMachine &TM, + MachineModuleInfo *MMI, + MCStreamer &Streamer) const override; // getCFIPersonalitySymbol - The symbol that gets passed to // .cfi_personality. - virtual MCSymbol * - getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI) const; + MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang, + const TargetMachine &TM, + MachineModuleInfo *MMI) const override; }; /// X86LinuxTargetObjectFile - This implementation is used for linux x86 /// and x86-64. class X86LinuxTargetObjectFile : public TargetLoweringObjectFileELF { - virtual void Initialize(MCContext &Ctx, const TargetMachine &TM); + void Initialize(MCContext &Ctx, const TargetMachine &TM) override; /// \brief Describe a TLS variable address within debug info. - virtual const MCExpr *getDebugThreadLocalSymbol(const MCSymbol *Sym) const; + const MCExpr *getDebugThreadLocalSymbol(const MCSymbol *Sym) const override; + }; + + /// \brief This implementation is used for Windows targets on x86 and x86-64. + class X86WindowsTargetObjectFile : public TargetLoweringObjectFileCOFF { + const MCExpr * + getExecutableRelativeSymbol(const ConstantExpr *CE, Mangler &Mang, + const TargetMachine &TM) const override; }; } // end namespace llvm diff --git a/lib/Target/X86/X86TargetTransformInfo.cpp b/lib/Target/X86/X86TargetTransformInfo.cpp index f88a666..c04964d 100644 --- a/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/lib/Target/X86/X86TargetTransformInfo.cpp @@ -17,10 +17,14 @@ #define DEBUG_TYPE "x86tti" #include "X86.h" #include "X86TargetMachine.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" -#include "llvm/Target/TargetLowering.h" #include "llvm/Target/CostTable.h" +#include "llvm/Target/TargetLowering.h" using namespace llvm; // Declare the pass initialization routine locally as target-specific passes @@ -30,9 +34,20 @@ namespace llvm { void initializeX86TTIPass(PassRegistry &); } +static cl::opt<bool> +UsePartialUnrolling("x86-use-partial-unrolling", cl::init(true), + cl::desc("Use partial unrolling for some X86 targets"), cl::Hidden); +static cl::opt<unsigned> +PartialUnrollingThreshold("x86-partial-unrolling-threshold", cl::init(0), + cl::desc("Threshold for X86 partial unrolling"), cl::Hidden); +static cl::opt<unsigned> +PartialUnrollingMaxBranches("x86-partial-max-branches", cl::init(2), + cl::desc("Threshold for taken branches in X86 partial unrolling"), + cl::Hidden); + namespace { -class X86TTI : public ImmutablePass, public TargetTransformInfo { +class X86TTI final : public ImmutablePass, public TargetTransformInfo { const X86Subtarget *ST; const X86TargetLowering *TLI; @@ -46,20 +61,16 @@ public: } X86TTI(const X86TargetMachine *TM) - : ImmutablePass(ID), ST(TM->getSubtargetImpl()), - TLI(TM->getTargetLowering()) { + : ImmutablePass(ID), ST(TM->getSubtargetImpl()), + TLI(TM->getTargetLowering()) { initializeX86TTIPass(*PassRegistry::getPassRegistry()); } - virtual void initializePass() { + void initializePass() override { pushTTIStack(this); } - virtual void finalizePass() { - popTTIStack(); - } - - virtual void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { TargetTransformInfo::getAnalysisUsage(AU); } @@ -67,7 +78,7 @@ public: static char ID; /// Provide necessary pointer adjustments for the two base classes. - virtual void *getAdjustedAnalysisPointer(const void *ID) { + void *getAdjustedAnalysisPointer(const void *ID) override { if (ID == &TargetTransformInfo::ID) return (TargetTransformInfo*)this; return this; @@ -75,35 +86,43 @@ public: /// \name Scalar TTI Implementations /// @{ - virtual PopcntSupportKind getPopcntSupport(unsigned TyWidth) const; + PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override; + void getUnrollingPreferences(Loop *L, + UnrollingPreferences &UP) const override; /// @} /// \name Vector TTI Implementations /// @{ - virtual unsigned getNumberOfRegisters(bool Vector) const; - virtual unsigned getRegisterBitWidth(bool Vector) const; - virtual unsigned getMaximumUnrollFactor() const; - virtual unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, - OperandValueKind, - OperandValueKind) const; - virtual unsigned getShuffleCost(ShuffleKind Kind, Type *Tp, - int Index, Type *SubTp) const; - virtual unsigned getCastInstrCost(unsigned Opcode, Type *Dst, - Type *Src) const; - virtual unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, - Type *CondTy) const; - virtual unsigned getVectorInstrCost(unsigned Opcode, Type *Val, - unsigned Index) const; - virtual unsigned getMemoryOpCost(unsigned Opcode, Type *Src, - unsigned Alignment, - unsigned AddressSpace) const; - - virtual unsigned getAddressComputationCost(Type *PtrTy, bool IsComplex) const; - - virtual unsigned getReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) const; + unsigned getNumberOfRegisters(bool Vector) const override; + unsigned getRegisterBitWidth(bool Vector) const override; + unsigned getMaximumUnrollFactor() const override; + unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind, + OperandValueKind) const override; + unsigned getShuffleCost(ShuffleKind Kind, Type *Tp, + int Index, Type *SubTp) const override; + unsigned getCastInstrCost(unsigned Opcode, Type *Dst, + Type *Src) const override; + unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, + Type *CondTy) const override; + unsigned getVectorInstrCost(unsigned Opcode, Type *Val, + unsigned Index) const override; + unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) const override; + + unsigned getAddressComputationCost(Type *PtrTy, + bool IsComplex) const override; + + unsigned getReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwiseForm) const override; + + unsigned getIntImmCost(const APInt &Imm, Type *Ty) const override; + + unsigned getIntImmCost(unsigned Opcode, unsigned Idx, const APInt &Imm, + Type *Ty) const override; + unsigned getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, + Type *Ty) const override; /// @} }; @@ -134,6 +153,93 @@ X86TTI::PopcntSupportKind X86TTI::getPopcntSupport(unsigned TyWidth) const { return ST->hasPOPCNT() ? PSK_FastHardware : PSK_Software; } +void X86TTI::getUnrollingPreferences(Loop *L, UnrollingPreferences &UP) const { + if (!UsePartialUnrolling) + return; + // According to the Intel 64 and IA-32 Architectures Optimization Reference + // Manual, Intel Core models and later have a loop stream detector + // (and associated uop queue) that can benefit from partial unrolling. + // The relevant requirements are: + // - The loop must have no more than 4 (8 for Nehalem and later) branches + // taken, and none of them may be calls. + // - The loop can have no more than 18 (28 for Nehalem and later) uops. + + // According to the Software Optimization Guide for AMD Family 15h Processors, + // models 30h-4fh (Steamroller and later) have a loop predictor and loop + // buffer which can benefit from partial unrolling. + // The relevant requirements are: + // - The loop must have fewer than 16 branches + // - The loop must have less than 40 uops in all executed loop branches + + unsigned MaxBranches, MaxOps; + if (PartialUnrollingThreshold.getNumOccurrences() > 0) { + MaxBranches = PartialUnrollingMaxBranches; + MaxOps = PartialUnrollingThreshold; + } else if (ST->isAtom()) { + // On the Atom, the throughput for taken branches is 2 cycles. For small + // simple loops, expand by a small factor to hide the backedge cost. + MaxBranches = 2; + MaxOps = 10; + } else if (ST->hasFSGSBase() && ST->hasXOP() /* Steamroller and later */) { + MaxBranches = 16; + MaxOps = 40; + } else if (ST->hasFMA4() /* Any other recent AMD */) { + return; + } else if (ST->hasAVX() || ST->hasSSE42() /* Nehalem and later */) { + MaxBranches = 8; + MaxOps = 28; + } else if (ST->hasSSSE3() /* Intel Core */) { + MaxBranches = 4; + MaxOps = 18; + } else { + return; + } + + // Scan the loop: don't unroll loops with calls, and count the potential + // number of taken branches (this is somewhat conservative because we're + // counting all block transitions as potential branches while in reality some + // of these will become implicit via block placement). + unsigned MaxDepth = 0; + for (df_iterator<BasicBlock*> DI = df_begin(L->getHeader()), + DE = df_end(L->getHeader()); DI != DE;) { + if (!L->contains(*DI)) { + DI.skipChildren(); + continue; + } + + MaxDepth = std::max(MaxDepth, DI.getPathLength()); + if (MaxDepth > MaxBranches) + return; + + for (BasicBlock::iterator I = DI->begin(), IE = DI->end(); I != IE; ++I) + if (isa<CallInst>(I) || isa<InvokeInst>(I)) { + ImmutableCallSite CS(I); + if (const Function *F = CS.getCalledFunction()) { + if (!isLoweredToCall(F)) + continue; + } + + return; + } + + ++DI; + } + + // Enable runtime and partial unrolling up to the specified size. + UP.Partial = UP.Runtime = true; + UP.PartialThreshold = UP.PartialOptSizeThreshold = MaxOps; + + // Set the maximum count based on the loop depth. The maximum number of + // branches taken in a loop (including the backedge) is equal to the maximum + // loop depth (the DFS path length from the loop header to any block in the + // loop). When the loop is unrolled, this depth (except for the backedge + // itself) is multiplied by the unrolling factor. This new unrolled depth + // must be less than the target-specific maximum branch count (which limits + // the number of taken branches in the uop buffer). + if (MaxDepth > 1) + UP.MaxCount = (MaxBranches-1)/(MaxDepth-1); +} + unsigned X86TTI::getNumberOfRegisters(bool Vector) const { if (Vector && !ST->hasSSE1()) return 0; @@ -214,6 +320,13 @@ unsigned X86TTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, // Look for AVX2 lowering tricks. if (ST->hasAVX2()) { + if (ISD == ISD::SHL && LT.second == MVT::v16i16 && + (Op2Info == TargetTransformInfo::OK_UniformConstantValue || + Op2Info == TargetTransformInfo::OK_NonUniformConstantValue)) + // On AVX2, a packed v16i16 shift left by a constant build_vector + // is lowered into a vector multiply (vpmullw). + return LT.first; + int Idx = CostTableLookup(AVX2CostTable, ISD, LT.second); if (Idx != -1) return LT.first * AVX2CostTable[Idx].Cost; @@ -246,6 +359,20 @@ unsigned X86TTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, return LT.first * SSE2UniformConstCostTable[Idx].Cost; } + if (ISD == ISD::SHL && + Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) { + EVT VT = LT.second; + if ((VT == MVT::v8i16 && ST->hasSSE2()) || + (VT == MVT::v4i32 && ST->hasSSE41())) + // Vector shift left by non uniform constant can be lowered + // into vector multiply (pmullw/pmulld). + return LT.first; + if (VT == MVT::v4i32 && ST->hasSSE2()) + // A vector shift left by non uniform constant is converted + // into a vector multiply; the new multiply is eventually + // lowered into a sequence of shuffles and 2 x pmuludq. + ISD = ISD::MUL; + } static const CostTblEntry<MVT::SimpleValueType> SSE2CostTable[] = { // We don't correctly identify costs of casts because they are marked as @@ -260,6 +387,7 @@ unsigned X86TTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, { ISD::SHL, MVT::v8i16, 8*10 }, // Scalarized. { ISD::SHL, MVT::v4i32, 2*5 }, // We optimized this using mul. { ISD::SHL, MVT::v2i64, 2*10 }, // Scalarized. + { ISD::SHL, MVT::v4i64, 4*10 }, // Scalarized. { ISD::SRL, MVT::v16i8, 16*10 }, // Scalarized. { ISD::SRL, MVT::v8i16, 8*10 }, // Scalarized. @@ -297,6 +425,7 @@ unsigned X86TTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, // We don't have to scalarize unsupported ops. We can issue two half-sized // operations and we only need to extract the upper YMM half. // Two ops + 1 extract + 1 insert = 4. + { ISD::MUL, MVT::v16i16, 4 }, { ISD::MUL, MVT::v8i32, 4 }, { ISD::SUB, MVT::v8i32, 4 }, { ISD::ADD, MVT::v8i32, 4 }, @@ -312,7 +441,15 @@ unsigned X86TTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, // Look for AVX1 lowering tricks. if (ST->hasAVX() && !ST->hasAVX2()) { - int Idx = CostTableLookup(AVX1CostTable, ISD, LT.second); + EVT VT = LT.second; + + // v16i16 and v8i32 shifts by non-uniform constants are lowered into a + // sequence of extract + two vector multiply + insert. + if (ISD == ISD::SHL && (VT == MVT::v8i32 || VT == MVT::v16i16) && + Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) + ISD = ISD::MUL; + + int Idx = CostTableLookup(AVX1CostTable, ISD, VT); if (Idx != -1) return LT.first * AVX1CostTable[Idx].Cost; } @@ -332,7 +469,7 @@ unsigned X86TTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, // 2x pmuludq, 2x shuffle. if (ISD == ISD::MUL && LT.second == MVT::v4i32 && ST->hasSSE2() && !ST->hasSSE41()) - return 6; + return LT.first * 6; // Fallback to the default implementation. return TargetTransformInfo::getArithmeticInstrCost(Opcode, Ty, Op1Info, @@ -400,16 +537,58 @@ unsigned X86TTI::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) const { return TargetTransformInfo::getCastInstrCost(Opcode, Dst, Src); static const TypeConversionCostTblEntry<MVT::SimpleValueType> + AVX2ConversionTbl[] = { + { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 1 }, + { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 1 }, + { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, 3 }, + { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, 3 }, + { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3 }, + { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3 }, + { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 1 }, + { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 1 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, 3 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, 3 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i8, 3 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i8, 3 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 1 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 1 }, + + { ISD::TRUNCATE, MVT::v4i8, MVT::v4i64, 2 }, + { ISD::TRUNCATE, MVT::v4i16, MVT::v4i64, 2 }, + { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 2 }, + { ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 2 }, + { ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 2 }, + { ISD::TRUNCATE, MVT::v8i32, MVT::v8i64, 4 }, + }; + + static const TypeConversionCostTblEntry<MVT::SimpleValueType> AVXConversionTbl[] = { - { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 1 }, - { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 1 }, - { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 1 }, - { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 1 }, - { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 1 }, - { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 1 }, - { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 1 }, - { ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 1 }, - { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, 2 }, + { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 4 }, + { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 4 }, + { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, 7 }, + { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, 4 }, + { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 7 }, + { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 4 }, + { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 4 }, + { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 4 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, 6 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, 4 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i8, 6 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i8, 4 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 6 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, + { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 4 }, + { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 4 }, + + { ISD::TRUNCATE, MVT::v4i8, MVT::v4i64, 4 }, + { ISD::TRUNCATE, MVT::v4i16, MVT::v4i64, 4 }, + { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 4 }, + { ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 4 }, + { ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 5 }, + { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, 4 }, + { ISD::TRUNCATE, MVT::v8i32, MVT::v8i64, 9 }, { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i1, 8 }, { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 8 }, @@ -436,17 +615,32 @@ unsigned X86TTI::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) const { { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i8, 2 }, { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i16, 2 }, { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, 6 }, - - { ISD::FP_TO_SINT, MVT::v8i8, MVT::v8f32, 1 }, + // The generic code to compute the scalar overhead is currently broken. + // Workaround this limitation by estimating the scalarization overhead + // here. We have roughly 10 instructions per scalar element. + // Multiply that by the vector width. + // FIXME: remove that when PR19268 is fixed. + { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 2*10 }, + { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i64, 4*10 }, + + { ISD::FP_TO_SINT, MVT::v8i8, MVT::v8f32, 7 }, { ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 1 }, - { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, 6 }, - { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, 9 }, - { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, 8 }, - { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i8, 6 }, - { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 6 }, - { ISD::TRUNCATE, MVT::v8i32, MVT::v8i64, 3 }, + // This node is expanded into scalarized operations but BasicTTI is overly + // optimistic estimating its cost. It computes 3 per element (one + // vector-extract, one scalar conversion and one vector-insert). The + // problem is that the inserts form a read-modify-write chain so latency + // should be factored in too. Inflating the cost per element by 1. + { ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f32, 8*4 }, + { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f64, 4*4 }, }; + if (ST->hasAVX2()) { + int Idx = ConvertCostTableLookup(AVX2ConversionTbl, ISD, + DstTy.getSimpleVT(), SrcTy.getSimpleVT()); + if (Idx != -1) + return AVX2ConversionTbl[Idx].Cost; + } + if (ST->hasAVX()) { int Idx = ConvertCostTableLookup(AVXConversionTbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT()); @@ -555,7 +749,7 @@ unsigned X86TTI::getScalarizationOverhead(Type *Ty, bool Insert, unsigned X86TTI::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, unsigned AddressSpace) const { - // Handle non power of two vectors such as <3 x float> + // Handle non-power-of-two vectors such as <3 x float> if (VectorType *VTy = dyn_cast<VectorType>(Src)) { unsigned NumElem = VTy->getVectorNumElements(); @@ -570,7 +764,7 @@ unsigned X86TTI::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, // Cost = 128 bit store + unpack + 64 bit store. return 3; - // Assume that all other non power-of-two numbers are scalarized. + // Assume that all other non-power-of-two numbers are scalarized. if (!isPowerOf2_32(NumElem)) { unsigned Cost = TargetTransformInfo::getMemoryOpCost(Opcode, VTy->getScalarType(), @@ -692,3 +886,109 @@ unsigned X86TTI::getReductionCost(unsigned Opcode, Type *ValTy, return TargetTransformInfo::getReductionCost(Opcode, ValTy, IsPairwise); } +unsigned X86TTI::getIntImmCost(const APInt &Imm, Type *Ty) const { + assert(Ty->isIntegerTy()); + + unsigned BitSize = Ty->getPrimitiveSizeInBits(); + if (BitSize == 0) + return ~0U; + + if (Imm == 0) + return TCC_Free; + + if (Imm.getBitWidth() <= 64 && + (isInt<32>(Imm.getSExtValue()) || isUInt<32>(Imm.getZExtValue()))) + return TCC_Basic; + else + return 2 * TCC_Basic; +} + +unsigned X86TTI::getIntImmCost(unsigned Opcode, unsigned Idx, const APInt &Imm, + Type *Ty) const { + assert(Ty->isIntegerTy()); + + unsigned BitSize = Ty->getPrimitiveSizeInBits(); + if (BitSize == 0) + return ~0U; + + unsigned ImmIdx = ~0U; + switch (Opcode) { + default: return TCC_Free; + case Instruction::GetElementPtr: + // Always hoist the base address of a GetElementPtr. This prevents the + // creation of new constants for every base constant that gets constant + // folded with the offset. + if (Idx == 0) + return 2 * TCC_Basic; + return TCC_Free; + case Instruction::Store: + ImmIdx = 0; + break; + case Instruction::Add: + case Instruction::Sub: + case Instruction::Mul: + case Instruction::UDiv: + case Instruction::SDiv: + case Instruction::URem: + case Instruction::SRem: + case Instruction::Shl: + case Instruction::LShr: + case Instruction::AShr: + case Instruction::And: + case Instruction::Or: + case Instruction::Xor: + case Instruction::ICmp: + ImmIdx = 1; + break; + case Instruction::Trunc: + case Instruction::ZExt: + case Instruction::SExt: + case Instruction::IntToPtr: + case Instruction::PtrToInt: + case Instruction::BitCast: + case Instruction::PHI: + case Instruction::Call: + case Instruction::Select: + case Instruction::Ret: + case Instruction::Load: + break; + } + + if ((Idx == ImmIdx) && + Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue())) + return TCC_Free; + + return X86TTI::getIntImmCost(Imm, Ty); +} + +unsigned X86TTI::getIntImmCost(Intrinsic::ID IID, unsigned Idx, + const APInt &Imm, Type *Ty) const { + assert(Ty->isIntegerTy()); + + unsigned BitSize = Ty->getPrimitiveSizeInBits(); + if (BitSize == 0) + return ~0U; + + switch (IID) { + default: return TCC_Free; + case Intrinsic::sadd_with_overflow: + case Intrinsic::uadd_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::usub_with_overflow: + case Intrinsic::smul_with_overflow: + case Intrinsic::umul_with_overflow: + if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue())) + return TCC_Free; + break; + case Intrinsic::experimental_stackmap: + if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) + return TCC_Free; + break; + case Intrinsic::experimental_patchpoint_void: + case Intrinsic::experimental_patchpoint_i64: + if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) + return TCC_Free; + break; + } + return X86TTI::getIntImmCost(Imm, Ty); +} diff --git a/lib/Target/X86/X86VZeroUpper.cpp b/lib/Target/X86/X86VZeroUpper.cpp index 66ae9c2..d4341b9 100644 --- a/lib/Target/X86/X86VZeroUpper.cpp +++ b/lib/Target/X86/X86VZeroUpper.cpp @@ -17,6 +17,7 @@ #define DEBUG_TYPE "x86-vzeroupper" #include "X86.h" #include "X86InstrInfo.h" +#include "X86Subtarget.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -30,73 +31,59 @@ using namespace llvm; STATISTIC(NumVZU, "Number of vzeroupper instructions inserted"); namespace { - struct VZeroUpperInserter : public MachineFunctionPass { - static char ID; - VZeroUpperInserter() : MachineFunctionPass(ID) {} - - virtual bool runOnMachineFunction(MachineFunction &MF); - bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB); + class VZeroUpperInserter : public MachineFunctionPass { + public: - virtual const char *getPassName() const { return "X86 vzeroupper inserter";} + VZeroUpperInserter() : MachineFunctionPass(ID) {} + bool runOnMachineFunction(MachineFunction &MF) override; + const char *getPassName() const override {return "X86 vzeroupper inserter";} private: - const TargetInstrInfo *TII; // Machine instruction info. - - // Any YMM register live-in to this function? - bool FnHasLiveInYmm; - - // BBState - Contains the state of each MBB: unknown, clean, dirty - SmallVector<uint8_t, 8> BBState; - // BBSolved - Keep track of all MBB which had been already analyzed - // and there is no further processing required. - BitVector BBSolved; - - // Machine Basic Blocks are classified according this pass: - // - // ST_UNKNOWN - The MBB state is unknown, meaning from the entry state - // until the MBB exit there isn't a instruction using YMM to change - // the state to dirty, or one of the incoming predecessors is unknown - // and there's not a dirty predecessor between them. - // - // ST_CLEAN - No YMM usage in the end of the MBB. A MBB could have - // instructions using YMM and be marked ST_CLEAN, as long as the state - // is cleaned by a vzeroupper before any call. + void processBasicBlock(MachineBasicBlock &MBB); + void insertVZeroUpper(MachineBasicBlock::iterator I, + MachineBasicBlock &MBB); + void addDirtySuccessor(MachineBasicBlock &MBB); + + typedef enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY } BlockExitState; + static const char* getBlockExitStateName(BlockExitState ST); + + // Core algorithm state: + // BlockState - Each block is either: + // - PASS_THROUGH: There are neither YMM dirtying instructions nor + // vzeroupper instructions in this block. + // - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this + // block that will ensure that YMM is clean on exit. + // - EXITS_DIRTY: An instruction in the block dirties YMM and no + // subsequent vzeroupper in the block clears it. // - // ST_DIRTY - Any MBB ending with a YMM usage not cleaned up by a - // vzeroupper instruction. + // AddedToDirtySuccessors - This flag is raised when a block is added to the + // DirtySuccessors list to ensure that it's not + // added multiple times. // - // ST_INIT - Placeholder for an empty state set - // - enum { - ST_UNKNOWN = 0, - ST_CLEAN = 1, - ST_DIRTY = 2, - ST_INIT = 3 + // FirstUnguardedCall - Records the location of the first unguarded call in + // each basic block that may need to be guarded by a + // vzeroupper. We won't know whether it actually needs + // to be guarded until we discover a predecessor that + // is DIRTY_OUT. + struct BlockState { + BlockState() : ExitState(PASS_THROUGH), AddedToDirtySuccessors(false) {} + BlockExitState ExitState; + bool AddedToDirtySuccessors; + MachineBasicBlock::iterator FirstUnguardedCall; }; + typedef SmallVector<BlockState, 8> BlockStateMap; + typedef SmallVector<MachineBasicBlock*, 8> DirtySuccessorsWorkList; - // computeState - Given two states, compute the resulting state, in - // the following way - // - // 1) One dirty state yields another dirty state - // 2) All states must be clean for the result to be clean - // 3) If none above and one unknown, the result state is also unknown - // - static unsigned computeState(unsigned PrevState, unsigned CurState) { - if (PrevState == ST_INIT) - return CurState; - - if (PrevState == ST_DIRTY || CurState == ST_DIRTY) - return ST_DIRTY; - - if (PrevState == ST_CLEAN && CurState == ST_CLEAN) - return ST_CLEAN; - - return ST_UNKNOWN; - } + BlockStateMap BlockStates; + DirtySuccessorsWorkList DirtySuccessors; + bool EverMadeChange; + const TargetInstrInfo *TII; + static char ID; }; + char VZeroUpperInserter::ID = 0; } @@ -104,29 +91,30 @@ FunctionPass *llvm::createX86IssueVZeroUpperPass() { return new VZeroUpperInserter(); } -static bool isYmmReg(unsigned Reg) { - return (Reg >= X86::YMM0 && Reg <= X86::YMM31); +const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) { + switch (ST) { + case PASS_THROUGH: return "Pass-through"; + case EXITS_DIRTY: return "Exits-dirty"; + case EXITS_CLEAN: return "Exits-clean"; + } + llvm_unreachable("Invalid block exit state."); } -static bool isZmmReg(unsigned Reg) { - return (Reg >= X86::ZMM0 && Reg <= X86::ZMM31); +static bool isYmmReg(unsigned Reg) { + return (Reg >= X86::YMM0 && Reg <= X86::YMM15); } static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) { for (MachineRegisterInfo::livein_iterator I = MRI.livein_begin(), E = MRI.livein_end(); I != E; ++I) - if (isYmmReg(I->first) || isZmmReg(I->first)) + if (isYmmReg(I->first)) return true; return false; } static bool clobbersAllYmmRegs(const MachineOperand &MO) { - for (unsigned reg = X86::YMM0; reg <= X86::YMM31; ++reg) { - if (!MO.clobbersPhysReg(reg)) - return false; - } - for (unsigned reg = X86::ZMM0; reg <= X86::ZMM31; ++reg) { + for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) { if (!MO.clobbersPhysReg(reg)) return false; } @@ -150,16 +138,13 @@ static bool hasYmmReg(MachineInstr *MI) { /// clobbersAnyYmmReg() - Check if any YMM register will be clobbered by this /// instruction. -static bool clobbersAnyYmmReg(MachineInstr *MI) { +static bool callClobbersAnyYmmReg(MachineInstr *MI) { + assert(MI->isCall() && "Can only be called on call instructions."); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI->getOperand(i); if (!MO.isRegMask()) continue; - for (unsigned reg = X86::YMM0; reg <= X86::YMM31; ++reg) { - if (MO.clobbersPhysReg(reg)) - return true; - } - for (unsigned reg = X86::ZMM0; reg <= X86::ZMM31; ++reg) { + for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) { if (MO.clobbersPhysReg(reg)) return true; } @@ -167,102 +152,44 @@ static bool clobbersAnyYmmReg(MachineInstr *MI) { return false; } -/// runOnMachineFunction - Loop over all of the basic blocks, inserting -/// vzero upper instructions before function calls. -bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) { - TII = MF.getTarget().getInstrInfo(); - MachineRegisterInfo &MRI = MF.getRegInfo(); - bool EverMadeChange = false; - - // Fast check: if the function doesn't use any ymm registers, we don't need - // to insert any VZEROUPPER instructions. This is constant-time, so it is - // cheap in the common case of no ymm use. - bool YMMUsed = false; - const TargetRegisterClass *RC = &X86::VR256RegClass; - for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); - i != e; i++) { - if (!MRI.reg_nodbg_empty(*i)) { - YMMUsed = true; - break; - } - } - if (!YMMUsed) - return EverMadeChange; - - // Pre-compute the existence of any live-in YMM registers to this function - FnHasLiveInYmm = checkFnHasLiveInYmm(MRI); - - assert(BBState.empty()); - BBState.resize(MF.getNumBlockIDs(), 0); - BBSolved.resize(MF.getNumBlockIDs(), 0); - - // Each BB state depends on all predecessors, loop over until everything - // converges. (Once we converge, we can implicitly mark everything that is - // still ST_UNKNOWN as ST_CLEAN.) - while (1) { - bool MadeChange = false; - - // Process all basic blocks. - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) - MadeChange |= processBasicBlock(MF, *I); +// Insert a vzeroupper instruction before I. +void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I, + MachineBasicBlock &MBB) { + DebugLoc dl = I->getDebugLoc(); + BuildMI(MBB, I, dl, TII->get(X86::VZEROUPPER)); + ++NumVZU; + EverMadeChange = true; +} - // If this iteration over the code changed anything, keep iterating. - if (!MadeChange) break; - EverMadeChange = true; +// Add MBB to the DirtySuccessors list if it hasn't already been added. +void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) { + if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) { + DirtySuccessors.push_back(&MBB); + BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true; } - - BBState.clear(); - BBSolved.clear(); - return EverMadeChange; } /// processBasicBlock - Loop over all of the instructions in the basic block, /// inserting vzero upper instructions before function calls. -bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF, - MachineBasicBlock &BB) { - bool Changed = false; - unsigned BBNum = BB.getNumber(); - - // Don't process already solved BBs - if (BBSolved[BBNum]) - return false; // No changes - - // Check the state of all predecessors - unsigned EntryState = ST_INIT; - for (MachineBasicBlock::const_pred_iterator PI = BB.pred_begin(), - PE = BB.pred_end(); PI != PE; ++PI) { - EntryState = computeState(EntryState, BBState[(*PI)->getNumber()]); - if (EntryState == ST_DIRTY) - break; - } +void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) { + // Start by assuming that the block PASS_THROUGH, which implies no unguarded + // calls. + BlockExitState CurState = PASS_THROUGH; + BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end(); - // The entry MBB for the function may set the initial state to dirty if - // the function receives any YMM incoming arguments - if (&BB == MF.begin()) { - EntryState = ST_CLEAN; - if (FnHasLiveInYmm) - EntryState = ST_DIRTY; - } - - // The current state is initialized according to the predecessors - unsigned CurState = EntryState; - bool BBHasCall = false; - - for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) { - DebugLoc dl = I->getDebugLoc(); + for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) { MachineInstr *MI = I; - bool isControlFlow = MI->isCall() || MI->isReturn(); // Shortcut: don't need to check regular instructions in dirty state. - if (!isControlFlow && CurState == ST_DIRTY) + if (!isControlFlow && CurState == EXITS_DIRTY) continue; if (hasYmmReg(MI)) { // We found a ymm-using instruction; this could be an AVX instruction, // or it could be control flow. - CurState = ST_DIRTY; + CurState = EXITS_DIRTY; continue; } @@ -276,11 +203,9 @@ bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF, // standard calling convention is not used (RegMask is not used to mark // register clobbered and register usage (def/imp-def/use) is well-dfined // and explicitly specified. - if (MI->isCall() && !clobbersAnyYmmReg(MI)) + if (MI->isCall() && !callClobbersAnyYmmReg(MI)) continue; - BBHasCall = true; - // The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX // registers. This instruction has zero latency. In addition, the processor // changes back to Clean state, after which execution of Intel SSE @@ -289,38 +214,101 @@ bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF, // execute SSE code. // FIXME: In some cases, we may want to move the VZEROUPPER into a // predecessor block. - if (CurState == ST_DIRTY) { - // Only insert the VZEROUPPER in case the entry state isn't unknown. - // When unknown, only compute the information within the block to have - // it available in the exit if possible, but don't change the block. - if (EntryState != ST_UNKNOWN) { - BuildMI(BB, I, dl, TII->get(X86::VZEROUPPER)); - ++NumVZU; - } - + if (CurState == EXITS_DIRTY) { // After the inserted VZEROUPPER the state becomes clean again, but // other YMM may appear before other subsequent calls or even before // the end of the BB. - CurState = ST_CLEAN; + insertVZeroUpper(I, MBB); + CurState = EXITS_CLEAN; + } else if (CurState == PASS_THROUGH) { + // If this block is currently in pass-through state and we encounter a + // call then whether we need a vzeroupper or not depends on whether this + // block has successors that exit dirty. Record the location of the call, + // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet. + // It will be inserted later if necessary. + BlockStates[MBB.getNumber()].FirstUnguardedCall = I; + CurState = EXITS_CLEAN; } } - DEBUG(dbgs() << "MBB #" << BBNum - << ", current state: " << CurState << '\n'); + DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: " + << getBlockExitStateName(CurState) << '\n'); + + if (CurState == EXITS_DIRTY) + for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(), + SE = MBB.succ_end(); + SI != SE; ++SI) + addDirtySuccessor(**SI); + + BlockStates[MBB.getNumber()].ExitState = CurState; +} + +/// runOnMachineFunction - Loop over all of the basic blocks, inserting +/// vzero upper instructions before function calls. +bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) { + if (MF.getTarget().getSubtarget<X86Subtarget>().hasAVX512()) + return false; + TII = MF.getTarget().getInstrInfo(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + EverMadeChange = false; - // A BB can only be considered solved when we both have done all the - // necessary transformations, and have computed the exit state. This happens - // in two cases: - // 1) We know the entry state: this immediately implies the exit state and - // all the necessary transformations. - // 2) There are no calls, and and a non-call instruction marks this block: - // no transformations are necessary, and we know the exit state. - if (EntryState != ST_UNKNOWN || (!BBHasCall && CurState != ST_UNKNOWN)) - BBSolved[BBNum] = true; + // Fast check: if the function doesn't use any ymm registers, we don't need + // to insert any VZEROUPPER instructions. This is constant-time, so it is + // cheap in the common case of no ymm use. + bool YMMUsed = false; + const TargetRegisterClass *RC = &X86::VR256RegClass; + for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); + i != e; i++) { + if (!MRI.reg_nodbg_empty(*i)) { + YMMUsed = true; + break; + } + } + if (!YMMUsed) { + return false; + } - if (CurState != BBState[BBNum]) - Changed = true; + assert(BlockStates.empty() && DirtySuccessors.empty() && + "X86VZeroUpper state should be clear"); + BlockStates.resize(MF.getNumBlockIDs()); + + // Process all blocks. This will compute block exit states, record the first + // unguarded call in each block, and add successors of dirty blocks to the + // DirtySuccessors list. + for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) + processBasicBlock(*I); + + // If any YMM regs are live in to this function, add the entry block to the + // DirtySuccessors list + if (checkFnHasLiveInYmm(MRI)) + addDirtySuccessor(MF.front()); + + // Re-visit all blocks that are successors of EXITS_DIRTY bsocks. Add + // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY + // through PASS_THROUGH blocks. + while (!DirtySuccessors.empty()) { + MachineBasicBlock &MBB = *DirtySuccessors.back(); + DirtySuccessors.pop_back(); + BlockState &BBState = BlockStates[MBB.getNumber()]; + + // MBB is a successor of a dirty block, so its first call needs to be + // guarded. + if (BBState.FirstUnguardedCall != MBB.end()) + insertVZeroUpper(BBState.FirstUnguardedCall, MBB); + + // If this successor was a pass-through block then it is now dirty, and its + // successors need to be added to the worklist (if they haven't been + // already). + if (BBState.ExitState == PASS_THROUGH) { + DEBUG(dbgs() << "MBB #" << MBB.getNumber() + << " was Pass-through, is now Dirty-out.\n"); + for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(), + SE = MBB.succ_end(); + SI != SE; ++SI) + addDirtySuccessor(**SI); + } + } - BBState[BBNum] = CurState; - return Changed; + BlockStates.clear(); + return EverMadeChange; } |