Fix VEX decoding in i386 mode. Fixes PR11008.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@140515 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 291 insertions, 2 deletions

diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
index 9e53345..ed2cb3f 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
@@ -391,7 +391,7 @@ static int readPrefixes(struct InternalInstruction* insn) {
       return -1;
     }
     
-    if (insn->mode == MODE_64BIT || byte1 & 0x8) {
+    if (insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) {
       insn->vexSize = 3;
       insn->necessaryPrefixLocation = insn->readerCursor - 1;
     }
@@ -433,7 +433,7 @@ static int readPrefixes(struct InternalInstruction* insn) {
       return -1;
     }
       
-    if (insn->mode == MODE_64BIT || byte1 & 0x8) {
+    if (insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) {
       insn->vexSize = 2;
     }
     else {
diff --git a/test/MC/Disassembler/X86/x86-32.txt b/test/MC/Disassembler/X86/x86-32.txt
index 38221f8..5c2a95d 100644
--- a/test/MC/Disassembler/X86/x86-32.txt
+++ b/test/MC/Disassembler/X86/x86-32.txt
@@ -30,3 +30,292 @@
 
 # CHECK: crc32w %ax, %eax
 0x66 0xf2 0x0f 0x38 0xf1 0xc0
+
+
+# CHECK: int	$33
+0xCD 0x21 
+
+# CHECK: int	$33
+0xCD 0x21
+
+
+# CHECK: addb	%al, (%eax)
+0 0
+
+# CHECK: calll	-1234
+0xe8 0x2e 0xfb 0xff 0xff
+
+# CHECK: lfence
+0x0f 0xae 0xe8
+
+# CHECK: mfence
+0x0f 0xae 0xf0
+
+# CHECK: monitor
+0x0f 0x01 0xc8
+
+# CHECK: mwait
+0x0f 0x01 0xc9
+
+# CHECK: vmcall
+0x0f 0x01 0xc1
+
+# CHECK: vmlaunch
+0x0f 0x01 0xc2
+
+# CHECK: vmresume
+0x0f 0x01 0xc3
+
+# CHECK: vmxoff
+0x0f 0x01 0xc4
+
+# CHECK: swapgs
+0x0f 0x01 0xf8
+
+# CHECK: rdtscp
+0x0f 0x01 0xf9
+
+# CHECK: vmxon
+0xf3 0x0f 0xc7 0x30
+
+# CHECK: vmptrld
+0x0f 0xc7 0x30
+
+# CHECK: vmptrst
+0x0f 0xc7 0x38
+
+# CHECK: movl $0, -4(%ebp)
+0xc7 0x45 0xfc 0x00 0x00 0x00 0x00
+
+# CHECK: movl	%cr0, %ecx
+0x0f 0x20 0xc1
+
+# CHECK: leal	4(%esp), %ecx
+0x8d 0x4c 0x24 0x04 
+
+# CHECK: enter	$1, $2
+0xc8 0x01 0x00 0x02
+
+# CHECK: movw	$47416, -66(%ebp)
+0x66 0xc7 0x45 0xbe 0x38 0xb9
+
+# CHECK: vaddpd	%ymm13, %ymm1, %ymm0
+0xc4 0xc1 0x75 0x58 0xc5
+
+# CHECK: vaddps	%ymm3, %ymm1, %ymm0
+0xc5 0xf4 0x58 0xc3
+
+# CHECK: vandpd	%ymm13, %ymm1, %ymm0
+0xc4 0xc1 0x75 0x54 0xc5
+
+# CHECK: vandps	%ymm3, %ymm1, %ymm0
+0xc5 0xf4 0x54 0xc3
+
+# CHECK: vzeroall
+0xc5 0xfc 0x77
+
+# CHECK: vcvtps2pd %xmm0, %ymm0
+0xc5 0xfc 0x5a 0xc0
+
+# CHECK: vandps (%edx), %xmm1, %xmm7
+0xc5 0xf0 0x54 0x3a
+
+# CHECK: vcvtss2sil %xmm0, %eax
+0xc5 0xfa 0x2d 0xc0
+
+# CHECK: vcvtsd2si %xmm0, %eax
+0xc5 0xfb 0x2d 0xc0
+
+# CHECK: vcvtsd2si %xmm0, %rax
+0xc4 0xe1 0xfb 0x2d 0xc0
+
+# CHECK: vmaskmovpd %xmm0, %xmm1, (%eax)
+0xc4 0xe2 0x71 0x2f 0x00
+
+# CHECK: vmovapd %xmm0, %xmm2
+0xc5 0xf9 0x28 0xd0
+
+# Check these special case instructions that the immediate is not sign-extend.
+# CHECK: blendps $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x0c 0xca 0x81
+
+# CHECK: blendpd $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x0d 0xca 0x81
+
+# CHECK: pblendw $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x0e 0xca 0x81
+
+# CHECK: mpsadbw $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x42 0xca 0x81
+
+# CHECK: dpps $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x40 0xca 0x81
+
+# CHECK: dppd $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x41 0xca 0x81
+
+# CHECK: insertps $129, %xmm2, %xmm1
+0x66 0x0f 0x3a 0x21 0xca 0x81
+
+# CHECK: vblendps $129, %ymm2, %ymm5, %ymm1
+0xc4 0xe3 0x55 0x0c 0xca 0x81
+
+# CHECK: vblendps $129, (%eax), %ymm5, %ymm1
+0xc4 0xe3 0x55 0x0c 0x08 0x81
+
+# CHECK: vblendpd $129, %ymm2, %ymm5, %ymm1
+0xc4 0xe3 0x55 0x0d 0xca 0x81
+
+# CHECK: vblendpd $129, (%eax), %ymm5, %ymm1
+0xc4 0xe3 0x55 0x0d 0x08 0x81
+
+# CHECK: vpblendw $129, %xmm2, %xmm5, %xmm1
+0xc4 0xe3 0x51 0x0e 0xca 0x81
+
+# CHECK: vmpsadbw $129, %xmm2, %xmm5, %xmm1
+0xc4 0xe3 0x51 0x42 0xca 0x81
+
+# CHECK: vdpps $129, %ymm2, %ymm5, %ymm1
+0xc4 0xe3 0x55 0x40 0xca 0x81
+
+# CHECK: vdpps $129, (%eax), %ymm5, %ymm1
+0xc4 0xe3 0x55 0x40 0x08 0x81
+
+# CHECK: vdppd $129, %xmm2, %xmm5, %xmm1
+0xc4 0xe3 0x51 0x41 0xca 0x81
+
+# CHECK: vinsertps $129, %xmm3, %xmm2, %xmm1
+0xc4 0xe3 0x69 0x21 0xcb 0x81
+
+# CHECK: pause
+0xf3 0x90
+
+# CHECK: addl %eax, %edi
+0x01 0xc7
+
+# CHECK: addl %edi, %eax
+0x03 0xc7
+
+# CHECK: movl %eax, %edi
+0x89 0xc7
+
+# CHECK: movl %edi, %eax
+0x8b 0xc7
+
+# CHECK: movups %xmm1, %xmm0
+0x0f 0x10 0xc1
+
+# CHECK: movups %xmm0, %xmm1
+0x0f 0x11 0xc1
+
+# CHECK: movaps %xmm1, %xmm0
+0x0f 0x28 0xc1
+
+# CHECK: movaps %xmm0, %xmm1
+0x0f 0x29 0xc1
+
+# CHECK: movupd %xmm1, %xmm0
+0x66 0x0f 0x10 0xc1
+
+# CHECK: movupd %xmm0, %xmm1
+0x66 0x0f 0x11 0xc1
+
+# CHECK: movapd %xmm1, %xmm0
+0x66 0x0f 0x28 0xc1
+
+# CHECK: movapd %xmm0, %xmm1
+0x66 0x0f 0x29 0xc1
+
+# CHECK: vmovups %xmm1, %xmm0
+0xc5 0xf8 0x10 0xc1
+
+# CHECK: vmovups %xmm0, %xmm1
+0xc5 0xf8 0x11 0xc1
+
+# CHECK: vmovaps %xmm1, %xmm0
+0xc5 0xf8 0x28 0xc1
+
+# CHECK: vmovaps %xmm0, %xmm1
+0xc5 0xf8 0x29 0xc1
+
+# CHECK: vmovupd %xmm1, %xmm0
+0xc5 0xf9 0x10 0xc1
+
+# CHECK: vmovupd %xmm0, %xmm1
+0xc5 0xf9 0x11 0xc1
+
+# CHECK: vmovapd %xmm1, %xmm0
+0xc5 0xf9 0x28 0xc1
+
+# CHECK: vmovapd %xmm0, %xmm1
+0xc5 0xf9 0x29 0xc1
+
+# CHECK: vmovups %ymm1, %ymm0
+0xc5 0xfc 0x10 0xc1
+
+# CHECK: vmovups %ymm0, %ymm1
+0xc5 0xfc 0x11 0xc1
+
+# CHECK: vmovaps %ymm1, %ymm0
+0xc5 0xfc 0x28 0xc1
+
+# CHECK: vmovaps %ymm0, %ymm1
+0xc5 0xfc 0x29 0xc1
+
+# CHECK: movdqa %xmm1, %xmm0
+0x66 0x0f 0x6f 0xc1
+
+# CHECK: movdqa %xmm0, %xmm1
+0x66 0x0f 0x7f 0xc1
+
+# CHECK: movdqu %xmm1, %xmm0
+0xf3 0x0f 0x6f 0xc1
+
+# CHECK: movdqu %xmm0, %xmm1
+0xf3 0x0f 0x7f 0xc1
+
+# CHECK: vmovdqa %xmm1, %xmm0
+0xc5 0xf9 0x6f 0xc1
+
+# CHECK: vmovdqa %xmm0, %xmm1
+0xc5 0xf9 0x7f 0xc1
+
+# CHECK: vmovdqa %ymm1, %ymm0
+0xc5 0xfd 0x6f 0xc1
+
+# CHECK: vmovdqa %ymm0, %ymm1
+0xc5 0xfd 0x7f 0xc1
+
+# CHECK: vmovdqu %xmm1, %xmm0
+0xc5 0xfa 0x6f 0xc1
+
+# CHECK: vmovdqu %xmm0, %xmm1
+0xc5 0xfa 0x7f 0xc1
+
+# CHECK: vmovdqu %ymm1, %ymm0
+0xc5 0xfe 0x6f 0xc1
+
+# CHECK: vmovdqu %ymm0, %ymm1
+0xc5 0xfe 0x7f 0xc1
+
+# CHECK: vblendvps %xmm4, %xmm1, %xmm2, %xmm3
+0xc4 0xe3 0x69 0x4a 0xd9 0x41
+
+# CHECK: vroundpd $0, %xmm0, %xmm0
+0xc4 0xe3 0x79 0x09 0xc0 0x00
+
+# CHECK: vroundps $0, %xmm0, %xmm0
+0xc4 0xe3 0x79 0x08 0xc0 0x00
+
+# CHECK: vroundpd $0, %ymm0, %ymm0
+0xc4 0xe3 0x7d 0x09 0xc0 0x00
+
+# CHECK: vroundps $0, %ymm0, %ymm0
+0xc4 0xe3 0x7d 0x08 0xc0 0x00
+
+# CHECK: vroundss $0, %xmm0, %xmm0, %xmm0
+0xc4 0xe3 0x79 0x0a 0xc0 0x00
+
+# CHECK: vroundsd $0, %xmm0, %xmm0, %xmm0
+0xc4 0xe3 0x79 0x0b 0xc0 0x00
+