x86-64: REX prefix is invalid with VEX etc

Just like for the data size prefix (see commit 7a8655d2bbdc ["x86: don't
abort() upon DATA16 prefix on (E)VEX encoded insn"]), any form of REX
prefix is invalid with VEX/XOP/EVEX.

diff --git a/gas/ChangeLog b/gas/ChangeLog
index c2fa02d03caf37586c66809e372f6491ef982ce3..aad2676110dfd22f336356f45a083f1c97c32bb6 100644 (file)
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,16 @@
+2020-06-25  Jan Beulich  <jbeulich@suse.com>
+
+       * config/tc-i386.c (md_assemble): Also reject explicit REX
+       prefixes with VEX and alike encoded insns. Zap consumed bits
+       from i.rex.
+       (output_insn): Don't ignore REX prefix for VEX and alike
+       encodings; abort() instead if encountered.
+       * testsuite/gas/i386/x86-64-pseudos.s: Move REX-with-VEX cases
+       ...
+       * testsuite/gas/i386/x86-64-pseudos-bad.s: ... here.
+       * testsuite/gas/i386/x86-64-pseudos.d,
+       testsuite/gas/i386/x86-64-pseudos-bad.l: Adjust expectations.
+
 2020-06-25  Jan Beulich  <jbeulich@suse.com>
 
        * config/tc-i386.c (process_operands): Translate explicit REX

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index c84192e03100431daa0ad3e3e9a8b9fe7bb8ab39..f1a7a3487699eaac61a06d391273199ad2c0314a 100644 (file)
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -4893,10 +4893,20 @@ md_assemble (char *line)
          return;
        }
 
+      /* Check for explicit REX prefix.  */
+      if (i.prefix[REX_PREFIX] || i.rex_encoding)
+       {
+         as_bad (_("REX prefix invalid with `%s'"), i.tm.name);
+         return;
+       }
+
       if (i.tm.opcode_modifier.vex)
        build_vex_prefix (t);
       else
        build_evex_prefix ();
+
+      /* The individual REX.RXBW bits got consumed.  */
+      i.rex &= REX_OPCODE;
     }
 
   /* Handle conversion of 'int $3' --> special int3 insn.  XOP or FMA4
@@ -9275,9 +9285,6 @@ output_insn (void)
            if (*q)
              switch (j)
                {
-               case REX_PREFIX:
-                 /* REX byte is encoded in VEX prefix.  */
-                 break;
                case SEG_PREFIX:
                case ADDR_PREFIX:
                  FRAG_APPEND_1_CHAR (*q);

diff --git a/gas/testsuite/gas/i386/x86-64-pseudos-bad.l b/gas/testsuite/gas/i386/x86-64-pseudos-bad.l
index 9f8f32de2aeb6855218db0dcbf3d1ea7896df729..3f9f67fcf4bd2b812ec7d754c7eac08230104b7e 100644 (file)
--- a/gas/testsuite/gas/i386/x86-64-pseudos-bad.l
+++ b/gas/testsuite/gas/i386/x86-64-pseudos-bad.l
@@ -1,3 +1,6 @@
 .*: Assembler messages:
 .*:3: Error: .*`vmovaps'.*
 .*:4: Error: .*`vmovaps'.*
+.*:5: Error: .*`vmovaps'.*
+.*:6: Error: .*`vmovaps'.*
+.*:7: Error: .*`rorx'.*

diff --git a/gas/testsuite/gas/i386/x86-64-pseudos-bad.s b/gas/testsuite/gas/i386/x86-64-pseudos-bad.s
index 805fd2dac7859096b2d79970d2b3d1dadd9217d8..3b923593a6aa25ae3ef26265655422ab66aa623f 100644 (file)
--- a/gas/testsuite/gas/i386/x86-64-pseudos-bad.s
+++ b/gas/testsuite/gas/i386/x86-64-pseudos-bad.s
@@ -2,3 +2,6 @@
 pseudos:
        {vex} vmovaps %xmm0, %xmm30
        {vex3} vmovaps %xmm30, %xmm0
+       {rex} vmovaps %xmm7,%xmm2
+       {rex} vmovaps %xmm17,%xmm2
+       {rex} rorx $7,%eax,%ebx

diff --git a/gas/testsuite/gas/i386/x86-64-pseudos.d b/gas/testsuite/gas/i386/x86-64-pseudos.d
index e0074af843bd1f3d4ee91cc37d10935d286756c5..529578bdd477900c8d792f7bc669f81c68aec3f3 100644 (file)
--- a/gas/testsuite/gas/i386/x86-64-pseudos.d
+++ b/gas/testsuite/gas/i386/x86-64-pseudos.d
@@ -310,9 +310,6 @@ Disassembly of section .text:
  +[a-f0-9]+:   41 0f 28 10             movaps \(%r8\),%xmm2
  +[a-f0-9]+:   40 0f 38 01 01          rex phaddw \(%rcx\),%mm0
  +[a-f0-9]+:   41 0f 38 01 00          phaddw \(%r8\),%mm0
- +[a-f0-9]+:   c5 f8 28 d7             vmovaps %xmm7,%xmm2
- +[a-f0-9]+:   62 b1 7c 08 28 d1       vmovaps %xmm17,%xmm2
- +[a-f0-9]+:   c4 e3 7b f0 d8 07       rorx   \$0x7,%eax,%ebx
  +[a-f0-9]+:   c4 e1 78 28 d7          vmovaps %xmm7,%xmm2
  +[a-f0-9]+:   c4 e1 78 28 d7          vmovaps %xmm7,%xmm2
  +[a-f0-9]+:   c4 e1 78 29 fa          vmovaps %xmm7,%xmm2
@@ -351,7 +348,4 @@ Disassembly of section .text:
  +[a-f0-9]+:   41 0f 28 10             movaps \(%r8\),%xmm2
  +[a-f0-9]+:   40 0f 38 01 01          rex phaddw \(%rcx\),%mm0
  +[a-f0-9]+:   41 0f 38 01 00          phaddw \(%r8\),%mm0
- +[a-f0-9]+:   c5 f8 28 d7             vmovaps %xmm7,%xmm2
- +[a-f0-9]+:   62 b1 7c 08 28 d1       vmovaps %xmm17,%xmm2
- +[a-f0-9]+:   c4 e3 7b f0 d8 07       rorx   \$0x7,%eax,%ebx
 #pass

diff --git a/gas/testsuite/gas/i386/x86-64-pseudos.s b/gas/testsuite/gas/i386/x86-64-pseudos.s
index 14b6c92b864c20b47f613e036dce60a2069cfbff..3bda5737ea1b363536ce940f5933db0c87607d58 100644 (file)
--- a/gas/testsuite/gas/i386/x86-64-pseudos.s
+++ b/gas/testsuite/gas/i386/x86-64-pseudos.s
@@ -314,9 +314,6 @@ _start:
        {rex} movaps (%r8),%xmm2
        {rex} phaddw (%rcx),%mm0
        {rex} phaddw (%r8),%mm0
-       {rex} vmovaps %xmm7,%xmm2
-       {rex} vmovaps %xmm17,%xmm2
-       {rex} rorx $7,%eax,%ebx
 
        .intel_syntax noprefix
        {vex3} vmovaps xmm2,xmm7
@@ -357,6 +354,3 @@ _start:
        {rex} movaps xmm2,XMMWORD PTR [r8]
        {rex} phaddw mm0,QWORD PTR [rcx]
        {rex} phaddw mm0,QWORD PTR [r8]
-       {rex} vmovaps xmm2,xmm7
-       {rex} vmovaps xmm2,xmm17
-       {rex} rorx ebx,eax,0x7