From: Jan Beulich <jbeulich@suse.com>
Date: Thu, 9 Jan 2020 10:40:04 +0000 (+0100)
Subject: x86: refine when to trigger optimizations
X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=72aea32839b5f4a2c8fa67e60451169b841941c0;p=binutils-gdb.git

x86: refine when to trigger optimizations

Checking just the base opcode without also checking this isn't a VEX
encoding, and without there being other insn properties avoiding a match
once respective VEX/XOP/EXEX-encoded insns would appear, is at least
dangerous. Add respective checks. At the same time there's no real need
to check the extension opcode to be None for the 0xA8 form - there's
nothing it can be confused with, and non-VEX-and-alike forms also can't
appear.
---

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 26a0075e079..660b9a205e9 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,9 @@
+2020-01-09  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (optimize_encoding): Add
+	is_any_vex_encoding() invocations. Drop respective
+	i.tm.extension_opcode == None checks.
+
 2020-01-09  Jan Beulich  <jbeulich@suse.com>
 
 	* config/tc-i386.c (md_assemble): Check RegRex is clear during
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 645fd38a12c..2c087e42e0d 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -3990,13 +3990,13 @@ optimize_encoding (void)
   unsigned int j;
 
   if (optimize_for_space
+      && !is_any_vex_encoding (&i.tm)
       && i.reg_operands == 1
       && i.imm_operands == 1
       && !i.types[1].bitfield.byte
       && i.op[0].imms->X_op == O_constant
       && fits_in_imm7 (i.op[0].imms->X_add_number)
-      && ((i.tm.base_opcode == 0xa8
-	   && i.tm.extension_opcode == None)
+      && (i.tm.base_opcode == 0xa8
 	  || (i.tm.base_opcode == 0xf6
 	      && i.tm.extension_opcode == 0x0)))
     {
@@ -4022,6 +4022,7 @@ optimize_encoding (void)
 	}
     }
   else if (flag_code == CODE_64BIT
+	   && !is_any_vex_encoding (&i.tm)
 	   && ((i.types[1].bitfield.qword
 		&& i.reg_operands == 1
 		&& i.imm_operands == 1
@@ -4030,9 +4031,8 @@ optimize_encoding (void)
 		     && i.tm.extension_opcode == None
 		     && fits_in_unsigned_long (i.op[0].imms->X_add_number))
 		    || (fits_in_imm31 (i.op[0].imms->X_add_number)
-			&& (((i.tm.base_opcode == 0x24
-			      || i.tm.base_opcode == 0xa8)
-			     && i.tm.extension_opcode == None)
+			&& ((i.tm.base_opcode == 0x24
+			     || i.tm.base_opcode == 0xa8)
 			    || (i.tm.base_opcode == 0x80
 				&& i.tm.extension_opcode == 0x4)
 			    || ((i.tm.base_opcode == 0xf6
@@ -4044,13 +4044,11 @@ optimize_encoding (void)
 	       || (i.types[0].bitfield.qword
 		   && ((i.reg_operands == 2
 			&& i.op[0].regs == i.op[1].regs
-			&& ((i.tm.base_opcode == 0x30
-			     || i.tm.base_opcode == 0x28)
-			    && i.tm.extension_opcode == None))
+			&& (i.tm.base_opcode == 0x30
+			    || i.tm.base_opcode == 0x28))
 		       || (i.reg_operands == 1
 			   && i.operands == 1
-			   && i.tm.base_opcode == 0x30
-			   && i.tm.extension_opcode == None)))))
+			   && i.tm.base_opcode == 0x30)))))
     {
       /* Optimize: -O:
 	   andq $imm31, %r64   -> andl $imm31, %r32
@@ -4091,6 +4089,7 @@ optimize_encoding (void)
     }
   else if (optimize > 1
 	   && !optimize_for_space
+	   && !is_any_vex_encoding (&i.tm)
 	   && i.reg_operands == 2
 	   && i.op[0].regs == i.op[1].regs
 	   && ((i.tm.base_opcode & ~(Opcode_D | 1)) == 0x8