ubsan: alpha-vms: shift exponent 536874240 is too large

C_OPR_ASH is supposed to be an arithmetic shift.  By the look of it,
this operator implemented logical shifts since the original binutils
support was added.  This patch corrects that and avoids some nonsense
ubsan complaints.  I chose to implement infinite precision shifts
rather than masking shift counts to the word size as the spec I had is
silent on what is supposed to happen with overlarge shift counts.

	* vms-alpha.c (_bfd_vms_slurp_etir <ETIR__C_OPR_ASH>): Implement
	shifts without undefined behaviour.

diff --git a/bfd/ChangeLog b/bfd/ChangeLog
index 5d1075e743d34cc462bebd7e889806e8492fce06..9c202b27fc0635008510c6160158794bb8106f0e 100644 (file)
--- a/bfd/ChangeLog
+++ b/bfd/ChangeLog
@@ -1,3 +1,8 @@
+2020-06-24  Alan Modra  <amodra@gmail.com>
+
+       * vms-alpha.c (_bfd_vms_slurp_etir <ETIR__C_OPR_ASH>): Implement
+       shifts without undefined behaviour.
+
 2020-06-23  H.J. Lu  <hongjiu.lu@intel.com>
 
        * elf-bfd.h (elf_link_hash_table): Add dt_pltgot_required and

diff --git a/bfd/vms-alpha.c b/bfd/vms-alpha.c
index e31a9e4ca10bca344dab67ea208b0784ffb85458..5bf32d67803e38830d321a812b37ab9578c6999f 100644 (file)
--- a/bfd/vms-alpha.c
+++ b/bfd/vms-alpha.c
@@ -34,6 +34,7 @@
 */
 
 #include "sysdep.h"
+#include <limits.h>
 #include "bfd.h"
 #include "bfdlink.h"
 #include "libbfd.h"
@@ -71,6 +72,9 @@
 \f
 
 #define MIN(a,b) ((a) < (b) ? (a) : (b))
+#ifndef CHAR_BIT
+#define CHAR_BIT 8
+#endif
 
 /* The r_type field in a reloc is one of the following values.  */
 #define ALPHA_R_IGNORE         0
@@ -2520,10 +2524,27 @@ _bfd_vms_slurp_etir (bfd *abfd, struct bfd_link_info *info)
                                  _bfd_vms_etir_name (cmd));
              return FALSE;
            }
-         if ((int)op2 < 0)             /* Shift right.  */
-           op1 >>= -(int)op2;
-         else                  /* Shift left.  */
-           op1 <<= (int)op2;
+         if ((bfd_signed_vma) op2 < 0)
+           {
+             /* Shift right.  */
+             bfd_vma sign;
+             op2 = -op2;
+             if (op2 >= CHAR_BIT * sizeof (op1))
+               op2 = CHAR_BIT * sizeof (op1) - 1;
+             /* op1 = (bfd_signed_vma) op1 >> op2; */
+             sign = op1 & ((bfd_vma) 1 << (CHAR_BIT * sizeof (op1) - 1));
+             op1 >>= op2;
+             sign >>= op2;
+             op1 = (op1 ^ sign) - sign;
+           }
+         else
+           {
+             /* Shift left.  */
+             if (op2 >= CHAR_BIT * sizeof (op1))
+               op1 = 0;
+             else
+               op1 <<= op2;
+           }
          if (!_bfd_vms_push (abfd, op1, RELC_NONE)) /* FIXME: sym.  */
            return FALSE;
          break;