Improve File I/O overflow detection in gdbserver (PR server/23198)

Currently, the function used by gdbserver to parse integers from
received File I/O commands will detect overflow and fail for any value
over 0xfffffff.  Among other things, this has the effect of limiting
the file offsets for reading or writing to about 268MB which can be
insufficient for particularly large libraries.

This change allows the parsing of integers up to the true maximum
positive value of 0x7fffffff, increasing the file size limit to about
2GB.

gdb/gdbserver/ChangeLog:
2018-05-23  Erik Kurzinger  <ekurzinger@nvidia.com>

	PR server/23198
	* hostio.c (require_int): Do not report overflow for integers
	between 0xfffffff and 0x7fffffff.

diff --git a/gdb/gdbserver/ChangeLog b/gdb/gdbserver/ChangeLog
index 54a07b8f30ca4e223f1fffdc0acf16944a260b41..7aa59467ac3ffbe386b3ec4401cc6fd929a78a03 100644 (file)
--- a/gdb/gdbserver/ChangeLog
+++ b/gdb/gdbserver/ChangeLog
@@ -1,3 +1,9 @@
+2018-05-23  Erik Kurzinger  <ekurzinger@nvidia.com>
+
+       PR server/23198
+       * hostio.c (require_int): Do not report overflow for integers
+       between 0xfffffff and 0x7fffffff.
+
 2018-05-22  Maciej W. Rozycki  <macro@mips.com>
 
        * linux-mips-low.c [HAVE_PTRACE_GETREGS] (mips_collect_register)

diff --git a/gdb/gdbserver/hostio.c b/gdb/gdbserver/hostio.c
index d2b5a71bade904343d54349f2e01980b3d822d5e..c621edfef56d514daa4f059c851fe2348f9a1989 100644 (file)
--- a/gdb/gdbserver/hostio.c
+++ b/gdb/gdbserver/hostio.c
@@ -96,22 +96,27 @@ static int
 require_int (char **pp, int *value)
 {
   char *p;
-  int count;
+  int count, firstdigit;
 
   p = *pp;
   *value = 0;
   count = 0;
+  firstdigit = -1;
 
   while (*p && *p != ',')
     {
       int nib;
 
-      /* Don't allow overflow.  */
-      if (count >= 7)
+      if (safe_fromhex (p[0], &nib))
        return -1;
 
-      if (safe_fromhex (p[0], &nib))
+      if (firstdigit == -1)
+       firstdigit = nib;
+
+      /* Don't allow overflow.  */
+      if (count >= 8 || (count == 7 && firstdigit >= 0x8))
        return -1;
+
       *value = *value * 16 + nib;
       p++;
       count++;