prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't overflow a jint.

	* prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't
	overflow a jint.
	(_Jv_NewPrimArray): Check for overflowing a jint, replacing a
	check for overflowing size_t, since the lower level functions
	take a jint.
	* testsuite/libjava.lang/newarray_overflow.java:  New file.
	* testsuite/libjava.lang/newarray_overflow.out:  New file.

From-SVN: r72577

diff --git a/libjava/ChangeLog b/libjava/ChangeLog
index 5715231caf764162054db2b11b3eb354aebe71ff..b34b427c6d8c99b379457870b85de97a88bb6445 100644 (file)
--- a/libjava/ChangeLog
+++ b/libjava/ChangeLog
@@ -1,3 +1,13 @@
+2003-10-17  Ralph Loader  <rcl@ihug.co.nz>
+
+       * prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't
+       overflow a jint.
+       (_Jv_NewPrimArray): Check for overflowing a jint, replacing a
+       check for overflowing size_t, since the lower level functions
+       take a jint.
+       * testsuite/libjava.lang/newarray_overflow.java:  New file.
+       * testsuite/libjava.lang/newarray_overflow.out:  New file.
+       
 2003-10-15  Michael Koch  <konqueror@gmx.de>
 
        * java/text/RuleBasedCollator.java

diff --git a/libjava/prims.cc b/libjava/prims.cc
index 71cd8c3ece0f9d2cc233a408c52d0535fb1e61d5..26e88329ed53d280412d8288c7e6985daa4b4b97 100644 (file)
--- a/libjava/prims.cc
+++ b/libjava/prims.cc
@@ -70,8 +70,10 @@ details.  */
 // around for use if we run out of memory.
 static java::lang::OutOfMemoryError *no_memory;
 
-// Largest representable size_t.
-#define SIZE_T_MAX ((size_t) (~ (size_t) 0))
+// Number of bytes in largest array object we create.  This could be
+// increased to the largest size_t value, so long as the appropriate
+// functions are changed to take a size_t argument instead of jint.
+#define MAX_OBJECT_SIZE ((1<<31) - 1)
 
 static const char *no_properties[] = { NULL };
 
@@ -481,6 +483,11 @@ _Jv_NewObjectArray (jsize count, jclass elementClass, jobject init)
   // Ensure that elements pointer is properly aligned.
   jobjectArray obj = NULL;
   size_t size = (size_t) elements (obj);
+  // Check for overflow.
+  if (__builtin_expect ((size_t) count > 
+                       (MAX_OBJECT_SIZE - 1 - size) / sizeof (jobject), false))
+    throw no_memory;
+
   size += count * sizeof (jobject);
 
   jclass klass = _Jv_GetArrayClass (elementClass,
@@ -516,7 +523,7 @@ _Jv_NewPrimArray (jclass eltype, jint count)
 
   // Check for overflow.
   if (__builtin_expect ((size_t) count > 
-                       (SIZE_T_MAX - size) / elsize, false))
+                       (MAX_OBJECT_SIZE - size) / elsize, false))
     throw no_memory;
 
   jclass klass = _Jv_GetArrayClass (eltype, 0);

diff --git a/libjava/testsuite/libjava.lang/newarray_overflow.java b/libjava/testsuite/libjava.lang/newarray_overflow.java
new file mode 100644 (file)
index 0000000..17370b5
--- /dev/null
+++ b/libjava/testsuite/libjava.lang/newarray_overflow.java
@@ -0,0 +1,88 @@
+/* This test checks for two slightly different overflow scenarios in
+ * array allocation.
+ *
+ * The first is that the number of bytes needed for an array size
+ * overflows on a 32 bit machine.
+ *
+ * The second is that on a 64 machine, the number of bytes silently
+ * gets truncated, resulting in too small an object being
+ * allocated.  */
+
+class newarray_overflow
+{
+  static boolean failed = false;
+
+  static void int_check()
+  {
+    int[] x;
+    try
+      {
+       x = new int [1 << 30];
+      }
+    catch (OutOfMemoryError e)
+      {
+       return;
+      }
+    /* If we really get away with it (64 bit machine), that's cool.  */
+    if (x == null) {
+      System.err.println ("int check: new returned null.");
+      failed = true;
+      return;
+    }
+    try
+      {
+       // Only check a few places so we don't thrash too badly.
+       for (int i = 0; i < x.length; i += (1 << 24))
+         if (x[i] != 0)
+           failed = true;
+      }
+    catch (Throwable e)
+      {
+       System.err.print ("int check: ");
+       System.err.println (e);
+       failed = true;
+      }
+  }
+
+  static void object_check()
+  {
+    Object[] x;
+    try
+      {
+       x = new Object [1 << 30];
+       System.err.println ("Alloc succeeded.");
+       System.err.println (x);
+      }
+    catch (OutOfMemoryError e)
+      {
+       return;
+      }
+    /* If we really get away with it (64 bit machine), that's cool.  */
+    if (x == null) {
+      System.err.println ("Object check: new returned null.");
+      failed = true;
+      return;
+    }
+    try
+      {
+       for (int i = 0; i < x.length; i += (1 << 24))
+         if (x[i] != null)
+           failed = true;
+      }
+    catch (Throwable e)
+      {
+       System.err.print ("Object check: ");
+       System.err.println (e);
+       failed = true;
+      }
+  }
+
+  public static void main (String[] ignore)
+  {
+    int_check();
+    object_check();
+
+    if (!failed)
+      System.out.println ("ok");
+  }
+}

diff --git a/libjava/testsuite/libjava.lang/newarray_overflow.out b/libjava/testsuite/libjava.lang/newarray_overflow.out
new file mode 100644 (file)
index 0000000..9766475
--- /dev/null
+++ b/libjava/testsuite/libjava.lang/newarray_overflow.out
@@ -0,0 +1 @@
+ok