llvmpipe: fix overflow bug in total texture size computation

v2: use uint64_t for the total_size variable, per Jose.

Also add two earlier checks for exceeding the max texture size.
For example a 1K^3 RGBA volume would overflow the lpr->image_stride
variable.

Use simple algebra to avoid overflow in intermediate values.
So instead of "x * y > z" use "x > z / y".

This should work if we happen to be on a platform that doesn't have
64-bit types.

Reviewed-by: Jose Fonseca <jfonseca@vmware.com>

diff --git a/src/gallium/drivers/llvmpipe/lp_texture.c b/src/gallium/drivers/llvmpipe/lp_texture.c
index c0a612cd06769302901b7d2bf0ce3f77e1c6fc2b..841df004d14bad0842226fc715afb8866348af03 100644 (file)
--- a/src/gallium/drivers/llvmpipe/lp_texture.c
+++ b/src/gallium/drivers/llvmpipe/lp_texture.c
@@ -113,7 +113,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
    unsigned width = pt->width0;
    unsigned height = pt->height0;
    unsigned depth = pt->depth0;
-   size_t total_size = 0;
+   uint64_t total_size = 0;
 
    assert(LP_MAX_TEXTURE_2D_LEVELS <= LP_MAX_TEXTURE_LEVELS);
    assert(LP_MAX_TEXTURE_3D_LEVELS <= LP_MAX_TEXTURE_LEVELS);
@@ -140,6 +140,12 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
 
          lpr->row_stride[level] = align(nblocksx * block_size, 16);
 
+         /* if row_stride * height > LP_MAX_TEXTURE_SIZE */
+         if (lpr->row_stride[level] > LP_MAX_TEXTURE_SIZE / nblocksy) {
+            /* image too large */
+            goto fail;
+         }
+
          lpr->img_stride[level] = lpr->row_stride[level] * nblocksy;
       }
 
@@ -172,7 +178,15 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
          }
       }
 
-      total_size += lpr->num_slices_faces[level] * lpr->img_stride[level];
+      /* if img_stride * num_slices_faces > LP_MAX_TEXTURE_SIZE */
+      if (lpr->img_stride[level] >
+          LP_MAX_TEXTURE_SIZE / lpr->num_slices_faces[level]) {
+         /* volume too large */
+         goto fail;
+      }
+
+      total_size += (uint64_t) lpr->num_slices_faces[level]
+                  * (uint64_t) lpr->img_stride[level];
       if (total_size > LP_MAX_TEXTURE_SIZE) {
          goto fail;
       }