mesa: Fix element count for byte-swaps in texstore, readpix and texgetimage

Some old format conversion code in pack.c implemented byte-swapping like this:

GLint comps = _mesa_components_in_format(dstFormat);
GLint swapSize = _mesa_sizeof_packed_type(dstType);
if (swapSize == 2)
   _mesa_swap2((GLushort *) dstAddr, n * comps);
else if (swapSize == 4)
   _mesa_swap4((GLuint *) dstAddr, n * comps);

where n is the pixel count. But this is incorrect for packed formats,
where _mesa_sizeof_packed_type is already returning the size of a pixel
instead of the size of a single component, so multiplying this by the
number of components in the format results in a larger element count
for _mesa_swap than we want.

Unfortunately, we followed the same implementation for byte-swapping
in the rewrite of the format conversion code for texstore, readpixels
and texgetimage.

This patch computes the correct element counts for _mesa_swap calls
by computing the bytes per pixel in the image and dividing that by the
swap size to obtain the number of swaps required per pixel. Then multiplies
that by the number of pixels in the image to obtain the swap count that
we need to use.

Also, when handling byte-swapping in texstore_rgba, we were ignoring
the image's depth. This patch fixes this too.

Reviewed-by: Jason Ekstrand <jason.ekstrand@intel.com>
Cc: "10.5" <mesa-stable@lists.freedesktop.org>

diff --git a/src/mesa/main/readpix.c b/src/mesa/main/readpix.c
index 85f900d6b3f02fde375f847055e2cb9a3996523b..ca4b9431bbe4934a6bc05f1c93295b8c96928ca6 100644 (file)
--- a/src/mesa/main/readpix.c
+++ b/src/mesa/main/readpix.c
@@ -605,12 +605,15 @@ read_rgba_pixels( struct gl_context *ctx,
 done_swap:
    /* Handle byte swapping if required */
    if (packing->SwapBytes) {
-      int components = _mesa_components_in_format(format);
       GLint swapSize = _mesa_sizeof_packed_type(type);
-      if (swapSize == 2)
-         _mesa_swap2((GLushort *) dst, width * height * components);
-      else if (swapSize == 4)
-         _mesa_swap4((GLuint *) dst, width * height * components);
+      if (swapSize == 2 || swapSize == 4) {
+         int swapsPerPixel = _mesa_bytes_per_pixel(format, type) / swapSize;
+         assert(_mesa_bytes_per_pixel(format, type) % swapSize == 0);
+         if (swapSize == 2)
+            _mesa_swap2((GLushort *) dst, width * height * swapsPerPixel);
+         else if (swapSize == 4)
+            _mesa_swap4((GLuint *) dst, width * height * swapsPerPixel);
+      }
    }
 
 done_unmap:

diff --git a/src/mesa/main/texgetimage.c b/src/mesa/main/texgetimage.c
index ee465e65434bfd80da3105e979c2e5a60b2734bc..405f085342612fda39b2d7f18172529f26881d58 100644 (file)
--- a/src/mesa/main/texgetimage.c
+++ b/src/mesa/main/texgetimage.c
@@ -511,12 +511,15 @@ get_tex_rgba_uncompressed(struct gl_context *ctx, GLuint dimensions,
    do_swap:
       /* Handle byte swapping if required */
       if (ctx->Pack.SwapBytes) {
-         int components = _mesa_components_in_format(format);
          GLint swapSize = _mesa_sizeof_packed_type(type);
-         if (swapSize == 2)
-            _mesa_swap2((GLushort *) dest, width * height * components);
-         else if (swapSize == 4)
-            _mesa_swap4((GLuint *) dest, width * height * components);
+         if (swapSize == 2 || swapSize == 4) {
+            int swapsPerPixel = _mesa_bytes_per_pixel(format, type) / swapSize;
+            assert(_mesa_bytes_per_pixel(format, type) % swapSize == 0);
+            if (swapSize == 2)
+               _mesa_swap2((GLushort *) dest, width * height * swapsPerPixel);
+            else if (swapSize == 4)
+               _mesa_swap4((GLuint *) dest, width * height * swapsPerPixel);
+         }
       }
 
       /* Unmap the src texture buffer */

diff --git a/src/mesa/main/texstore.c b/src/mesa/main/texstore.c
index 9c7080ab82ecb0753408d94605a23c5022ff1460..f25c0d7e0e1b21a0cf19d14e0f09a2e6b355fcec 100644 (file)
--- a/src/mesa/main/texstore.c
+++ b/src/mesa/main/texstore.c
@@ -728,15 +728,19 @@ texstore_rgba(TEXSTORE_PARAMS)
        */
       GLint swapSize = _mesa_sizeof_packed_type(srcType);
       if (swapSize == 2 || swapSize == 4) {
-         int components = _mesa_components_in_format(srcFormat);
-         int elementCount = srcWidth * srcHeight * components;
-         tempImage = malloc(elementCount * swapSize);
+         int bytesPerPixel = _mesa_bytes_per_pixel(srcFormat, srcType);
+         assert(bytesPerPixel % swapSize == 0);
+         int swapsPerPixel = bytesPerPixel / swapSize;
+         int elementCount = srcWidth * srcHeight * srcDepth;
+         tempImage = malloc(elementCount * bytesPerPixel);
          if (!tempImage)
             return GL_FALSE;
          if (swapSize == 2)
-            _mesa_swap2_copy(tempImage, (GLushort *) srcAddr, elementCount);
+            _mesa_swap2_copy(tempImage, (GLushort *) srcAddr,
+                             elementCount * swapsPerPixel);
          else
-            _mesa_swap4_copy(tempImage, (GLuint *) srcAddr, elementCount);
+            _mesa_swap4_copy(tempImage, (GLuint *) srcAddr,
+                             elementCount * swapsPerPixel);
          srcAddr = tempImage;
       }
    }