rgtc: fix issues with compressor and signed types.

With signed types we weren't hitting this test however the comment
stating this doesn't happen often doesn't apply when using signed
types since an all 0 block is quite common which isn't abs min or max.

this fixes the limits correctly again also.

Signed-off-by: Dave Airlie <airlied@redhat.com>

diff --git a/src/mesa/main/texcompress_rgtc.c b/src/mesa/main/texcompress_rgtc.c
index 6b4e3b13cbef5e72c29e0127a945b79278e3ee30..1b873035b683784bf7a5c918a07d14d8b2295c2f 100644 (file)
--- a/src/mesa/main/texcompress_rgtc.c
+++ b/src/mesa/main/texcompress_rgtc.c
@@ -364,7 +364,7 @@ static void _fetch_texel_rgtc_s(GLint srcRowStride, const GLbyte *pixdata,
    else if (code < 6)
       decode = ((alpha0 * (6 - code) + (alpha1 * (code - 1))) / 5);
    else if (code == 6)
-      decode = -127;
+      decode = -128;
    else
       decode = 127;
 
@@ -442,8 +442,8 @@ _mesa_fetch_texel_2d_f_signed_rg_rgtc2(const struct gl_texture_image *texImage,
 
 #define TAG(x) signed_##x
 #define TYPE GLbyte
-#define T_MIN (GLbyte)-127
-#define T_MAX (GLbyte)128
+#define T_MIN (GLbyte)-128
+#define T_MAX (GLbyte)127
 
 #include "texcompress_rgtc_tmp.h"
 

diff --git a/src/mesa/main/texcompress_rgtc_tmp.h b/src/mesa/main/texcompress_rgtc_tmp.h
index 0f830a5d9049a7921810e596c398088a34f9155d..9377a6bd3bee2a5db7ea9d0c237c2bc7ec885e81 100644 (file)
--- a/src/mesa/main/texcompress_rgtc_tmp.h
+++ b/src/mesa/main/texcompress_rgtc_tmp.h
@@ -73,9 +73,9 @@ static void TAG(encode_rgtc_chan)(TYPE *blkaddr, TYPE srccolors[4][4],
    }
 
 
-   if ((alphabase[0] > alphabase[1]) && !(alphaabsmin && alphaabsmax)) { /* one color, either max or min */
+   if (((alphabase[0] > alphabase[1]) && !(alphaabsmin && alphaabsmax))
+       || (alphabase[0] == alphabase[1] && !alphaabsmin && !alphaabsmax)) { /* one color, either max or min */
       /* shortcut here since it is a very common case (and also avoids later problems) */
-      /* || (alphabase[0] == alphabase[1] && !alphaabsmin && !alphaabsmax) */
       /* could also thest for alpha0 == alpha1 (and not min/max), but probably not common, so don't bother */
 
       *blkaddr++ = srccolors[0][0];
@@ -223,8 +223,8 @@ static void TAG(encode_rgtc_chan)(TYPE *blkaddr, TYPE srccolors[4][4],
          GLshort blockerrlin2 = 0;
          TYPE nralphainrangelow = 0;
          TYPE nralphainrangehigh = 0;
-         alphatest[0] = 0xff;
-         alphatest[1] = 0x0;
+         alphatest[0] = T_MAX;
+         alphatest[1] = T_MIN;
          /* if we have large range it's likely there are values close to 0/255, try to map them to 0/255 */
          for (j = 0; j < numypixels; j++) {
             for (i = 0; i < numxpixels; i++) {
@@ -236,8 +236,8 @@ static void TAG(encode_rgtc_chan)(TYPE *blkaddr, TYPE srccolors[4][4],
          }
           /* shouldn't happen too often, don't really care about those degenerated cases */
           if (alphatest[1] <= alphatest[0]) {
-             alphatest[0] = 1;
-             alphatest[1] = 254;
+             alphatest[0] = T_MIN+1;
+             alphatest[1] = T_MAX-1;
          }
          for (aindex = 0; aindex < 5; aindex++) {
          /* don't forget here is always rounded down */
@@ -305,7 +305,7 @@ static void TAG(encode_rgtc_chan)(TYPE *blkaddr, TYPE srccolors[4][4],
          }
          alphatest[1] = alphatest[1] + (blockerrlin2 / nralphainrangehigh);
          if (alphatest[1] > T_MAX) {
-            alphatest[1] = T_MIN;
+            alphatest[1] = T_MAX;
          }
 
          alphablockerror3 = 0;
@@ -354,23 +354,36 @@ static void TAG(encode_rgtc_chan)(TYPE *blkaddr, TYPE srccolors[4][4],
          }
       }
    }
+
   /* write the alpha values and encoding back. */
    if ((alphablockerror1 <= alphablockerror2) && (alphablockerror1 <= alphablockerror3)) {
 #if RGTC_DEBUG
       if (alphablockerror1 > 96) fprintf(stderr, "enc1 used, error %d\n", alphablockerror1);
+      fprintf(stderr,"w1: min %d max %d au0 %d au1 %d\n",
+             T_MIN, T_MAX,
+             alphause[1], alphause[0]);
 #endif
+
       TAG(write_rgtc_encoded_channel)( blkaddr, alphause[1], alphause[0], alphaenc1 );
    }
    else if (alphablockerror2 <= alphablockerror3) {
 #if RGTC_DEBUG
       if (alphablockerror2 > 96) fprintf(stderr, "enc2 used, error %d\n", alphablockerror2);
+      fprintf(stderr,"w2: min %d max %d au0 %d au1 %d\n",
+             T_MIN, T_MAX,
+             alphabase[0], alphabase[1]);
 #endif
+
       TAG(write_rgtc_encoded_channel)( blkaddr, alphabase[0], alphabase[1], alphaenc2 );
    }
    else {
 #if RGTC_DEBUG
       fprintf(stderr, "enc3 used, error %d\n", alphablockerror3);
+      fprintf(stderr,"w3: min %d max %d au0 %d au1 %d\n",
+             T_MIN, T_MAX,
+             alphatest[0], alphatest[1]);
 #endif
+
       TAG(write_rgtc_encoded_channel)( blkaddr, (TYPE)alphatest[0], (TYPE)alphatest[1], alphaenc3 );
    }
 }