src/mesa/main/texcompress_rgtc.c

   1 /*
   2  * Copyright (C) 2011 Red Hat Inc.
   3  *
   4  * block compression parts are:
   5  * Copyright (C) 2004  Roland Scheidegger   All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24  * DEALINGS IN THE SOFTWARE.
  25  *
  26  * Author:
  27  *    Dave Airlie
  28  */
  29
  30 /**
  31  * \file texcompress_rgtc.c
  32  * GL_EXT_texture_compression_rgtc support.
  33  */
  34
  35
  36 #include "glheader.h"
  37 #include "imports.h"
  38 #include "colormac.h"
  39 #include "image.h"
  40 #include "macros.h"
  41 #include "mfeatures.h"
  42 #include "mipmap.h"
  43 #include "texcompress.h"
  44 #include "texcompress_rgtc.h"
  45 #include "texstore.h"
  46
  47 #define RGTC_DEBUG 0
  48
  49 static void encode_rgtc_chan_u(GLubyte *blkaddr, GLubyte srccolors[4][4],
  50                              GLint numxpixels, GLint numypixels);
  51 static void encode_rgtc_chan_s(GLbyte *blkaddr, GLbyte srccolors[4][4],
  52                              GLint numxpixels, GLint numypixels);
  53
  54 static void extractsrc_u( GLubyte srcpixels[4][4], const GLchan *srcaddr,
  55                           GLint srcRowStride, GLint numxpixels, GLint numypixels, GLint comps)
  56 {
  57    GLubyte i, j;
  58    const GLchan *curaddr;
  59    for (j = 0; j < numypixels; j++) {
  60       curaddr = srcaddr + j * srcRowStride * comps;
  61       for (i = 0; i < numxpixels; i++) {
  62          srcpixels[j][i] = *curaddr / (CHAN_MAX / 255);
  63          curaddr += comps;
  64       }
  65    }
  66 }
  67
  68 static void extractsrc_s( GLbyte srcpixels[4][4], const GLfloat *srcaddr,
  69                           GLint srcRowStride, GLint numxpixels, GLint numypixels, GLint comps)
  70 {
  71    GLubyte i, j;
  72    const GLfloat *curaddr;
  73    for (j = 0; j < numypixels; j++) {
  74       curaddr = srcaddr + j * srcRowStride * comps;
  75       for (i = 0; i < numxpixels; i++) {
  76          srcpixels[j][i] = FLOAT_TO_BYTE_TEX(*curaddr);
  77          curaddr += comps;
  78       }
  79    }
  80 }
  81
  82
  83 GLboolean
  84 _mesa_texstore_red_rgtc1(TEXSTORE_PARAMS)
  85 {
  86    GLubyte *dst;
  87    const GLint texWidth = dstRowStride * 4 / 8; /* a bit of a hack */
  88    const GLchan *tempImage = NULL;
  89    int i, j;
  90    int numxpixels, numypixels;
  91    const GLchan *srcaddr;
  92    GLubyte srcpixels[4][4];
  93    GLubyte *blkaddr;
  94    GLint dstRowDiff;
  95    ASSERT(dstFormat == MESA_FORMAT_RED_RGTC1);
  96    ASSERT(dstXoffset % 4 == 0);
  97    ASSERT(dstYoffset % 4 == 0);
  98    ASSERT(dstZoffset % 4 == 0);
  99    (void) dstZoffset;
 100    (void) dstImageOffsets;
 101
 102
 103    tempImage = _mesa_make_temp_chan_image(ctx, dims,
 104                                           baseInternalFormat,
 105                                           _mesa_get_format_base_format(dstFormat),
 106                                           srcWidth, srcHeight, srcDepth,
 107                                           srcFormat, srcType, srcAddr,
 108                                           srcPacking);
 109    if (!tempImage)
 110       return GL_FALSE; /* out of memory */
 111
 112    dst = _mesa_compressed_image_address(dstXoffset, dstYoffset, 0,
 113                                         dstFormat,
 114                                         texWidth, (GLubyte *) dstAddr);
 115
 116    blkaddr = dst;
 117    dstRowDiff = dstRowStride >= (srcWidth * 4) ? dstRowStride - (((srcWidth + 3) & ~3) * 4) : 0;
 118    for (j = 0; j < srcHeight; j+=4) {
 119       if (srcHeight > j + 3) numypixels = 4;
 120       else numypixels = srcHeight - j;
 121       srcaddr = tempImage + j * srcWidth;
 122       for (i = 0; i < srcWidth; i += 4) {
 123          if (srcWidth > i + 3) numxpixels = 4;
 124          else numxpixels = srcWidth - i;
 125          extractsrc_u(srcpixels, srcaddr, srcWidth, numxpixels, numypixels, 1);
 126          encode_rgtc_chan_u(blkaddr, srcpixels, numxpixels, numypixels);
 127          srcaddr += numxpixels;
 128          blkaddr += 8;
 129       }
 130       blkaddr += dstRowDiff;
 131    }
 132    if (tempImage)
 133       free((void *) tempImage);
 134
 135    return GL_TRUE;
 136 }
 137
 138 GLboolean
 139 _mesa_texstore_signed_red_rgtc1(TEXSTORE_PARAMS)
 140 {
 141    GLbyte *dst;
 142    const GLint texWidth = dstRowStride * 4 / 8; /* a bit of a hack */
 143    const GLfloat *tempImage = NULL;
 144    int i, j;
 145    int numxpixels, numypixels;
 146    const GLfloat *srcaddr;
 147    GLbyte srcpixels[4][4];
 148    GLbyte *blkaddr;
 149    GLint dstRowDiff;
 150    ASSERT(dstFormat == MESA_FORMAT_SIGNED_RED_RGTC1);
 151    ASSERT(dstXoffset % 4 == 0);
 152    ASSERT(dstYoffset % 4 == 0);
 153    ASSERT(dstZoffset % 4 == 0);
 154    (void) dstZoffset;
 155    (void) dstImageOffsets;
 156
 157    tempImage = _mesa_make_temp_float_image(ctx, dims,
 158                                            baseInternalFormat,
 159                                            _mesa_get_format_base_format(dstFormat),
 160                                            srcWidth, srcHeight, srcDepth,
 161                                            srcFormat, srcType, srcAddr,
 162                                            srcPacking, 0x0);
 163    if (!tempImage)
 164       return GL_FALSE; /* out of memory */
 165
 166    dst = (GLbyte *)_mesa_compressed_image_address(dstXoffset, dstYoffset, 0,
 167                                                   dstFormat,
 168                                                   texWidth, (GLubyte *) dstAddr);
 169
 170    blkaddr = dst;
 171    dstRowDiff = dstRowStride >= (srcWidth * 4) ? dstRowStride - (((srcWidth + 3) & ~3) * 4) : 0;
 172    for (j = 0; j < srcHeight; j+=4) {
 173       if (srcHeight > j + 3) numypixels = 4;
 174       else numypixels = srcHeight - j;
 175       srcaddr = tempImage + j * srcWidth;
 176       for (i = 0; i < srcWidth; i += 4) {
 177          if (srcWidth > i + 3) numxpixels = 4;
 178          else numxpixels = srcWidth - i;
 179          extractsrc_s(srcpixels, srcaddr, srcWidth, numxpixels, numypixels, 1);
 180          encode_rgtc_chan_s(blkaddr, srcpixels, numxpixels, numypixels);
 181          srcaddr += numxpixels;
 182          blkaddr += 8;
 183       }
 184       blkaddr += dstRowDiff;
 185    }
 186    if (tempImage)
 187       free((void *) tempImage);
 188
 189    return GL_TRUE;
 190 }
 191
 192 GLboolean
 193 _mesa_texstore_rg_rgtc2(TEXSTORE_PARAMS)
 194 {
 195    GLubyte *dst;
 196    const GLint texWidth = dstRowStride * 4 / 16; /* a bit of a hack */
 197    const GLchan *tempImage = NULL;
 198    int i, j;
 199    int numxpixels, numypixels;
 200    const GLchan *srcaddr;
 201    GLubyte srcpixels[4][4];
 202    GLubyte *blkaddr;
 203    GLint dstRowDiff;
 204
 205    ASSERT(dstFormat == MESA_FORMAT_RG_RGTC2);
 206    ASSERT(dstXoffset % 4 == 0);
 207    ASSERT(dstYoffset % 4 == 0);
 208    ASSERT(dstZoffset % 4 == 0);
 209    (void) dstZoffset;
 210    (void) dstImageOffsets;
 211
 212    tempImage = _mesa_make_temp_chan_image(ctx, dims,
 213                                           baseInternalFormat,
 214                                           _mesa_get_format_base_format(dstFormat),
 215                                           srcWidth, srcHeight, srcDepth,
 216                                           srcFormat, srcType, srcAddr,
 217                                           srcPacking);
 218    if (!tempImage)
 219       return GL_FALSE; /* out of memory */
 220
 221    dst = _mesa_compressed_image_address(dstXoffset, dstYoffset, 0,
 222                                         dstFormat,
 223                                         texWidth, (GLubyte *) dstAddr);
 224
 225    blkaddr = dst;
 226    dstRowDiff = dstRowStride >= (srcWidth * 8) ? dstRowStride - (((srcWidth + 7) & ~7) * 8) : 0;
 227    for (j = 0; j < srcHeight; j+=4) {
 228       if (srcHeight > j + 3) numypixels = 4;
 229       else numypixels = srcHeight - j;
 230       srcaddr = tempImage + j * srcWidth * 2;
 231       for (i = 0; i < srcWidth; i += 4) {
 232          if (srcWidth > i + 3) numxpixels = 4;
 233          else numxpixels = srcWidth - i;
 234          extractsrc_u(srcpixels, srcaddr, srcWidth, numxpixels, numypixels, 2);
 235          encode_rgtc_chan_u(blkaddr, srcpixels, numxpixels, numypixels);
 236
 237          blkaddr += 8;
 238          extractsrc_u(srcpixels, (GLchan *)srcaddr + 1, srcWidth, numxpixels, numypixels, 2);
 239          encode_rgtc_chan_u(blkaddr, srcpixels, numxpixels, numypixels);
 240
 241          blkaddr += 8;
 242
 243          srcaddr += numxpixels * 2;
 244       }
 245       blkaddr += dstRowDiff;
 246    }
 247    if (tempImage)
 248       free((void *) tempImage);
 249
 250    return GL_TRUE;
 251 }
 252
 253 GLboolean
 254 _mesa_texstore_signed_rg_rgtc2(TEXSTORE_PARAMS)
 255 {
 256    GLbyte *dst;
 257    const GLint texWidth = dstRowStride * 4 / 16; /* a bit of a hack */
 258    const GLfloat *tempImage = NULL;
 259    int i, j;
 260    int numxpixels, numypixels;
 261    const GLfloat *srcaddr;
 262    GLbyte srcpixels[4][4];
 263    GLbyte *blkaddr;
 264    GLint dstRowDiff;
 265
 266    ASSERT(dstFormat == MESA_FORMAT_SIGNED_RG_RGTC2);
 267    ASSERT(dstXoffset % 4 == 0);
 268    ASSERT(dstYoffset % 4 == 0);
 269    ASSERT(dstZoffset % 4 == 0);
 270    (void) dstZoffset;
 271    (void) dstImageOffsets;
 272
 273    tempImage = _mesa_make_temp_float_image(ctx, dims,
 274                                            baseInternalFormat,
 275                                            _mesa_get_format_base_format(dstFormat),
 276                                            srcWidth, srcHeight, srcDepth,
 277                                            srcFormat, srcType, srcAddr,
 278                                            srcPacking, 0x0);
 279    if (!tempImage)
 280       return GL_FALSE; /* out of memory */
 281
 282    dst = (GLbyte *)_mesa_compressed_image_address(dstXoffset, dstYoffset, 0,
 283                                                   dstFormat,
 284                                                   texWidth, (GLubyte *) dstAddr);
 285
 286    blkaddr = dst;
 287    dstRowDiff = dstRowStride >= (srcWidth * 8) ? dstRowStride - (((srcWidth + 7) & ~7) * 8) : 0;
 288    for (j = 0; j < srcHeight; j += 4) {
 289       if (srcHeight > j + 3) numypixels = 4;
 290       else numypixels = srcHeight - j;
 291       srcaddr = tempImage + j * srcWidth * 2;
 292       for (i = 0; i < srcWidth; i += 4) {
 293          if (srcWidth > i + 3) numxpixels = 4;
 294          else numxpixels = srcWidth - i;
 295
 296          extractsrc_s(srcpixels, srcaddr, srcWidth, numxpixels, numypixels, 2);
 297          encode_rgtc_chan_s(blkaddr, srcpixels, numxpixels, numypixels);
 298          blkaddr += 8;
 299
 300          extractsrc_s(srcpixels, srcaddr + 1, srcWidth, numxpixels, numypixels, 2);
 301          encode_rgtc_chan_s(blkaddr, srcpixels, numxpixels, numypixels);
 302          blkaddr += 8;
 303
 304          srcaddr += numxpixels * 2;
 305
 306       }
 307       blkaddr += dstRowDiff;
 308    }
 309    if (tempImage)
 310       free((void *) tempImage);
 311
 312    return GL_TRUE;
 313 }
 314
 315 static void _fetch_texel_rgtc_u(GLint srcRowStride, const GLubyte *pixdata,
 316                                 GLint i, GLint j, GLchan *value, int comps)
 317 {
 318    GLchan decode;
 319    const GLubyte *blksrc = (pixdata + ((srcRowStride + 3) / 4 * (j / 4) + (i / 4)) * 8 * comps);
 320    const GLubyte alpha0 = blksrc[0];
 321    const GLubyte alpha1 = blksrc[1];
 322    const GLubyte bit_pos = ((j&3) * 4 + (i&3)) * 3;
 323    const GLubyte acodelow = blksrc[2 + bit_pos / 8];
 324    const GLubyte acodehigh = blksrc[3 + bit_pos / 8];
 325    const GLubyte code = (acodelow >> (bit_pos & 0x7) |
 326       (acodehigh  << (8 - (bit_pos & 0x7)))) & 0x7;
 327
 328    if (code == 0)
 329       decode = UBYTE_TO_CHAN( alpha0 );
 330    else if (code == 1)
 331       decode = UBYTE_TO_CHAN( alpha1 );
 332    else if (alpha0 > alpha1)
 333       decode = UBYTE_TO_CHAN( ((alpha0 * (8 - code) + (alpha1 * (code - 1))) / 7) );
 334    else if (code < 6)
 335       decode = UBYTE_TO_CHAN( ((alpha0 * (6 - code) + (alpha1 * (code - 1))) / 5) );
 336    else if (code == 6)
 337       decode = 0;
 338    else
 339       decode = CHAN_MAX;
 340
 341    *value = decode;
 342 }
 343
 344
 345 static void _fetch_texel_rgtc_s(GLint srcRowStride, const GLbyte *pixdata,
 346                                 GLint i, GLint j, GLbyte *value, int comps)
 347 {
 348    GLbyte decode;
 349    const GLbyte *blksrc = (pixdata + ((srcRowStride + 3) / 4 * (j / 4) + (i / 4)) * 8 * comps);
 350    const GLbyte alpha0 = blksrc[0];
 351    const GLbyte alpha1 = blksrc[1];
 352    const GLbyte bit_pos = ((j&3) * 4 + (i&3)) * 3;
 353    const GLbyte acodelow = blksrc[2 + bit_pos / 8];
 354    const GLbyte acodehigh = blksrc[3 + bit_pos / 8];
 355    const GLbyte code = (acodelow >> (bit_pos & 0x7) |
 356       (acodehigh  << (8 - (bit_pos & 0x7)))) & 0x7;
 357
 358    if (code == 0)
 359       decode = alpha0;
 360    else if (code == 1)
 361       decode = alpha1;
 362    else if (alpha0 > alpha1)
 363       decode = ((alpha0 * (8 - code) + (alpha1 * (code - 1))) / 7);
 364    else if (code < 6)
 365       decode = ((alpha0 * (6 - code) + (alpha1 * (code - 1))) / 5);
 366    else if (code == 6)
 367       decode = -128;
 368    else
 369       decode = 127;
 370
 371    *value = decode;
 372 }
 373
 374 void
 375 _mesa_fetch_texel_2d_f_red_rgtc1(const struct gl_texture_image *texImage,
 376                                  GLint i, GLint j, GLint k, GLfloat *texel)
 377 {
 378    GLchan red;
 379    _fetch_texel_rgtc_u(texImage->RowStride, (GLubyte *)(texImage->Data),
 380                        i, j, &red, 1);
 381    texel[RCOMP] = CHAN_TO_FLOAT(red);
 382    texel[GCOMP] = 0.0;
 383    texel[BCOMP] = 0.0;
 384    texel[ACOMP] = 1.0;
 385 }
 386
 387 void
 388 _mesa_fetch_texel_2d_f_signed_red_rgtc1(const struct gl_texture_image *texImage,
 389                                         GLint i, GLint j, GLint k, GLfloat *texel)
 390 {
 391    GLbyte red;
 392    _fetch_texel_rgtc_s(texImage->RowStride, (GLbyte *)(texImage->Data),
 393                        i, j, &red, 1);
 394    texel[RCOMP] = BYTE_TO_FLOAT_TEX(red);
 395    texel[GCOMP] = 0.0;
 396    texel[BCOMP] = 0.0;
 397    texel[ACOMP] = 1.0;
 398 }
 399
 400 void
 401 _mesa_fetch_texel_2d_f_rg_rgtc2(const struct gl_texture_image *texImage,
 402                                  GLint i, GLint j, GLint k, GLfloat *texel)
 403 {
 404    GLchan red, green;
 405    _fetch_texel_rgtc_u(texImage->RowStride, (GLubyte *)(texImage->Data),
 406                      i, j, &red, 2);
 407    _fetch_texel_rgtc_u(texImage->RowStride, (GLubyte *)(texImage->Data) + 8,
 408                      i, j, &green, 2);
 409    texel[RCOMP] = CHAN_TO_FLOAT(red);
 410    texel[GCOMP] = CHAN_TO_FLOAT(green);
 411    texel[BCOMP] = 0.0;
 412    texel[ACOMP] = 1.0;
 413 }
 414
 415 void
 416 _mesa_fetch_texel_2d_f_signed_rg_rgtc2(const struct gl_texture_image *texImage,
 417                                        GLint i, GLint j, GLint k, GLfloat *texel)
 418 {
 419    GLbyte red, green;
 420    _fetch_texel_rgtc_s(texImage->RowStride, (GLbyte *)(texImage->Data),
 421                      i, j, &red, 2);
 422    _fetch_texel_rgtc_s(texImage->RowStride, (GLbyte *)(texImage->Data) + 8,
 423                      i, j, &green, 2);
 424    texel[RCOMP] = BYTE_TO_FLOAT_TEX(red);
 425    texel[GCOMP] = BYTE_TO_FLOAT_TEX(green);
 426    texel[BCOMP] = 0.0;
 427    texel[ACOMP] = 1.0;
 428 }
 429
 430 static void write_rgtc_encoded_channel(GLubyte *blkaddr,
 431                                        GLubyte alphabase1,
 432                                        GLubyte alphabase2,
 433                                        GLubyte alphaenc[16])
 434 {
 435    *blkaddr++ = alphabase1;
 436    *blkaddr++ = alphabase2;
 437    *blkaddr++ = alphaenc[0] | (alphaenc[1] << 3) | ((alphaenc[2] & 3) << 6);
 438    *blkaddr++ = (alphaenc[2] >> 2) | (alphaenc[3] << 1) | (alphaenc[4] << 4) | ((alphaenc[5] & 1) << 7);
 439    *blkaddr++ = (alphaenc[5] >> 1) | (alphaenc[6] << 2) | (alphaenc[7] << 5);
 440    *blkaddr++ = alphaenc[8] | (alphaenc[9] << 3) | ((alphaenc[10] & 3) << 6);
 441    *blkaddr++ = (alphaenc[10] >> 2) | (alphaenc[11] << 1) | (alphaenc[12] << 4) | ((alphaenc[13] & 1) << 7);
 442    *blkaddr++ = (alphaenc[13] >> 1) | (alphaenc[14] << 2) | (alphaenc[15] << 5);
 443 }
 444
 445 static void encode_rgtc_chan_u(GLubyte *blkaddr, GLubyte srccolors[4][4],
 446                              GLint numxpixels, GLint numypixels)
 447 {
 448    GLubyte alphabase[2], alphause[2];
 449    GLshort alphatest[2] = { 0 };
 450    GLuint alphablockerror1, alphablockerror2, alphablockerror3;
 451    GLubyte i, j, aindex, acutValues[7];
 452    GLubyte alphaenc1[16], alphaenc2[16], alphaenc3[16];
 453    GLboolean alphaabsmin = GL_FALSE;
 454    GLboolean alphaabsmax = GL_FALSE;
 455    GLshort alphadist;
 456
 457    /* find lowest and highest alpha value in block, alphabase[0] lowest, alphabase[1] highest */
 458    alphabase[0] = 0xff; alphabase[1] = 0x0;
 459    for (j = 0; j < numypixels; j++) {
 460       for (i = 0; i < numxpixels; i++) {
 461          if (srccolors[j][i] == 0)
 462             alphaabsmin = GL_TRUE;
 463          else if (srccolors[j][i] == 255)
 464             alphaabsmax = GL_TRUE;
 465          else {
 466             if (srccolors[j][i] > alphabase[1])
 467                alphabase[1] = srccolors[j][i];
 468             if (srccolors[j][i] < alphabase[0])
 469                alphabase[0] = srccolors[j][i];
 470          }
 471       }
 472    }
 473
 474
 475    if ((alphabase[0] > alphabase[1]) && !(alphaabsmin && alphaabsmax)) { /* one color, either max or min */
 476       /* shortcut here since it is a very common case (and also avoids later problems) */
 477       /* || (alphabase[0] == alphabase[1] && !alphaabsmin && !alphaabsmax) */
 478       /* could also thest for alpha0 == alpha1 (and not min/max), but probably not common, so don't bother */
 479
 480       *blkaddr++ = srccolors[0][0];
 481       blkaddr++;
 482       *blkaddr++ = 0;
 483       *blkaddr++ = 0;
 484       *blkaddr++ = 0;
 485       *blkaddr++ = 0;
 486       *blkaddr++ = 0;
 487       *blkaddr++ = 0;
 488 #if RGTC_DEBUG
 489       fprintf(stderr, "enc0 used\n");
 490 #endif
 491       return;
 492    }
 493
 494    /* find best encoding for alpha0 > alpha1 */
 495    /* it's possible this encoding is better even if both alphaabsmin and alphaabsmax are true */
 496    alphablockerror1 = 0x0;
 497    alphablockerror2 = 0xffffffff;
 498    alphablockerror3 = 0xffffffff;
 499    if (alphaabsmin) alphause[0] = 0;
 500    else alphause[0] = alphabase[0];
 501    if (alphaabsmax) alphause[1] = 255;
 502    else alphause[1] = alphabase[1];
 503    /* calculate the 7 cut values, just the middle between 2 of the computed alpha values */
 504    for (aindex = 0; aindex < 7; aindex++) {
 505       /* don't forget here is always rounded down */
 506       acutValues[aindex] = (alphause[0] * (2*aindex + 1) + alphause[1] * (14 - (2*aindex + 1))) / 14;
 507    }
 508
 509    for (j = 0; j < numypixels; j++) {
 510       for (i = 0; i < numxpixels; i++) {
 511          /* maybe it's overkill to have the most complicated calculation just for the error
 512             calculation which we only need to figure out if encoding1 or encoding2 is better... */
 513          if (srccolors[j][i] > acutValues[0]) {
 514             alphaenc1[4*j + i] = 0;
 515             alphadist = srccolors[j][i] - alphause[1];
 516          }
 517          else if (srccolors[j][i] > acutValues[1]) {
 518             alphaenc1[4*j + i] = 2;
 519             alphadist = srccolors[j][i] - (alphause[1] * 6 + alphause[0] * 1) / 7;
 520          }
 521          else if (srccolors[j][i] > acutValues[2]) {
 522             alphaenc1[4*j + i] = 3;
 523             alphadist = srccolors[j][i] - (alphause[1] * 5 + alphause[0] * 2) / 7;
 524          }
 525          else if (srccolors[j][i] > acutValues[3]) {
 526             alphaenc1[4*j + i] = 4;
 527             alphadist = srccolors[j][i] - (alphause[1] * 4 + alphause[0] * 3) / 7;
 528          }
 529          else if (srccolors[j][i] > acutValues[4]) {
 530             alphaenc1[4*j + i] = 5;
 531             alphadist = srccolors[j][i] - (alphause[1] * 3 + alphause[0] * 4) / 7;
 532          }
 533          else if (srccolors[j][i] > acutValues[5]) {
 534             alphaenc1[4*j + i] = 6;
 535             alphadist = srccolors[j][i] - (alphause[1] * 2 + alphause[0] * 5) / 7;
 536          }
 537          else if (srccolors[j][i] > acutValues[6]) {
 538             alphaenc1[4*j + i] = 7;
 539             alphadist = srccolors[j][i] - (alphause[1] * 1 + alphause[0] * 6) / 7;
 540          }
 541          else {
 542             alphaenc1[4*j + i] = 1;
 543             alphadist = srccolors[j][i] - alphause[0];
 544          }
 545          alphablockerror1 += alphadist * alphadist;
 546       }
 547    }
 548
 549 #if RGTC_DEBUG
 550    for (i = 0; i < 16; i++) {
 551       fprintf(stderr, "%d ", alphaenc1[i]);
 552    }
 553    fprintf(stderr, "cutVals ");
 554    for (i = 0; i < 8; i++) {
 555       fprintf(stderr, "%d ", acutValues[i]);
 556    }
 557    fprintf(stderr, "srcVals ");
 558    for (j = 0; j < numypixels; j++) {
 559       for (i = 0; i < numxpixels; i++) {
 560          fprintf(stderr, "%d ", srccolors[j][i]);
 561       }
 562    }
 563    fprintf(stderr, "\n");
 564 #endif
 565
 566    /* it's not very likely this encoding is better if both alphaabsmin and alphaabsmax
 567       are false but try it anyway */
 568    if (alphablockerror1 >= 32) {
 569
 570       /* don't bother if encoding is already very good, this condition should also imply
 571       we have valid alphabase colors which we absolutely need (alphabase[0] <= alphabase[1]) */
 572       alphablockerror2 = 0;
 573       for (aindex = 0; aindex < 5; aindex++) {
 574          /* don't forget here is always rounded down */
 575          acutValues[aindex] = (alphabase[0] * (10 - (2*aindex + 1)) + alphabase[1] * (2*aindex + 1)) / 10;
 576       }
 577       for (j = 0; j < numypixels; j++) {
 578          for (i = 0; i < numxpixels; i++) {
 579              /* maybe it's overkill to have the most complicated calculation just for the error
 580                calculation which we only need to figure out if encoding1 or encoding2 is better... */
 581             if (srccolors[j][i] == 0) {
 582                alphaenc2[4*j + i] = 6;
 583                alphadist = 0;
 584             }
 585             else if (srccolors[j][i] == 255) {
 586                alphaenc2[4*j + i] = 7;
 587                alphadist = 0;
 588             }
 589             else if (srccolors[j][i] <= acutValues[0]) {
 590                alphaenc2[4*j + i] = 0;
 591                alphadist = srccolors[j][i] - alphabase[0];
 592             }
 593             else if (srccolors[j][i] <= acutValues[1]) {
 594                alphaenc2[4*j + i] = 2;
 595                alphadist = srccolors[j][i] - (alphabase[0] * 4 + alphabase[1] * 1) / 5;
 596             }
 597             else if (srccolors[j][i] <= acutValues[2]) {
 598                alphaenc2[4*j + i] = 3;
 599                alphadist = srccolors[j][i] - (alphabase[0] * 3 + alphabase[1] * 2) / 5;
 600             }
 601             else if (srccolors[j][i] <= acutValues[3]) {
 602                alphaenc2[4*j + i] = 4;
 603                alphadist = srccolors[j][i] - (alphabase[0] * 2 + alphabase[1] * 3) / 5;
 604             }
 605             else if (srccolors[j][i] <= acutValues[4]) {
 606                alphaenc2[4*j + i] = 5;
 607                alphadist = srccolors[j][i] - (alphabase[0] * 1 + alphabase[1] * 4) / 5;
 608             }
 609             else {
 610                alphaenc2[4*j + i] = 1;
 611                alphadist = srccolors[j][i] - alphabase[1];
 612             }
 613             alphablockerror2 += alphadist * alphadist;
 614          }
 615       }
 616
 617
 618       /* skip this if the error is already very small
 619          this encoding is MUCH better on average than #2 though, but expensive! */
 620       if ((alphablockerror2 > 96) && (alphablockerror1 > 96)) {
 621          GLshort blockerrlin1 = 0;
 622          GLshort blockerrlin2 = 0;
 623          GLubyte nralphainrangelow = 0;
 624          GLubyte nralphainrangehigh = 0;
 625          alphatest[0] = 0xff;
 626          alphatest[1] = 0x0;
 627          /* if we have large range it's likely there are values close to 0/255, try to map them to 0/255 */
 628          for (j = 0; j < numypixels; j++) {
 629             for (i = 0; i < numxpixels; i++) {
 630                if ((srccolors[j][i] > alphatest[1]) && (srccolors[j][i] < (255 -(alphabase[1] - alphabase[0]) / 28)))
 631                   alphatest[1] = srccolors[j][i];
 632                if ((srccolors[j][i] < alphatest[0]) && (srccolors[j][i] > (alphabase[1] - alphabase[0]) / 28))
 633                   alphatest[0] = srccolors[j][i];
 634             }
 635          }
 636           /* shouldn't happen too often, don't really care about those degenerated cases */
 637           if (alphatest[1] <= alphatest[0]) {
 638              alphatest[0] = 1;
 639              alphatest[1] = 254;
 640          }
 641          for (aindex = 0; aindex < 5; aindex++) {
 642          /* don't forget here is always rounded down */
 643             acutValues[aindex] = (alphatest[0] * (10 - (2*aindex + 1)) + alphatest[1] * (2*aindex + 1)) / 10;
 644          }
 645
 646          /* find the "average" difference between the alpha values and the next encoded value.
 647             This is then used to calculate new base values.
 648             Should there be some weighting, i.e. those values closer to alphatest[x] have more weight,
 649             since they will see more improvement, and also because the values in the middle are somewhat
 650             likely to get no improvement at all (because the base values might move in different directions)?
 651             OTOH it would mean the values in the middle are even less likely to get an improvement
 652          */
 653          for (j = 0; j < numypixels; j++) {
 654             for (i = 0; i < numxpixels; i++) {
 655                if (srccolors[j][i] <= alphatest[0] / 2) {
 656                }
 657                else if (srccolors[j][i] > ((255 + alphatest[1]) / 2)) {
 658                }
 659                else if (srccolors[j][i] <= acutValues[0]) {
 660                   blockerrlin1 += (srccolors[j][i] - alphatest[0]);
 661                   nralphainrangelow += 1;
 662                }
 663                else if (srccolors[j][i] <= acutValues[1]) {
 664                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5);
 665                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5);
 666                   nralphainrangelow += 1;
 667                   nralphainrangehigh += 1;
 668                }
 669                else if (srccolors[j][i] <= acutValues[2]) {
 670                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5);
 671                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5);
 672                   nralphainrangelow += 1;
 673                   nralphainrangehigh += 1;
 674                }
 675                else if (srccolors[j][i] <= acutValues[3]) {
 676                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5);
 677                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5);
 678                   nralphainrangelow += 1;
 679                   nralphainrangehigh += 1;
 680                }
 681                else if (srccolors[j][i] <= acutValues[4]) {
 682                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5);
 683                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5);
 684                   nralphainrangelow += 1;
 685                   nralphainrangehigh += 1;
 686                   }
 687                else {
 688                   blockerrlin2 += (srccolors[j][i] - alphatest[1]);
 689                   nralphainrangehigh += 1;
 690                }
 691             }
 692          }
 693          /* shouldn't happen often, needed to avoid div by zero */
 694          if (nralphainrangelow == 0) nralphainrangelow = 1;
 695          if (nralphainrangehigh == 0) nralphainrangehigh = 1;
 696          alphatest[0] = alphatest[0] + (blockerrlin1 / nralphainrangelow);
 697 #if RGTC_DEBUG
 698          fprintf(stderr, "block err lin low %d, nr %d\n", blockerrlin1, nralphainrangelow);
 699          fprintf(stderr, "block err lin high %d, nr %d\n", blockerrlin2, nralphainrangehigh);
 700 #endif
 701          /* again shouldn't really happen often... */
 702          if (alphatest[0] < 0) {
 703             alphatest[0] = 0;
 704          }
 705          alphatest[1] = alphatest[1] + (blockerrlin2 / nralphainrangehigh);
 706          if (alphatest[1] > 255) {
 707             alphatest[1] = 255;
 708          }
 709
 710          alphablockerror3 = 0;
 711          for (aindex = 0; aindex < 5; aindex++) {
 712          /* don't forget here is always rounded down */
 713             acutValues[aindex] = (alphatest[0] * (10 - (2*aindex + 1)) + alphatest[1] * (2*aindex + 1)) / 10;
 714          }
 715          for (j = 0; j < numypixels; j++) {
 716             for (i = 0; i < numxpixels; i++) {
 717                 /* maybe it's overkill to have the most complicated calculation just for the error
 718                   calculation which we only need to figure out if encoding1 or encoding2 is better... */
 719                if (srccolors[j][i] <= alphatest[0] / 2) {
 720                   alphaenc3[4*j + i] = 6;
 721                   alphadist = srccolors[j][i];
 722                }
 723                else if (srccolors[j][i] > ((255 + alphatest[1]) / 2)) {
 724                   alphaenc3[4*j + i] = 7;
 725                   alphadist = 255 - srccolors[j][i];
 726                }
 727                else if (srccolors[j][i] <= acutValues[0]) {
 728                   alphaenc3[4*j + i] = 0;
 729                   alphadist = srccolors[j][i] - alphatest[0];
 730                }
 731                else if (srccolors[j][i] <= acutValues[1]) {
 732                  alphaenc3[4*j + i] = 2;
 733                  alphadist = srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5;
 734                }
 735                else if (srccolors[j][i] <= acutValues[2]) {
 736                   alphaenc3[4*j + i] = 3;
 737                   alphadist = srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5;
 738                }
 739                else if (srccolors[j][i] <= acutValues[3]) {
 740                   alphaenc3[4*j + i] = 4;
 741                   alphadist = srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5;
 742                }
 743                else if (srccolors[j][i] <= acutValues[4]) {
 744                   alphaenc3[4*j + i] = 5;
 745                   alphadist = srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5;
 746                }
 747                else {
 748                   alphaenc3[4*j + i] = 1;
 749                   alphadist = srccolors[j][i] - alphatest[1];
 750                }
 751                alphablockerror3 += alphadist * alphadist;
 752             }
 753          }
 754       }
 755    }
 756   /* write the alpha values and encoding back. */
 757    if ((alphablockerror1 <= alphablockerror2) && (alphablockerror1 <= alphablockerror3)) {
 758 #if RGTC_DEBUG
 759       if (alphablockerror1 > 96) fprintf(stderr, "enc1 used, error %d\n", alphablockerror1);
 760 #endif
 761       write_rgtc_encoded_channel( blkaddr, alphause[1], alphause[0], alphaenc1 );
 762    }
 763    else if (alphablockerror2 <= alphablockerror3) {
 764 #if RGTC_DEBUG
 765       if (alphablockerror2 > 96) fprintf(stderr, "enc2 used, error %d\n", alphablockerror2);
 766 #endif
 767       write_rgtc_encoded_channel( blkaddr, alphabase[0], alphabase[1], alphaenc2 );
 768    }
 769    else {
 770 #if RGTC_DEBUG
 771       fprintf(stderr, "enc3 used, error %d\n", alphablockerror3);
 772 #endif
 773       write_rgtc_encoded_channel( blkaddr, (GLubyte)alphatest[0], (GLubyte)alphatest[1], alphaenc3 );
 774    }
 775 }
 776
 777
 778 static void write_rgtc_encoded_channel_s(GLbyte *blkaddr,
 779                                          GLbyte alphabase1,
 780                                          GLbyte alphabase2,
 781                                          GLbyte alphaenc[16])
 782 {
 783    *blkaddr++ = alphabase1;
 784    *blkaddr++ = alphabase2;
 785    *blkaddr++ = alphaenc[0] | (alphaenc[1] << 3) | ((alphaenc[2] & 3) << 6);
 786    *blkaddr++ = (alphaenc[2] >> 2) | (alphaenc[3] << 1) | (alphaenc[4] << 4) | ((alphaenc[5] & 1) << 7);
 787    *blkaddr++ = (alphaenc[5] >> 1) | (alphaenc[6] << 2) | (alphaenc[7] << 5);
 788    *blkaddr++ = alphaenc[8] | (alphaenc[9] << 3) | ((alphaenc[10] & 3) << 6);
 789    *blkaddr++ = (alphaenc[10] >> 2) | (alphaenc[11] << 1) | (alphaenc[12] << 4) | ((alphaenc[13] & 1) << 7);
 790    *blkaddr++ = (alphaenc[13] >> 1) | (alphaenc[14] << 2) | (alphaenc[15] << 5);
 791 }
 792
 793 static void encode_rgtc_chan_s(GLbyte *blkaddr, GLbyte srccolors[4][4],
 794                                GLint numxpixels, GLint numypixels)
 795 {
 796    GLbyte alphabase[2], alphause[2];
 797    GLshort alphatest[2] = { 0 };
 798    GLuint alphablockerror1, alphablockerror2, alphablockerror3;
 799    GLbyte i, j, aindex, acutValues[7];
 800    GLbyte alphaenc1[16], alphaenc2[16], alphaenc3[16];
 801    GLboolean alphaabsmin = GL_FALSE;
 802    GLboolean alphaabsmax = GL_FALSE;
 803    GLshort alphadist;
 804
 805    /* find lowest and highest alpha value in block, alphabase[0] lowest, alphabase[1] highest */
 806    alphabase[0] = 0xff; alphabase[1] = 0x0;
 807    for (j = 0; j < numypixels; j++) {
 808       for (i = 0; i < numxpixels; i++) {
 809          if (srccolors[j][i] == 0)
 810             alphaabsmin = GL_TRUE;
 811          else if (srccolors[j][i] == 255)
 812             alphaabsmax = GL_TRUE;
 813          else {
 814             if (srccolors[j][i] > alphabase[1])
 815                alphabase[1] = srccolors[j][i];
 816             if (srccolors[j][i] < alphabase[0])
 817                alphabase[0] = srccolors[j][i];
 818          }
 819       }
 820    }
 821
 822
 823    if ((alphabase[0] > alphabase[1]) && !(alphaabsmin && alphaabsmax)) { /* one color, either max or min */
 824       /* shortcut here since it is a very common case (and also avoids later problems) */
 825       /* || (alphabase[0] == alphabase[1] && !alphaabsmin && !alphaabsmax) */
 826       /* could also thest for alpha0 == alpha1 (and not min/max), but probably not common, so don't bother */
 827
 828       *blkaddr++ = srccolors[0][0];
 829       blkaddr++;
 830       *blkaddr++ = 0;
 831       *blkaddr++ = 0;
 832       *blkaddr++ = 0;
 833       *blkaddr++ = 0;
 834       *blkaddr++ = 0;
 835       *blkaddr++ = 0;
 836 #if RGTC_DEBUG
 837       fprintf(stderr, "enc0 used\n");
 838 #endif
 839       return;
 840    }
 841
 842    /* find best encoding for alpha0 > alpha1 */
 843    /* it's possible this encoding is better even if both alphaabsmin and alphaabsmax are true */
 844    alphablockerror1 = 0x0;
 845    alphablockerror2 = 0xffffffff;
 846    alphablockerror3 = 0xffffffff;
 847    if (alphaabsmin) alphause[0] = 0;
 848    else alphause[0] = alphabase[0];
 849    if (alphaabsmax) alphause[1] = 255;
 850    else alphause[1] = alphabase[1];
 851    /* calculate the 7 cut values, just the middle between 2 of the computed alpha values */
 852    for (aindex = 0; aindex < 7; aindex++) {
 853       /* don't forget here is always rounded down */
 854       acutValues[aindex] = (alphause[0] * (2*aindex + 1) + alphause[1] * (14 - (2*aindex + 1))) / 14;
 855    }
 856
 857    for (j = 0; j < numypixels; j++) {
 858       for (i = 0; i < numxpixels; i++) {
 859          /* maybe it's overkill to have the most complicated calculation just for the error
 860             calculation which we only need to figure out if encoding1 or encoding2 is better... */
 861          if (srccolors[j][i] > acutValues[0]) {
 862             alphaenc1[4*j + i] = 0;
 863             alphadist = srccolors[j][i] - alphause[1];
 864          }
 865          else if (srccolors[j][i] > acutValues[1]) {
 866             alphaenc1[4*j + i] = 2;
 867             alphadist = srccolors[j][i] - (alphause[1] * 6 + alphause[0] * 1) / 7;
 868          }
 869          else if (srccolors[j][i] > acutValues[2]) {
 870             alphaenc1[4*j + i] = 3;
 871             alphadist = srccolors[j][i] - (alphause[1] * 5 + alphause[0] * 2) / 7;
 872          }
 873          else if (srccolors[j][i] > acutValues[3]) {
 874             alphaenc1[4*j + i] = 4;
 875             alphadist = srccolors[j][i] - (alphause[1] * 4 + alphause[0] * 3) / 7;
 876          }
 877          else if (srccolors[j][i] > acutValues[4]) {
 878             alphaenc1[4*j + i] = 5;
 879             alphadist = srccolors[j][i] - (alphause[1] * 3 + alphause[0] * 4) / 7;
 880          }
 881          else if (srccolors[j][i] > acutValues[5]) {
 882             alphaenc1[4*j + i] = 6;
 883             alphadist = srccolors[j][i] - (alphause[1] * 2 + alphause[0] * 5) / 7;
 884          }
 885          else if (srccolors[j][i] > acutValues[6]) {
 886             alphaenc1[4*j + i] = 7;
 887             alphadist = srccolors[j][i] - (alphause[1] * 1 + alphause[0] * 6) / 7;
 888          }
 889          else {
 890             alphaenc1[4*j + i] = 1;
 891             alphadist = srccolors[j][i] - alphause[0];
 892          }
 893          alphablockerror1 += alphadist * alphadist;
 894       }
 895    }
 896 #if RGTC_DEBUG
 897    for (i = 0; i < 16; i++) {
 898       fprintf(stderr, "%d ", alphaenc1[i]);
 899    }
 900    fprintf(stderr, "cutVals ");
 901    for (i = 0; i < 8; i++) {
 902       fprintf(stderr, "%d ", acutValues[i]);
 903    }
 904    fprintf(stderr, "srcVals ");
 905    for (j = 0; j < numypixels; j++)
 906       for (i = 0; i < numxpixels; i++) {
 907          fprintf(stderr, "%d ", srccolors[j][i]);
 908       }
 909
 910    fprintf(stderr, "\n");
 911 #endif
 912
 913    /* it's not very likely this encoding is better if both alphaabsmin and alphaabsmax
 914       are false but try it anyway */
 915    if (alphablockerror1 >= 32) {
 916
 917       /* don't bother if encoding is already very good, this condition should also imply
 918       we have valid alphabase colors which we absolutely need (alphabase[0] <= alphabase[1]) */
 919       alphablockerror2 = 0;
 920       for (aindex = 0; aindex < 5; aindex++) {
 921          /* don't forget here is always rounded down */
 922          acutValues[aindex] = (alphabase[0] * (10 - (2*aindex + 1)) + alphabase[1] * (2*aindex + 1)) / 10;
 923       }
 924       for (j = 0; j < numypixels; j++) {
 925          for (i = 0; i < numxpixels; i++) {
 926              /* maybe it's overkill to have the most complicated calculation just for the error
 927                calculation which we only need to figure out if encoding1 or encoding2 is better... */
 928             if (srccolors[j][i] == 0) {
 929                alphaenc2[4*j + i] = 6;
 930                alphadist = 0;
 931             }
 932             else if (srccolors[j][i] == 255) {
 933                alphaenc2[4*j + i] = 7;
 934                alphadist = 0;
 935             }
 936             else if (srccolors[j][i] <= acutValues[0]) {
 937                alphaenc2[4*j + i] = 0;
 938                alphadist = srccolors[j][i] - alphabase[0];
 939             }
 940             else if (srccolors[j][i] <= acutValues[1]) {
 941                alphaenc2[4*j + i] = 2;
 942                alphadist = srccolors[j][i] - (alphabase[0] * 4 + alphabase[1] * 1) / 5;
 943             }
 944             else if (srccolors[j][i] <= acutValues[2]) {
 945                alphaenc2[4*j + i] = 3;
 946                alphadist = srccolors[j][i] - (alphabase[0] * 3 + alphabase[1] * 2) / 5;
 947             }
 948             else if (srccolors[j][i] <= acutValues[3]) {
 949                alphaenc2[4*j + i] = 4;
 950                alphadist = srccolors[j][i] - (alphabase[0] * 2 + alphabase[1] * 3) / 5;
 951             }
 952             else if (srccolors[j][i] <= acutValues[4]) {
 953                alphaenc2[4*j + i] = 5;
 954                alphadist = srccolors[j][i] - (alphabase[0] * 1 + alphabase[1] * 4) / 5;
 955             }
 956             else {
 957                alphaenc2[4*j + i] = 1;
 958                alphadist = srccolors[j][i] - alphabase[1];
 959             }
 960             alphablockerror2 += alphadist * alphadist;
 961          }
 962       }
 963
 964
 965       /* skip this if the error is already very small
 966          this encoding is MUCH better on average than #2 though, but expensive! */
 967       if ((alphablockerror2 > 96) && (alphablockerror1 > 96)) {
 968          GLshort blockerrlin1 = 0;
 969          GLshort blockerrlin2 = 0;
 970          GLubyte nralphainrangelow = 0;
 971          GLubyte nralphainrangehigh = 0;
 972          alphatest[0] = 0xff;
 973          alphatest[1] = 0x0;
 974          /* if we have large range it's likely there are values close to 0/255, try to map them to 0/255 */
 975          for (j = 0; j < numypixels; j++) {
 976             for (i = 0; i < numxpixels; i++) {
 977                if ((srccolors[j][i] > alphatest[1]) && (srccolors[j][i] < (255 -(alphabase[1] - alphabase[0]) / 28)))
 978                   alphatest[1] = srccolors[j][i];
 979                if ((srccolors[j][i] < alphatest[0]) && (srccolors[j][i] > (alphabase[1] - alphabase[0]) / 28))
 980                   alphatest[0] = srccolors[j][i];
 981             }
 982          }
 983           /* shouldn't happen too often, don't really care about those degenerated cases */
 984           if (alphatest[1] <= alphatest[0]) {
 985              alphatest[0] = 1;
 986              alphatest[1] = 254;
 987          }
 988          for (aindex = 0; aindex < 5; aindex++) {
 989          /* don't forget here is always rounded down */
 990             acutValues[aindex] = (alphatest[0] * (10 - (2*aindex + 1)) + alphatest[1] * (2*aindex + 1)) / 10;
 991          }
 992
 993          /* find the "average" difference between the alpha values and the next encoded value.
 994             This is then used to calculate new base values.
 995             Should there be some weighting, i.e. those values closer to alphatest[x] have more weight,
 996             since they will see more improvement, and also because the values in the middle are somewhat
 997             likely to get no improvement at all (because the base values might move in different directions)?
 998             OTOH it would mean the values in the middle are even less likely to get an improvement
 999          */
1000          for (j = 0; j < numypixels; j++) {
1001             for (i = 0; i < numxpixels; i++) {
1002                if (srccolors[j][i] <= alphatest[0] / 2) {
1003                }
1004                else if (srccolors[j][i] > ((255 + alphatest[1]) / 2)) {
1005                }
1006                else if (srccolors[j][i] <= acutValues[0]) {
1007                   blockerrlin1 += (srccolors[j][i] - alphatest[0]);
1008                   nralphainrangelow += 1;
1009                }
1010                else if (srccolors[j][i] <= acutValues[1]) {
1011                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5);
1012                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5);
1013                   nralphainrangelow += 1;
1014                   nralphainrangehigh += 1;
1015                }
1016                else if (srccolors[j][i] <= acutValues[2]) {
1017                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5);
1018                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5);
1019                   nralphainrangelow += 1;
1020                   nralphainrangehigh += 1;
1021                }
1022                else if (srccolors[j][i] <= acutValues[3]) {
1023                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5);
1024                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5);
1025                   nralphainrangelow += 1;
1026                   nralphainrangehigh += 1;
1027                }
1028                else if (srccolors[j][i] <= acutValues[4]) {
1029                   blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5);
1030                   blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5);
1031                   nralphainrangelow += 1;
1032                   nralphainrangehigh += 1;
1033                   }
1034                else {
1035                   blockerrlin2 += (srccolors[j][i] - alphatest[1]);
1036                   nralphainrangehigh += 1;
1037                }
1038             }
1039          }
1040          /* shouldn't happen often, needed to avoid div by zero */
1041          if (nralphainrangelow == 0) nralphainrangelow = 1;
1042          if (nralphainrangehigh == 0) nralphainrangehigh = 1;
1043          alphatest[0] = alphatest[0] + (blockerrlin1 / nralphainrangelow);
1044 #if RGTC_DEBUG
1045          fprintf(stderr, "block err lin low %d, nr %d\n", blockerrlin1, nralphainrangelow);
1046          fprintf(stderr, "block err lin high %d, nr %d\n", blockerrlin2, nralphainrangehigh);
1047 #endif
1048          /* again shouldn't really happen often... */
1049          if (alphatest[0] < 0) {
1050             alphatest[0] = 0;
1051          }
1052          alphatest[1] = alphatest[1] + (blockerrlin2 / nralphainrangehigh);
1053          if (alphatest[1] > 255) {
1054             alphatest[1] = 255;
1055          }
1056
1057          alphablockerror3 = 0;
1058          for (aindex = 0; aindex < 5; aindex++) {
1059          /* don't forget here is always rounded down */
1060             acutValues[aindex] = (alphatest[0] * (10 - (2*aindex + 1)) + alphatest[1] * (2*aindex + 1)) / 10;
1061          }
1062          for (j = 0; j < numypixels; j++) {
1063             for (i = 0; i < numxpixels; i++) {
1064                 /* maybe it's overkill to have the most complicated calculation just for the error
1065                   calculation which we only need to figure out if encoding1 or encoding2 is better... */
1066                if (srccolors[j][i] <= alphatest[0] / 2) {
1067                   alphaenc3[4*j + i] = 6;
1068                   alphadist = srccolors[j][i];
1069                }
1070                else if (srccolors[j][i] > ((255 + alphatest[1]) / 2)) {
1071                   alphaenc3[4*j + i] = 7;
1072                   alphadist = 255 - srccolors[j][i];
1073                }
1074                else if (srccolors[j][i] <= acutValues[0]) {
1075                   alphaenc3[4*j + i] = 0;
1076                   alphadist = srccolors[j][i] - alphatest[0];
1077                }
1078                else if (srccolors[j][i] <= acutValues[1]) {
1079                  alphaenc3[4*j + i] = 2;
1080                  alphadist = srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5;
1081                }
1082                else if (srccolors[j][i] <= acutValues[2]) {
1083                   alphaenc3[4*j + i] = 3;
1084                   alphadist = srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5;
1085                }
1086                else if (srccolors[j][i] <= acutValues[3]) {
1087                   alphaenc3[4*j + i] = 4;
1088                   alphadist = srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5;
1089                }
1090                else if (srccolors[j][i] <= acutValues[4]) {
1091                   alphaenc3[4*j + i] = 5;
1092                   alphadist = srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5;
1093                }
1094                else {
1095                   alphaenc3[4*j + i] = 1;
1096                   alphadist = srccolors[j][i] - alphatest[1];
1097                }
1098                alphablockerror3 += alphadist * alphadist;
1099             }
1100          }
1101       }
1102    }
1103   /* write the alpha values and encoding back. */
1104    if ((alphablockerror1 <= alphablockerror2) && (alphablockerror1 <= alphablockerror3)) {
1105 #if RGTC_DEBUG
1106       if (alphablockerror1 > 96) fprintf(stderr, "enc1 used, error %d\n", alphablockerror1);
1107 #endif
1108       write_rgtc_encoded_channel_s( blkaddr, alphause[1], alphause[0], alphaenc1 );
1109    }
1110    else if (alphablockerror2 <= alphablockerror3) {
1111 #if RGTC_DEBUG
1112       if (alphablockerror2 > 96) fprintf(stderr, "enc2 used, error %d\n", alphablockerror2);
1113 #endif
1114       write_rgtc_encoded_channel_s( blkaddr, alphabase[0], alphabase[1], alphaenc2 );
1115    }
1116    else {
1117 #if RGTC_DEBUG
1118       fprintf(stderr, "enc3 used, error %d\n", alphablockerror3);
1119 #endif
1120       write_rgtc_encoded_channel_s( blkaddr, (GLubyte)alphatest[0], (GLubyte)alphatest[1], alphaenc3 );
1121    }
1122 }