-#if RGTC_DEBUG
- for (i = 0; i < 16; i++) {
- fprintf(stderr, "%d ", alphaenc1[i]);
- }
- fprintf(stderr, "cutVals ");
- for (i = 0; i < 8; i++) {
- fprintf(stderr, "%d ", acutValues[i]);
- }
- fprintf(stderr, "srcVals ");
- for (j = 0; j < numypixels; j++) {
- for (i = 0; i < numxpixels; i++) {
- fprintf(stderr, "%d ", srccolors[j][i]);
- }
- }
- fprintf(stderr, "\n");
-#endif
-
- /* it's not very likely this encoding is better if both alphaabsmin and alphaabsmax
- are false but try it anyway */
- if (alphablockerror1 >= 32) {
-
- /* don't bother if encoding is already very good, this condition should also imply
- we have valid alphabase colors which we absolutely need (alphabase[0] <= alphabase[1]) */
- alphablockerror2 = 0;
- for (aindex = 0; aindex < 5; aindex++) {
- /* don't forget here is always rounded down */
- acutValues[aindex] = (alphabase[0] * (10 - (2*aindex + 1)) + alphabase[1] * (2*aindex + 1)) / 10;
- }
- for (j = 0; j < numypixels; j++) {
- for (i = 0; i < numxpixels; i++) {
- /* maybe it's overkill to have the most complicated calculation just for the error
- calculation which we only need to figure out if encoding1 or encoding2 is better... */
- if (srccolors[j][i] == 0) {
- alphaenc2[4*j + i] = 6;
- alphadist = 0;
- }
- else if (srccolors[j][i] == 255) {
- alphaenc2[4*j + i] = 7;
- alphadist = 0;
- }
- else if (srccolors[j][i] <= acutValues[0]) {
- alphaenc2[4*j + i] = 0;
- alphadist = srccolors[j][i] - alphabase[0];
- }
- else if (srccolors[j][i] <= acutValues[1]) {
- alphaenc2[4*j + i] = 2;
- alphadist = srccolors[j][i] - (alphabase[0] * 4 + alphabase[1] * 1) / 5;
- }
- else if (srccolors[j][i] <= acutValues[2]) {
- alphaenc2[4*j + i] = 3;
- alphadist = srccolors[j][i] - (alphabase[0] * 3 + alphabase[1] * 2) / 5;
- }
- else if (srccolors[j][i] <= acutValues[3]) {
- alphaenc2[4*j + i] = 4;
- alphadist = srccolors[j][i] - (alphabase[0] * 2 + alphabase[1] * 3) / 5;
- }
- else if (srccolors[j][i] <= acutValues[4]) {
- alphaenc2[4*j + i] = 5;
- alphadist = srccolors[j][i] - (alphabase[0] * 1 + alphabase[1] * 4) / 5;
- }
- else {
- alphaenc2[4*j + i] = 1;
- alphadist = srccolors[j][i] - alphabase[1];
- }
- alphablockerror2 += alphadist * alphadist;
- }
- }
-
-
- /* skip this if the error is already very small
- this encoding is MUCH better on average than #2 though, but expensive! */
- if ((alphablockerror2 > 96) && (alphablockerror1 > 96)) {
- GLshort blockerrlin1 = 0;
- GLshort blockerrlin2 = 0;
- GLubyte nralphainrangelow = 0;
- GLubyte nralphainrangehigh = 0;
- alphatest[0] = 0xff;
- alphatest[1] = 0x0;
- /* if we have large range it's likely there are values close to 0/255, try to map them to 0/255 */
- for (j = 0; j < numypixels; j++) {
- for (i = 0; i < numxpixels; i++) {
- if ((srccolors[j][i] > alphatest[1]) && (srccolors[j][i] < (255 -(alphabase[1] - alphabase[0]) / 28)))
- alphatest[1] = srccolors[j][i];
- if ((srccolors[j][i] < alphatest[0]) && (srccolors[j][i] > (alphabase[1] - alphabase[0]) / 28))
- alphatest[0] = srccolors[j][i];
- }
- }
- /* shouldn't happen too often, don't really care about those degenerated cases */
- if (alphatest[1] <= alphatest[0]) {
- alphatest[0] = 1;
- alphatest[1] = 254;
- }
- for (aindex = 0; aindex < 5; aindex++) {
- /* don't forget here is always rounded down */
- acutValues[aindex] = (alphatest[0] * (10 - (2*aindex + 1)) + alphatest[1] * (2*aindex + 1)) / 10;
- }
-
- /* find the "average" difference between the alpha values and the next encoded value.
- This is then used to calculate new base values.
- Should there be some weighting, i.e. those values closer to alphatest[x] have more weight,
- since they will see more improvement, and also because the values in the middle are somewhat
- likely to get no improvement at all (because the base values might move in different directions)?
- OTOH it would mean the values in the middle are even less likely to get an improvement
- */
- for (j = 0; j < numypixels; j++) {
- for (i = 0; i < numxpixels; i++) {
- if (srccolors[j][i] <= alphatest[0] / 2) {
- }
- else if (srccolors[j][i] > ((255 + alphatest[1]) / 2)) {
- }
- else if (srccolors[j][i] <= acutValues[0]) {
- blockerrlin1 += (srccolors[j][i] - alphatest[0]);
- nralphainrangelow += 1;
- }
- else if (srccolors[j][i] <= acutValues[1]) {
- blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5);
- blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5);
- nralphainrangelow += 1;
- nralphainrangehigh += 1;
- }
- else if (srccolors[j][i] <= acutValues[2]) {
- blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5);
- blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5);
- nralphainrangelow += 1;
- nralphainrangehigh += 1;
- }
- else if (srccolors[j][i] <= acutValues[3]) {
- blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5);
- blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5);
- nralphainrangelow += 1;
- nralphainrangehigh += 1;
- }
- else if (srccolors[j][i] <= acutValues[4]) {
- blockerrlin1 += (srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5);
- blockerrlin2 += (srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5);
- nralphainrangelow += 1;
- nralphainrangehigh += 1;
- }
- else {
- blockerrlin2 += (srccolors[j][i] - alphatest[1]);
- nralphainrangehigh += 1;
- }
- }
- }
- /* shouldn't happen often, needed to avoid div by zero */
- if (nralphainrangelow == 0) nralphainrangelow = 1;
- if (nralphainrangehigh == 0) nralphainrangehigh = 1;
- alphatest[0] = alphatest[0] + (blockerrlin1 / nralphainrangelow);
-#if RGTC_DEBUG
- fprintf(stderr, "block err lin low %d, nr %d\n", blockerrlin1, nralphainrangelow);
- fprintf(stderr, "block err lin high %d, nr %d\n", blockerrlin2, nralphainrangehigh);
-#endif
- /* again shouldn't really happen often... */
- if (alphatest[0] < 0) {
- alphatest[0] = 0;
- }
- alphatest[1] = alphatest[1] + (blockerrlin2 / nralphainrangehigh);
- if (alphatest[1] > 255) {
- alphatest[1] = 255;
- }
-
- alphablockerror3 = 0;
- for (aindex = 0; aindex < 5; aindex++) {
- /* don't forget here is always rounded down */
- acutValues[aindex] = (alphatest[0] * (10 - (2*aindex + 1)) + alphatest[1] * (2*aindex + 1)) / 10;
- }
- for (j = 0; j < numypixels; j++) {
- for (i = 0; i < numxpixels; i++) {
- /* maybe it's overkill to have the most complicated calculation just for the error
- calculation which we only need to figure out if encoding1 or encoding2 is better... */
- if (srccolors[j][i] <= alphatest[0] / 2) {
- alphaenc3[4*j + i] = 6;
- alphadist = srccolors[j][i];
- }
- else if (srccolors[j][i] > ((255 + alphatest[1]) / 2)) {
- alphaenc3[4*j + i] = 7;
- alphadist = 255 - srccolors[j][i];
- }
- else if (srccolors[j][i] <= acutValues[0]) {
- alphaenc3[4*j + i] = 0;
- alphadist = srccolors[j][i] - alphatest[0];
- }
- else if (srccolors[j][i] <= acutValues[1]) {
- alphaenc3[4*j + i] = 2;
- alphadist = srccolors[j][i] - (alphatest[0] * 4 + alphatest[1] * 1) / 5;
- }
- else if (srccolors[j][i] <= acutValues[2]) {
- alphaenc3[4*j + i] = 3;
- alphadist = srccolors[j][i] - (alphatest[0] * 3 + alphatest[1] * 2) / 5;
- }
- else if (srccolors[j][i] <= acutValues[3]) {
- alphaenc3[4*j + i] = 4;
- alphadist = srccolors[j][i] - (alphatest[0] * 2 + alphatest[1] * 3) / 5;
- }
- else if (srccolors[j][i] <= acutValues[4]) {
- alphaenc3[4*j + i] = 5;
- alphadist = srccolors[j][i] - (alphatest[0] * 1 + alphatest[1] * 4) / 5;
- }
- else {
- alphaenc3[4*j + i] = 1;
- alphadist = srccolors[j][i] - alphatest[1];
- }
- alphablockerror3 += alphadist * alphadist;
- }
- }
- }
- }
- /* write the alpha values and encoding back. */
- if ((alphablockerror1 <= alphablockerror2) && (alphablockerror1 <= alphablockerror3)) {
-#if RGTC_DEBUG
- if (alphablockerror1 > 96) fprintf(stderr, "enc1 used, error %d\n", alphablockerror1);
-#endif
- write_rgtc_encoded_channel( blkaddr, alphause[1], alphause[0], alphaenc1 );
- }
- else if (alphablockerror2 <= alphablockerror3) {
-#if RGTC_DEBUG
- if (alphablockerror2 > 96) fprintf(stderr, "enc2 used, error %d\n", alphablockerror2);
-#endif
- write_rgtc_encoded_channel( blkaddr, alphabase[0], alphabase[1], alphaenc2 );
- }
- else {
-#if RGTC_DEBUG
- fprintf(stderr, "enc3 used, error %d\n", alphablockerror3);
-#endif
- write_rgtc_encoded_channel( blkaddr, (GLubyte)alphatest[0], (GLubyte)alphatest[1], alphaenc3 );
- }