projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
swrast: Fix unsigned promotion in pointer arithmetic
[mesa.git] / src / mesa / swrast / s_stencil.c
1 /*
2 * Mesa 3-D graphics library
3 * Version: 7.1
4 *
5 * Copyright (C) 1999-2007 Brian Paul All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included
15 * in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * BRIAN PAUL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
21 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25
26 #include "main/glheader.h"
27 #include "main/context.h"
28 #include "main/imports.h"
29 #include "main/format_pack.h"
30 #include "main/format_unpack.h"
31
32 #include "s_context.h"
33 #include "s_depth.h"
34 #include "s_stencil.h"
35 #include "s_span.h"
36
37
38
39 /* Stencil Logic:
40
41 IF stencil test fails THEN
42 Apply fail-op to stencil value
43 Don't write the pixel (RGBA,Z)
44 ELSE
45 IF doing depth test && depth test fails THEN
46 Apply zfail-op to stencil value
47 Write RGBA and Z to appropriate buffers
48 ELSE
49 Apply zpass-op to stencil value
50 ENDIF
51
52 */
53
54
55
56 /**
57 * Compute/return the offset of the stencil value in a pixel.
58 * For example, if the format is Z24+S8, the position of the stencil bits
59 * within the 4-byte pixel will be either 0 or 3.
60 */
61 static GLint
62 get_stencil_offset(gl_format format)
63 {
64 const GLubyte one = 1;
65 GLubyte pixel[MAX_PIXEL_BYTES];
66 GLint bpp = _mesa_get_format_bytes(format);
67 GLint i;
68
69 assert(_mesa_get_format_bits(format, GL_STENCIL_BITS) == 8);
70 memset(pixel, 0, sizeof(pixel));
71 _mesa_pack_ubyte_stencil_row(format, 1, &one, pixel);
72
73 for (i = 0; i < bpp; i++) {
74 if (pixel[i])
75 return i;
76 }
77
78 _mesa_problem(NULL, "get_stencil_offset() failed\n");
79 return 0;
80 }
81
82
83 /** Clamp the stencil value to [0, 255] */
84 static inline GLubyte
85 clamp(GLint val)
86 {
87 if (val < 0)
88 return 0;
89 else if (val > 255)
90 return 255;
91 else
92 return val;
93 }
94
95
96 #define STENCIL_OP(NEW_VAL) \
97 if (invmask == 0) { \
98 for (i = j = 0; i < n; i++, j += stride) { \
99 if (mask[i]) { \
100 GLubyte s = stencil[j]; \
101 (void) s; \
102 stencil[j] = (GLubyte) (NEW_VAL); \
103 } \
104 } \
105 } \
106 else { \
107 for (i = j = 0; i < n; i++, j += stride) { \
108 if (mask[i]) { \
109 GLubyte s = stencil[j]; \
110 stencil[j] = (GLubyte) ((invmask & s) | (wrtmask & (NEW_VAL))); \
111 } \
112 } \
113 }
114
115
116 /**
117 * Apply the given stencil operator to the array of stencil values.
118 * Don't touch stencil[i] if mask[i] is zero.
119 * @param n number of stencil values
120 * @param oper the stencil buffer operator
121 * @param face 0 or 1 for front or back face operation
122 * @param stencil array of stencil values (in/out)
123 * @param mask array [n] of flag: 1=apply operator, 0=don't apply operator
124 * @param stride stride between stencil values
125 */
126 static void
127 apply_stencil_op(const struct gl_context *ctx, GLenum oper, GLuint face,
128 GLuint n, GLubyte stencil[], const GLubyte mask[],
129 GLint stride)
130 {
131 const GLubyte ref = ctx->Stencil.Ref[face];
132 const GLubyte wrtmask = ctx->Stencil.WriteMask[face];
133 const GLubyte invmask = (GLubyte) (~wrtmask);
134 GLuint i, j;
135
136 switch (oper) {
137 case GL_KEEP:
138 /* do nothing */
139 break;
140 case GL_ZERO:
141 /* replace stencil buf values with zero */
142 STENCIL_OP(0);
143 break;
144 case GL_REPLACE:
145 /* replace stencil buf values with ref value */
146 STENCIL_OP(ref);
147 break;
148 case GL_INCR:
149 /* increment stencil buf values, with clamping */
150 STENCIL_OP(clamp(s + 1));
151 break;
152 case GL_DECR:
153 /* increment stencil buf values, with clamping */
154 STENCIL_OP(clamp(s - 1));
155 break;
156 case GL_INCR_WRAP_EXT:
157 /* increment stencil buf values, without clamping */
158 STENCIL_OP(s + 1);
159 break;
160 case GL_DECR_WRAP_EXT:
161 /* increment stencil buf values, without clamping */
162 STENCIL_OP(s - 1);
163 break;
164 case GL_INVERT:
165 /* replace stencil buf values with inverted value */
166 STENCIL_OP(~s);
167 break;
168 default:
169 _mesa_problem(ctx, "Bad stencil op in apply_stencil_op");
170 }
171 }
172
173
174
175 #define STENCIL_TEST(FUNC) \
176 for (i = j = 0; i < n; i++, j += stride) { \
177 if (mask[i]) { \
178 s = (GLubyte) (stencil[j] & valueMask); \
179 if (FUNC) { \
180 /* stencil pass */ \
181 fail[i] = 0; \
182 } \
183 else { \
184 /* stencil fail */ \
185 fail[i] = 1; \
186 mask[i] = 0; \
187 } \
188 } \
189 else { \
190 fail[i] = 0; \
191 } \
192 }
193
194
195
196 /**
197 * Apply stencil test to an array of stencil values (before depth buffering).
198 * For the values that fail, we'll apply the GL_STENCIL_FAIL operator to
199 * the stencil values.
200 *
201 * @param face 0 or 1 for front or back-face polygons
202 * @param n number of pixels in the array
203 * @param stencil array of [n] stencil values (in/out)
204 * @param mask array [n] of flag: 0=skip the pixel, 1=stencil the pixel,
205 * values are set to zero where the stencil test fails.
206 * @param stride stride between stencil values
207 * @return GL_FALSE = all pixels failed, GL_TRUE = zero or more pixels passed.
208 */
209 static GLboolean
210 do_stencil_test(struct gl_context *ctx, GLuint face, GLuint n,
211 GLubyte stencil[], GLubyte mask[], GLint stride)
212 {
213 GLubyte fail[MAX_WIDTH];
214 GLboolean allfail = GL_FALSE;
215 GLuint i, j;
216 const GLuint valueMask = ctx->Stencil.ValueMask[face];
217 const GLubyte ref = (GLubyte) (ctx->Stencil.Ref[face] & valueMask);
218 GLubyte s;
219
220 /*
221 * Perform stencil test. The results of this operation are stored
222 * in the fail[] array:
223 * IF fail[i] is non-zero THEN
224 * the stencil fail operator is to be applied
225 * ELSE
226 * the stencil fail operator is not to be applied
227 * ENDIF
228 */
229 switch (ctx->Stencil.Function[face]) {
230 case GL_NEVER:
231 STENCIL_TEST(0);
232 allfail = GL_TRUE;
233 break;
234 case GL_LESS:
235 STENCIL_TEST(ref < s);
236 break;
237 case GL_LEQUAL:
238 STENCIL_TEST(ref <= s);
239 break;
240 case GL_GREATER:
241 STENCIL_TEST(ref > s);
242 break;
243 case GL_GEQUAL:
244 STENCIL_TEST(ref >= s);
245 break;
246 case GL_EQUAL:
247 STENCIL_TEST(ref == s);
248 break;
249 case GL_NOTEQUAL:
250 STENCIL_TEST(ref != s);
251 break;
252 case GL_ALWAYS:
253 STENCIL_TEST(1);
254 break;
255 default:
256 _mesa_problem(ctx, "Bad stencil func in gl_stencil_span");
257 return 0;
258 }
259
260 if (ctx->Stencil.FailFunc[face] != GL_KEEP) {
261 apply_stencil_op(ctx, ctx->Stencil.FailFunc[face], face, n, stencil,
262 fail, stride);
263 }
264
265 return !allfail;
266 }
267
268
269 /**
270 * Compute the zpass/zfail masks by comparing the pre- and post-depth test
271 * masks.
272 */
273 static inline void
274 compute_pass_fail_masks(GLuint n, const GLubyte origMask[],
275 const GLubyte newMask[],
276 GLubyte passMask[], GLubyte failMask[])
277 {
278 GLuint i;
279 for (i = 0; i < n; i++) {
280 ASSERT(newMask[i] == 0 || newMask[i] == 1);
281 passMask[i] = origMask[i] & newMask[i];
282 failMask[i] = origMask[i] & (newMask[i] ^ 1);
283 }
284 }
285
286
287 /**
288 * Get 8-bit stencil values from random locations in the stencil buffer.
289 */
290 static void
291 get_s8_values(struct gl_context *ctx, struct gl_renderbuffer *rb,
292 GLuint count, const GLint x[], const GLint y[],
293 GLubyte stencil[])
294 {
295 const GLint w = rb->Width, h = rb->Height;
296 const GLubyte *map = (const GLubyte *) rb->Data;
297 GLuint i;
298
299 if (rb->Format == MESA_FORMAT_S8) {
300 const GLint rowStride = rb->RowStride;
301 for (i = 0; i < count; i++) {
302 if (x[i] >= 0 && y[i] >= 0 && x[i] < w && y[i] < h) {
303 stencil[i] = *(map + y[i] * rowStride + x[i]);
304 }
305 }
306 }
307 else {
308 const GLint bpp = _mesa_get_format_bytes(rb->Format);
309 const GLint rowStride = rb->RowStride * bpp;
310 for (i = 0; i < count; i++) {
311 if (x[i] >= 0 && y[i] >= 0 && x[i] < w && y[i] < h) {
312 const GLubyte *src = map + y[i] * rowStride + x[i] * bpp;
313 _mesa_unpack_ubyte_stencil_row(rb->Format, 1, src, &stencil[i]);
314 }
315 }
316 }
317 }
318
319
320 /**
321 * Put 8-bit stencil values at random locations into the stencil buffer.
322 */
323 static void
324 put_s8_values(struct gl_context *ctx, struct gl_renderbuffer *rb,
325 GLuint count, const GLint x[], const GLint y[],
326 const GLubyte stencil[])
327 {
328 const GLint w = rb->Width, h = rb->Height;
329 GLuint i;
330
331 for (i = 0; i < count; i++) {
332 if (x[i] >= 0 && y[i] >= 0 && x[i] < w && y[i] < h) {
333 GLubyte *dst = _swrast_pixel_address(rb, x[i], y[i]);
334 _mesa_pack_ubyte_stencil_row(rb->Format, 1, &stencil[i], dst);
335 }
336 }
337 }
338
339
340 /**
341 * /return GL_TRUE = one or more fragments passed,
342 * GL_FALSE = all fragments failed.
343 */
344 GLboolean
345 _swrast_stencil_and_ztest_span(struct gl_context *ctx, SWspan *span)
346 {
347 struct gl_framebuffer *fb = ctx->DrawBuffer;
348 struct gl_renderbuffer *rb = fb->Attachment[BUFFER_STENCIL].Renderbuffer;
349 const GLint stencilOffset = get_stencil_offset(rb->Format);
350 const GLint stencilStride = _mesa_get_format_bytes(rb->Format);
351 const GLuint face = (span->facing == 0) ? 0 : ctx->Stencil._BackFace;
352 const GLuint count = span->end;
353 GLubyte *mask = span->array->mask;
354 GLubyte stencilTemp[MAX_WIDTH];
355 GLubyte *stencilBuf;
356
357 if (span->arrayMask & SPAN_XY) {
358 /* read stencil values from random locations */
359 get_s8_values(ctx, rb, count, span->array->x, span->array->y,
360 stencilTemp);
361 stencilBuf = stencilTemp;
362 }
363 else {
364 /* Processing a horizontal run of pixels. Since stencil is always
365 * 8 bits for all MESA_FORMATs, we just need to use the right offset
366 * and stride to access them.
367 */
368 stencilBuf = _swrast_pixel_address(rb, span->x, span->y) + stencilOffset;
369 }
370
371 /*
372 * Apply the stencil test to the fragments.
373 * failMask[i] is 1 if the stencil test failed.
374 */
375 if (!do_stencil_test(ctx, face, count, stencilBuf, mask, stencilStride)) {
376 /* all fragments failed the stencil test, we're done. */
377 span->writeAll = GL_FALSE;
378 if (span->arrayMask & SPAN_XY) {
379 /* need to write the updated stencil values back to the buffer */
380 put_s8_values(ctx, rb, count, span->array->x, span->array->y,
381 stencilTemp);
382 }
383 return GL_FALSE;
384 }
385
386 /*
387 * Some fragments passed the stencil test, apply depth test to them
388 * and apply Zpass and Zfail stencil ops.
389 */
390 if (ctx->Depth.Test == GL_FALSE ||
391 ctx->DrawBuffer->Attachment[BUFFER_DEPTH].Renderbuffer == NULL) {
392 /*
393 * No depth buffer, just apply zpass stencil function to active pixels.
394 */
395 apply_stencil_op(ctx, ctx->Stencil.ZPassFunc[face], face, count,
396 stencilBuf, mask, stencilStride);
397 }
398 else {
399 /*
400 * Perform depth buffering, then apply zpass or zfail stencil function.
401 */
402 GLubyte passMask[MAX_WIDTH], failMask[MAX_WIDTH], origMask[MAX_WIDTH];
403
404 /* save the current mask bits */
405 memcpy(origMask, mask, count * sizeof(GLubyte));
406
407 /* apply the depth test */
408 _swrast_depth_test_span(ctx, span);
409
410 compute_pass_fail_masks(count, origMask, mask, passMask, failMask);
411
412 /* apply the pass and fail operations */
413 if (ctx->Stencil.ZFailFunc[face] != GL_KEEP) {
414 apply_stencil_op(ctx, ctx->Stencil.ZFailFunc[face], face,
415 count, stencilBuf, failMask, stencilStride);
416 }
417 if (ctx->Stencil.ZPassFunc[face] != GL_KEEP) {
418 apply_stencil_op(ctx, ctx->Stencil.ZPassFunc[face], face,
419 count, stencilBuf, passMask, stencilStride);
420 }
421 }
422
423 /* Write updated stencil values back into hardware stencil buffer */
424 if (span->arrayMask & SPAN_XY) {
425 put_s8_values(ctx, rb, count, span->array->x, span->array->y,
426 stencilBuf);
427 }
428
429 span->writeAll = GL_FALSE;
430
431 return GL_TRUE; /* one or more fragments passed both tests */
432 }
433
434
435
436
437 /**
438 * Return a span of stencil values from the stencil buffer.
439 * Used for glRead/CopyPixels
440 * Input: n - how many pixels
441 * x,y - location of first pixel
442 * Output: stencil - the array of stencil values
443 */
444 void
445 _swrast_read_stencil_span(struct gl_context *ctx, struct gl_renderbuffer *rb,
446 GLint n, GLint x, GLint y, GLubyte stencil[])
447 {
448 GLubyte *src;
449
450 if (y < 0 || y >= (GLint) rb->Height ||
451 x + n <= 0 || x >= (GLint) rb->Width) {
452 /* span is completely outside framebuffer */
453 return; /* undefined values OK */
454 }
455
456 if (x < 0) {
457 GLint dx = -x;
458 x = 0;
459 n -= dx;
460 stencil += dx;
461 }
462 if (x + n > (GLint) rb->Width) {
463 GLint dx = x + n - rb->Width;
464 n -= dx;
465 }
466 if (n <= 0) {
467 return;
468 }
469
470 src = _swrast_pixel_address(rb, x, y);
471 _mesa_unpack_ubyte_stencil_row(rb->Format, n, src, stencil);
472 }
473
474
475
476 /**
477 * Write a span of stencil values to the stencil buffer. This function
478 * applies the stencil write mask when needed.
479 * Used for glDraw/CopyPixels
480 * Input: n - how many pixels
481 * x, y - location of first pixel
482 * stencil - the array of stencil values
483 */
484 void
485 _swrast_write_stencil_span(struct gl_context *ctx, GLint n, GLint x, GLint y,
486 const GLubyte stencil[] )
487 {
488 struct gl_framebuffer *fb = ctx->DrawBuffer;
489 struct gl_renderbuffer *rb = fb->Attachment[BUFFER_STENCIL].Renderbuffer;
490 const GLuint stencilMax = (1 << fb->Visual.stencilBits) - 1;
491 const GLuint stencilMask = ctx->Stencil.WriteMask[0];
492 GLubyte *stencilBuf;
493
494 if (y < 0 || y >= (GLint) rb->Height ||
495 x + n <= 0 || x >= (GLint) rb->Width) {
496 /* span is completely outside framebuffer */
497 return; /* undefined values OK */
498 }
499 if (x < 0) {
500 GLint dx = -x;
501 x = 0;
502 n -= dx;
503 stencil += dx;
504 }
505 if (x + n > (GLint) rb->Width) {
506 GLint dx = x + n - rb->Width;
507 n -= dx;
508 }
509 if (n <= 0) {
510 return;
511 }
512
513 stencilBuf = _swrast_pixel_address(rb, x, y);
514
515 if ((stencilMask & stencilMax) != stencilMax) {
516 /* need to apply writemask */
517 GLubyte destVals[MAX_WIDTH], newVals[MAX_WIDTH];
518 GLint i;
519
520 _mesa_unpack_ubyte_stencil_row(rb->Format, n, stencilBuf, destVals);
521 for (i = 0; i < n; i++) {
522 newVals[i]
523 = (stencil[i] & stencilMask) | (destVals[i] & ~stencilMask);
524 }
525 _mesa_pack_ubyte_stencil_row(rb->Format, n, newVals, stencilBuf);
526 }
527 else {
528 _mesa_pack_ubyte_stencil_row(rb->Format, n, stencil, stencilBuf);
529 }
530 }
531
532
533
534 /**
535 * Clear the stencil buffer. If the buffer is a combined
536 * depth+stencil buffer, only the stencil bits will be touched.
537 */
538 void
539 _swrast_clear_stencil_buffer(struct gl_context *ctx)
540 {
541 struct gl_renderbuffer *rb =
542 ctx->DrawBuffer->Attachment[BUFFER_STENCIL].Renderbuffer;
543 const GLubyte stencilBits = ctx->DrawBuffer->Visual.stencilBits;
544 const GLuint writeMask = ctx->Stencil.WriteMask[0];
545 const GLuint stencilMax = (1 << stencilBits) - 1;
546 GLint x, y, width, height;
547 GLubyte *map;
548 GLint rowStride, i, j;
549 GLbitfield mapMode;
550
551 if (!rb || writeMask == 0)
552 return;
553
554 /* compute region to clear */
555 x = ctx->DrawBuffer->_Xmin;
556 y = ctx->DrawBuffer->_Ymin;
557 width = ctx->DrawBuffer->_Xmax - ctx->DrawBuffer->_Xmin;
558 height = ctx->DrawBuffer->_Ymax - ctx->DrawBuffer->_Ymin;
559
560 mapMode = GL_MAP_WRITE_BIT;
561 if ((writeMask & stencilMax) != stencilMax) {
562 /* need to mask stencil values */
563 mapMode |= GL_MAP_READ_BIT;
564 }
565 else if (_mesa_get_format_bits(rb->Format, GL_DEPTH_BITS) > 0) {
566 /* combined depth+stencil, need to mask Z values */
567 mapMode |= GL_MAP_READ_BIT;
568 }
569
570 ctx->Driver.MapRenderbuffer(ctx, rb, x, y, width, height,
571 mapMode, &map, &rowStride);
572 if (!map) {
573 _mesa_error(ctx, GL_OUT_OF_MEMORY, "glClear(stencil)");
574 return;
575 }
576
577 switch (rb->Format) {
578 case MESA_FORMAT_S8:
579 {
580 GLubyte clear = ctx->Stencil.Clear & writeMask & 0xff;
581 GLubyte mask = (~writeMask) & 0xff;
582 if (mask != 0) {
583 /* masked clear */
584 for (i = 0; i < height; i++) {
585 GLubyte *row = map;
586 for (j = 0; j < width; j++) {
587 row[j] = (row[j] & mask) | clear;
588 }
589 map += rowStride;
590 }
591 }
592 else if (rowStride == width) {
593 /* clear whole buffer */
594 memset(map, clear, width * height);
595 }
596 else {
597 /* clear scissored */
598 for (i = 0; i < height; i++) {
599 memset(map, clear, width);
600 map += rowStride;
601 }
602 }
603 }
604 break;
605 case MESA_FORMAT_S8_Z24:
606 {
607 GLuint clear = (ctx->Stencil.Clear & writeMask & 0xff) << 24;
608 GLuint mask = (((~writeMask) & 0xff) << 24) | 0xffffff;
609 for (i = 0; i < height; i++) {
610 GLuint *row = (GLuint *) map;
611 for (j = 0; j < width; j++) {
612 row[j] = (row[j] & mask) | clear;
613 }
614 map += rowStride;
615 }
616 }
617 break;
618 case MESA_FORMAT_Z24_S8:
619 {
620 GLuint clear = ctx->Stencil.Clear & writeMask & 0xff;
621 GLuint mask = 0xffffff00 | ((~writeMask) & 0xff);
622 for (i = 0; i < height; i++) {
623 GLuint *row = (GLuint *) map;
624 for (j = 0; j < width; j++) {
625 row[j] = (row[j] & mask) | clear;
626 }
627 map += rowStride;
628 }
629 }
630 break;
631 default:
632 _mesa_problem(ctx, "Unexpected stencil buffer format %s"
633 " in _swrast_clear_stencil_buffer()",
634 _mesa_get_format_name(rb->Format));
635 }
636
637 ctx->Driver.UnmapRenderbuffer(ctx, rb);
638 }