Merge remote branch 'main/master' into radeon-rewrite
[mesa.git] / src / mesa / main / stencil.c
1 /*
2 * Mesa 3-D graphics library
3 * Version: 7.1
4 *
5 * Copyright (C) 1999-2007 Brian Paul All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included
15 * in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * BRIAN PAUL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
21 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25
26 /**
27 * \file stencil.c
28 * Stencil operations.
29 *
30 * Note: There's some conflict between GL_EXT_stencil_two_side and
31 * OpenGL 2.0's two-sided stencil feature.
32 *
33 * With GL_EXT_stencil_two_side, calling glStencilOp/Func/Mask() only the
34 * front OR back face state (as set by glActiveStencilFaceEXT) is set.
35 *
36 * But with OpenGL 2.0, calling glStencilOp/Func/Mask() sets BOTH the
37 * front AND back state.
38 *
39 * Also, note that GL_ATI_separate_stencil is different as well:
40 * glStencilFuncSeparateATI(GLenum frontfunc, GLenum backfunc, ...) vs.
41 * glStencilFuncSeparate(GLenum face, GLenum func, ...).
42 *
43 * This problem is solved by keeping three sets of stencil state:
44 * state[0] = GL_FRONT state.
45 * state[1] = OpenGL 2.0 / GL_ATI_separate_stencil GL_BACK state.
46 * state[2] = GL_EXT_stencil_two_side GL_BACK state.
47 */
48
49
50 #include "glheader.h"
51 #include "imports.h"
52 #include "context.h"
53 #include "macros.h"
54 #include "stencil.h"
55 #include "mtypes.h"
56
57
58 static GLboolean
59 validate_stencil_op(GLcontext *ctx, GLenum op)
60 {
61 switch (op) {
62 case GL_KEEP:
63 case GL_ZERO:
64 case GL_REPLACE:
65 case GL_INCR:
66 case GL_DECR:
67 case GL_INVERT:
68 return GL_TRUE;
69 case GL_INCR_WRAP_EXT:
70 case GL_DECR_WRAP_EXT:
71 if (ctx->Extensions.EXT_stencil_wrap) {
72 return GL_TRUE;
73 }
74 /* FALL-THROUGH */
75 default:
76 return GL_FALSE;
77 }
78 }
79
80
81 static GLboolean
82 validate_stencil_func(GLcontext *ctx, GLenum func)
83 {
84 switch (func) {
85 case GL_NEVER:
86 case GL_LESS:
87 case GL_LEQUAL:
88 case GL_GREATER:
89 case GL_GEQUAL:
90 case GL_EQUAL:
91 case GL_NOTEQUAL:
92 case GL_ALWAYS:
93 return GL_TRUE;
94 default:
95 return GL_FALSE;
96 }
97 }
98
99
100 /**
101 * Set the clear value for the stencil buffer.
102 *
103 * \param s clear value.
104 *
105 * \sa glClearStencil().
106 *
107 * Updates gl_stencil_attrib::Clear. On change
108 * flushes the vertices and notifies the driver via
109 * the dd_function_table::ClearStencil callback.
110 */
111 void GLAPIENTRY
112 _mesa_ClearStencil( GLint s )
113 {
114 GET_CURRENT_CONTEXT(ctx);
115 ASSERT_OUTSIDE_BEGIN_END(ctx);
116
117 if (ctx->Stencil.Clear == (GLuint) s)
118 return;
119
120 FLUSH_VERTICES(ctx, _NEW_STENCIL);
121 ctx->Stencil.Clear = (GLuint) s;
122
123 if (ctx->Driver.ClearStencil) {
124 ctx->Driver.ClearStencil( ctx, s );
125 }
126 }
127
128
129 /**
130 * Set the function and reference value for stencil testing.
131 *
132 * \param frontfunc front test function.
133 * \param backfunc back test function.
134 * \param ref front and back reference value.
135 * \param mask front and back bitmask.
136 *
137 * \sa glStencilFunc().
138 *
139 * Verifies the parameters and updates the respective values in
140 * __GLcontextRec::Stencil. On change flushes the vertices and notifies the
141 * driver via the dd_function_table::StencilFunc callback.
142 */
143 void GLAPIENTRY
144 _mesa_StencilFuncSeparateATI( GLenum frontfunc, GLenum backfunc, GLint ref, GLuint mask )
145 {
146 GET_CURRENT_CONTEXT(ctx);
147 const GLint stencilMax = (1 << ctx->DrawBuffer->Visual.stencilBits) - 1;
148 ASSERT_OUTSIDE_BEGIN_END(ctx);
149
150 if (!validate_stencil_func(ctx, frontfunc)) {
151 _mesa_error(ctx, GL_INVALID_ENUM,
152 "glStencilFuncSeparateATI(frontfunc)");
153 return;
154 }
155 if (!validate_stencil_func(ctx, backfunc)) {
156 _mesa_error(ctx, GL_INVALID_ENUM,
157 "glStencilFuncSeparateATI(backfunc)");
158 return;
159 }
160
161 ref = CLAMP( ref, 0, stencilMax );
162
163 /* set both front and back state */
164 if (ctx->Stencil.Function[0] == frontfunc &&
165 ctx->Stencil.Function[1] == backfunc &&
166 ctx->Stencil.ValueMask[0] == mask &&
167 ctx->Stencil.ValueMask[1] == mask &&
168 ctx->Stencil.Ref[0] == ref &&
169 ctx->Stencil.Ref[1] == ref)
170 return;
171 FLUSH_VERTICES(ctx, _NEW_STENCIL);
172 ctx->Stencil.Function[0] = frontfunc;
173 ctx->Stencil.Function[1] = backfunc;
174 ctx->Stencil.Ref[0] = ctx->Stencil.Ref[1] = ref;
175 ctx->Stencil.ValueMask[0] = ctx->Stencil.ValueMask[1] = mask;
176 if (ctx->Driver.StencilFuncSeparate) {
177 ctx->Driver.StencilFuncSeparate(ctx, GL_FRONT,
178 frontfunc, ref, mask);
179 ctx->Driver.StencilFuncSeparate(ctx, GL_BACK,
180 backfunc, ref, mask);
181 }
182 }
183
184
185 /**
186 * Set the function and reference value for stencil testing.
187 *
188 * \param func test function.
189 * \param ref reference value.
190 * \param mask bitmask.
191 *
192 * \sa glStencilFunc().
193 *
194 * Verifies the parameters and updates the respective values in
195 * __GLcontextRec::Stencil. On change flushes the vertices and notifies the
196 * driver via the dd_function_table::StencilFunc callback.
197 */
198 void GLAPIENTRY
199 _mesa_StencilFunc( GLenum func, GLint ref, GLuint mask )
200 {
201 GET_CURRENT_CONTEXT(ctx);
202 const GLint stencilMax = (1 << ctx->DrawBuffer->Visual.stencilBits) - 1;
203 const GLint face = ctx->Stencil.ActiveFace;
204 ASSERT_OUTSIDE_BEGIN_END(ctx);
205
206 if (!validate_stencil_func(ctx, func)) {
207 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilFunc(func)");
208 return;
209 }
210
211 ref = CLAMP( ref, 0, stencilMax );
212
213 if (face != 0) {
214 if (ctx->Stencil.Function[face] == func &&
215 ctx->Stencil.ValueMask[face] == mask &&
216 ctx->Stencil.Ref[face] == ref)
217 return;
218 FLUSH_VERTICES(ctx, _NEW_STENCIL);
219 ctx->Stencil.Function[face] = func;
220 ctx->Stencil.Ref[face] = ref;
221 ctx->Stencil.ValueMask[face] = mask;
222
223 /* Only propagate the change to the driver if EXT_stencil_two_side
224 * is enabled.
225 */
226 if (ctx->Driver.StencilFuncSeparate && ctx->Stencil.TestTwoSide) {
227 ctx->Driver.StencilFuncSeparate(ctx, GL_BACK, func, ref, mask);
228 }
229 }
230 else {
231 /* set both front and back state */
232 if (ctx->Stencil.Function[0] == func &&
233 ctx->Stencil.Function[1] == func &&
234 ctx->Stencil.ValueMask[0] == mask &&
235 ctx->Stencil.ValueMask[1] == mask &&
236 ctx->Stencil.Ref[0] == ref &&
237 ctx->Stencil.Ref[1] == ref)
238 return;
239 FLUSH_VERTICES(ctx, _NEW_STENCIL);
240 ctx->Stencil.Function[0] = ctx->Stencil.Function[1] = func;
241 ctx->Stencil.Ref[0] = ctx->Stencil.Ref[1] = ref;
242 ctx->Stencil.ValueMask[0] = ctx->Stencil.ValueMask[1] = mask;
243 if (ctx->Driver.StencilFuncSeparate) {
244 ctx->Driver.StencilFuncSeparate(ctx,
245 ((ctx->Stencil.TestTwoSide)
246 ? GL_FRONT : GL_FRONT_AND_BACK),
247 func, ref, mask);
248 }
249 }
250 }
251
252
253 /**
254 * Set the stencil writing mask.
255 *
256 * \param mask bit-mask to enable/disable writing of individual bits in the
257 * stencil planes.
258 *
259 * \sa glStencilMask().
260 *
261 * Updates gl_stencil_attrib::WriteMask. On change flushes the vertices and
262 * notifies the driver via the dd_function_table::StencilMask callback.
263 */
264 void GLAPIENTRY
265 _mesa_StencilMask( GLuint mask )
266 {
267 GET_CURRENT_CONTEXT(ctx);
268 const GLint face = ctx->Stencil.ActiveFace;
269
270 ASSERT_OUTSIDE_BEGIN_END(ctx);
271
272 if (face != 0) {
273 /* Only modify the EXT_stencil_two_side back-face state.
274 */
275 if (ctx->Stencil.WriteMask[face] == mask)
276 return;
277 FLUSH_VERTICES(ctx, _NEW_STENCIL);
278 ctx->Stencil.WriteMask[face] = mask;
279
280 /* Only propagate the change to the driver if EXT_stencil_two_side
281 * is enabled.
282 */
283 if (ctx->Driver.StencilMaskSeparate && ctx->Stencil.TestTwoSide) {
284 ctx->Driver.StencilMaskSeparate(ctx, GL_BACK, mask);
285 }
286 }
287 else {
288 /* set both front and back state */
289 if (ctx->Stencil.WriteMask[0] == mask &&
290 ctx->Stencil.WriteMask[1] == mask)
291 return;
292 FLUSH_VERTICES(ctx, _NEW_STENCIL);
293 ctx->Stencil.WriteMask[0] = ctx->Stencil.WriteMask[1] = mask;
294 if (ctx->Driver.StencilMaskSeparate) {
295 ctx->Driver.StencilMaskSeparate(ctx,
296 ((ctx->Stencil.TestTwoSide)
297 ? GL_FRONT : GL_FRONT_AND_BACK),
298 mask);
299 }
300 }
301 }
302
303
304 /**
305 * Set the stencil test actions.
306 *
307 * \param fail action to take when stencil test fails.
308 * \param zfail action to take when stencil test passes, but depth test fails.
309 * \param zpass action to take when stencil test passes and the depth test
310 * passes (or depth testing is not enabled).
311 *
312 * \sa glStencilOp().
313 *
314 * Verifies the parameters and updates the respective fields in
315 * __GLcontextRec::Stencil. On change flushes the vertices and notifies the
316 * driver via the dd_function_table::StencilOp callback.
317 */
318 void GLAPIENTRY
319 _mesa_StencilOp(GLenum fail, GLenum zfail, GLenum zpass)
320 {
321 GET_CURRENT_CONTEXT(ctx);
322 const GLint face = ctx->Stencil.ActiveFace;
323
324 ASSERT_OUTSIDE_BEGIN_END(ctx);
325
326 if (!validate_stencil_op(ctx, fail)) {
327 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOp(sfail)");
328 return;
329 }
330 if (!validate_stencil_op(ctx, zfail)) {
331 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOp(zfail)");
332 return;
333 }
334 if (!validate_stencil_op(ctx, zpass)) {
335 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOp(zpass)");
336 return;
337 }
338
339 if (face != 0) {
340 /* only set active face state */
341 if (ctx->Stencil.ZFailFunc[face] == zfail &&
342 ctx->Stencil.ZPassFunc[face] == zpass &&
343 ctx->Stencil.FailFunc[face] == fail)
344 return;
345 FLUSH_VERTICES(ctx, _NEW_STENCIL);
346 ctx->Stencil.ZFailFunc[face] = zfail;
347 ctx->Stencil.ZPassFunc[face] = zpass;
348 ctx->Stencil.FailFunc[face] = fail;
349
350 /* Only propagate the change to the driver if EXT_stencil_two_side
351 * is enabled.
352 */
353 if (ctx->Driver.StencilOpSeparate && ctx->Stencil.TestTwoSide) {
354 ctx->Driver.StencilOpSeparate(ctx, GL_BACK, fail, zfail, zpass);
355 }
356 }
357 else {
358 /* set both front and back state */
359 if (ctx->Stencil.ZFailFunc[0] == zfail &&
360 ctx->Stencil.ZFailFunc[1] == zfail &&
361 ctx->Stencil.ZPassFunc[0] == zpass &&
362 ctx->Stencil.ZPassFunc[1] == zpass &&
363 ctx->Stencil.FailFunc[0] == fail &&
364 ctx->Stencil.FailFunc[1] == fail)
365 return;
366 FLUSH_VERTICES(ctx, _NEW_STENCIL);
367 ctx->Stencil.ZFailFunc[0] = ctx->Stencil.ZFailFunc[1] = zfail;
368 ctx->Stencil.ZPassFunc[0] = ctx->Stencil.ZPassFunc[1] = zpass;
369 ctx->Stencil.FailFunc[0] = ctx->Stencil.FailFunc[1] = fail;
370 if (ctx->Driver.StencilOpSeparate) {
371 ctx->Driver.StencilOpSeparate(ctx,
372 ((ctx->Stencil.TestTwoSide)
373 ? GL_FRONT : GL_FRONT_AND_BACK),
374 fail, zfail, zpass);
375 }
376 }
377 }
378
379
380
381 #if _HAVE_FULL_GL
382 /* GL_EXT_stencil_two_side */
383 void GLAPIENTRY
384 _mesa_ActiveStencilFaceEXT(GLenum face)
385 {
386 GET_CURRENT_CONTEXT(ctx);
387 ASSERT_OUTSIDE_BEGIN_END(ctx);
388
389 if (!ctx->Extensions.EXT_stencil_two_side) {
390 _mesa_error(ctx, GL_INVALID_OPERATION, "glActiveStencilFaceEXT");
391 return;
392 }
393
394 if (face == GL_FRONT || face == GL_BACK) {
395 FLUSH_VERTICES(ctx, _NEW_STENCIL);
396 ctx->Stencil.ActiveFace = (face == GL_FRONT) ? 0 : 2;
397 }
398 else {
399 _mesa_error(ctx, GL_INVALID_ENUM, "glActiveStencilFaceEXT(face)");
400 }
401 }
402 #endif
403
404
405
406 /**
407 * OpenGL 2.0 function.
408 * \todo Make StencilOp() call this function. And eventually remove the
409 * ctx->Driver.StencilOp function and use ctx->Driver.StencilOpSeparate
410 * instead.
411 */
412 void GLAPIENTRY
413 _mesa_StencilOpSeparate(GLenum face, GLenum sfail, GLenum zfail, GLenum zpass)
414 {
415 GLboolean set = GL_FALSE;
416 GET_CURRENT_CONTEXT(ctx);
417 ASSERT_OUTSIDE_BEGIN_END(ctx);
418
419 if (!validate_stencil_op(ctx, sfail)) {
420 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(sfail)");
421 return;
422 }
423 if (!validate_stencil_op(ctx, zfail)) {
424 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(zfail)");
425 return;
426 }
427 if (!validate_stencil_op(ctx, zpass)) {
428 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(zpass)");
429 return;
430 }
431 if (face != GL_FRONT && face != GL_BACK && face != GL_FRONT_AND_BACK) {
432 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(face)");
433 return;
434 }
435
436 if (face != GL_BACK) {
437 /* set front */
438 if (ctx->Stencil.ZFailFunc[0] != zfail ||
439 ctx->Stencil.ZPassFunc[0] != zpass ||
440 ctx->Stencil.FailFunc[0] != sfail){
441 FLUSH_VERTICES(ctx, _NEW_STENCIL);
442 ctx->Stencil.ZFailFunc[0] = zfail;
443 ctx->Stencil.ZPassFunc[0] = zpass;
444 ctx->Stencil.FailFunc[0] = sfail;
445 set = GL_TRUE;
446 }
447 }
448 if (face != GL_FRONT) {
449 /* set back */
450 if (ctx->Stencil.ZFailFunc[1] != zfail ||
451 ctx->Stencil.ZPassFunc[1] != zpass ||
452 ctx->Stencil.FailFunc[1] != sfail) {
453 FLUSH_VERTICES(ctx, _NEW_STENCIL);
454 ctx->Stencil.ZFailFunc[1] = zfail;
455 ctx->Stencil.ZPassFunc[1] = zpass;
456 ctx->Stencil.FailFunc[1] = sfail;
457 set = GL_TRUE;
458 }
459 }
460 if (set && ctx->Driver.StencilOpSeparate) {
461 ctx->Driver.StencilOpSeparate(ctx, face, sfail, zfail, zpass);
462 }
463 }
464
465
466 /* OpenGL 2.0 */
467 void GLAPIENTRY
468 _mesa_StencilFuncSeparate(GLenum face, GLenum func, GLint ref, GLuint mask)
469 {
470 GET_CURRENT_CONTEXT(ctx);
471 const GLint stencilMax = (1 << ctx->DrawBuffer->Visual.stencilBits) - 1;
472 ASSERT_OUTSIDE_BEGIN_END(ctx);
473
474 if (face != GL_FRONT && face != GL_BACK && face != GL_FRONT_AND_BACK) {
475 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilFuncSeparate(face)");
476 return;
477 }
478 if (!validate_stencil_func(ctx, func)) {
479 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilFuncSeparate(func)");
480 return;
481 }
482
483 ref = CLAMP(ref, 0, stencilMax);
484
485 FLUSH_VERTICES(ctx, _NEW_STENCIL);
486
487 if (face != GL_BACK) {
488 /* set front */
489 ctx->Stencil.Function[0] = func;
490 ctx->Stencil.Ref[0] = ref;
491 ctx->Stencil.ValueMask[0] = mask;
492 }
493 if (face != GL_FRONT) {
494 /* set back */
495 ctx->Stencil.Function[1] = func;
496 ctx->Stencil.Ref[1] = ref;
497 ctx->Stencil.ValueMask[1] = mask;
498 }
499 if (ctx->Driver.StencilFuncSeparate) {
500 ctx->Driver.StencilFuncSeparate(ctx, face, func, ref, mask);
501 }
502 }
503
504
505 /* OpenGL 2.0 */
506 void GLAPIENTRY
507 _mesa_StencilMaskSeparate(GLenum face, GLuint mask)
508 {
509 GET_CURRENT_CONTEXT(ctx);
510 ASSERT_OUTSIDE_BEGIN_END(ctx);
511
512 if (face != GL_FRONT && face != GL_BACK && face != GL_FRONT_AND_BACK) {
513 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilaMaskSeparate(face)");
514 return;
515 }
516
517 FLUSH_VERTICES(ctx, _NEW_STENCIL);
518
519 if (face != GL_BACK) {
520 ctx->Stencil.WriteMask[0] = mask;
521 }
522 if (face != GL_FRONT) {
523 ctx->Stencil.WriteMask[1] = mask;
524 }
525 if (ctx->Driver.StencilMaskSeparate) {
526 ctx->Driver.StencilMaskSeparate(ctx, face, mask);
527 }
528 }
529
530
531 /**
532 * Update derived stencil state.
533 */
534 void
535 _mesa_update_stencil(GLcontext *ctx)
536 {
537 const GLint face = ctx->Stencil._BackFace;
538
539 ctx->Stencil._Enabled = (ctx->Stencil.Enabled &&
540 ctx->DrawBuffer->Visual.stencilBits > 0);
541
542 ctx->Stencil._TestTwoSide =
543 ctx->Stencil._Enabled &&
544 (ctx->Stencil.Function[0] != ctx->Stencil.Function[face] ||
545 ctx->Stencil.FailFunc[0] != ctx->Stencil.FailFunc[face] ||
546 ctx->Stencil.ZPassFunc[0] != ctx->Stencil.ZPassFunc[face] ||
547 ctx->Stencil.ZFailFunc[0] != ctx->Stencil.ZFailFunc[face] ||
548 ctx->Stencil.Ref[0] != ctx->Stencil.Ref[face] ||
549 ctx->Stencil.ValueMask[0] != ctx->Stencil.ValueMask[face] ||
550 ctx->Stencil.WriteMask[0] != ctx->Stencil.WriteMask[face]);
551 }
552
553
554 /**
555 * Initialize the context stipple state.
556 *
557 * \param ctx GL context.
558 *
559 * Initializes __GLcontextRec::Stencil attribute group.
560 */
561 void
562 _mesa_init_stencil(GLcontext *ctx)
563 {
564 ctx->Stencil.Enabled = GL_FALSE;
565 ctx->Stencil.TestTwoSide = GL_FALSE;
566 ctx->Stencil.ActiveFace = 0; /* 0 = GL_FRONT, 2 = GL_BACK */
567 ctx->Stencil.Function[0] = GL_ALWAYS;
568 ctx->Stencil.Function[1] = GL_ALWAYS;
569 ctx->Stencil.Function[2] = GL_ALWAYS;
570 ctx->Stencil.FailFunc[0] = GL_KEEP;
571 ctx->Stencil.FailFunc[1] = GL_KEEP;
572 ctx->Stencil.FailFunc[2] = GL_KEEP;
573 ctx->Stencil.ZPassFunc[0] = GL_KEEP;
574 ctx->Stencil.ZPassFunc[1] = GL_KEEP;
575 ctx->Stencil.ZPassFunc[2] = GL_KEEP;
576 ctx->Stencil.ZFailFunc[0] = GL_KEEP;
577 ctx->Stencil.ZFailFunc[1] = GL_KEEP;
578 ctx->Stencil.ZFailFunc[2] = GL_KEEP;
579 ctx->Stencil.Ref[0] = 0;
580 ctx->Stencil.Ref[1] = 0;
581 ctx->Stencil.Ref[2] = 0;
582 ctx->Stencil.ValueMask[0] = ~0U;
583 ctx->Stencil.ValueMask[1] = ~0U;
584 ctx->Stencil.ValueMask[2] = ~0U;
585 ctx->Stencil.WriteMask[0] = ~0U;
586 ctx->Stencil.WriteMask[1] = ~0U;
587 ctx->Stencil.WriteMask[2] = ~0U;
588 ctx->Stencil.Clear = 0;
589 ctx->Stencil._BackFace = 1;
590 }