mesa: fix stencil state problem when GL_ATI_separate_stencil wasn't enabled
[mesa.git] / src / mesa / main / stencil.c
1 /*
2 * Mesa 3-D graphics library
3 * Version: 7.1
4 *
5 * Copyright (C) 1999-2007 Brian Paul All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included
15 * in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * BRIAN PAUL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
21 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25
26 /**
27 * \file stencil.c
28 * Stencil operations.
29 *
30 * Note: There's an incompatibility between GL_EXT_stencil_two_side and
31 * OpenGL 2.0's two-sided stencil feature.
32 *
33 * With GL_EXT_stencil_two_side, calling glStencilOp/Func/Mask() only the
34 * front OR back face state (as set by glActiveStencilFaceEXT) is set.
35 *
36 * But with OpenGL 2.0, calling glStencilOp/Func/Mask() sets BOTH the
37 * front AND back state.
38 *
39 * So either we advertise the GL_EXT_stencil_two_side extension, or OpenGL
40 * 2.0, but not both.
41 *
42 * Also, note that GL_ATI_separate_stencil is different as well:
43 * glStencilFuncSeparateATI(GLenum frontfunc, GLenum backfunc, ...) vs.
44 * glStencilFuncSeparate(GLenum face, GLenum func, ...).
45 */
46
47
48 #include "glheader.h"
49 #include "imports.h"
50 #include "context.h"
51 #include "macros.h"
52 #include "stencil.h"
53 #include "mtypes.h"
54
55
56 static GLboolean
57 validate_stencil_op(GLcontext *ctx, GLenum op)
58 {
59 switch (op) {
60 case GL_KEEP:
61 case GL_ZERO:
62 case GL_REPLACE:
63 case GL_INCR:
64 case GL_DECR:
65 case GL_INVERT:
66 return GL_TRUE;
67 case GL_INCR_WRAP_EXT:
68 case GL_DECR_WRAP_EXT:
69 if (ctx->Extensions.EXT_stencil_wrap) {
70 return GL_TRUE;
71 }
72 /* FALL-THROUGH */
73 default:
74 return GL_FALSE;
75 }
76 }
77
78
79 static GLboolean
80 validate_stencil_func(GLcontext *ctx, GLenum func)
81 {
82 switch (func) {
83 case GL_NEVER:
84 case GL_LESS:
85 case GL_LEQUAL:
86 case GL_GREATER:
87 case GL_GEQUAL:
88 case GL_EQUAL:
89 case GL_NOTEQUAL:
90 case GL_ALWAYS:
91 return GL_TRUE;
92 default:
93 return GL_FALSE;
94 }
95 }
96
97
98 /**
99 * Set the clear value for the stencil buffer.
100 *
101 * \param s clear value.
102 *
103 * \sa glClearStencil().
104 *
105 * Updates gl_stencil_attrib::Clear. On change
106 * flushes the vertices and notifies the driver via
107 * the dd_function_table::ClearStencil callback.
108 */
109 void GLAPIENTRY
110 _mesa_ClearStencil( GLint s )
111 {
112 GET_CURRENT_CONTEXT(ctx);
113 ASSERT_OUTSIDE_BEGIN_END(ctx);
114
115 if (ctx->Stencil.Clear == (GLuint) s)
116 return;
117
118 FLUSH_VERTICES(ctx, _NEW_STENCIL);
119 ctx->Stencil.Clear = (GLuint) s;
120
121 if (ctx->Driver.ClearStencil) {
122 ctx->Driver.ClearStencil( ctx, s );
123 }
124 }
125
126
127 /**
128 * Set the function and reference value for stencil testing.
129 *
130 * \param frontfunc front test function.
131 * \param backfunc back test function.
132 * \param ref front and back reference value.
133 * \param mask front and back bitmask.
134 *
135 * \sa glStencilFunc().
136 *
137 * Verifies the parameters and updates the respective values in
138 * __GLcontextRec::Stencil. On change flushes the vertices and notifies the
139 * driver via the dd_function_table::StencilFunc callback.
140 */
141 void GLAPIENTRY
142 _mesa_StencilFuncSeparateATI( GLenum frontfunc, GLenum backfunc, GLint ref, GLuint mask )
143 {
144 GET_CURRENT_CONTEXT(ctx);
145 const GLint stencilMax = (1 << ctx->DrawBuffer->Visual.stencilBits) - 1;
146 ASSERT_OUTSIDE_BEGIN_END(ctx);
147
148 if (!validate_stencil_func(ctx, frontfunc)) {
149 _mesa_error(ctx, GL_INVALID_ENUM,
150 "glStencilFuncSeparateATI(frontfunc)");
151 return;
152 }
153 if (!validate_stencil_func(ctx, backfunc)) {
154 _mesa_error(ctx, GL_INVALID_ENUM,
155 "glStencilFuncSeparateATI(backfunc)");
156 return;
157 }
158
159 ref = CLAMP( ref, 0, stencilMax );
160
161 /* set both front and back state */
162 if (ctx->Stencil.Function[0] == frontfunc &&
163 ctx->Stencil.Function[1] == backfunc &&
164 ctx->Stencil.ValueMask[0] == mask &&
165 ctx->Stencil.ValueMask[1] == mask &&
166 ctx->Stencil.Ref[0] == ref &&
167 ctx->Stencil.Ref[1] == ref)
168 return;
169 FLUSH_VERTICES(ctx, _NEW_STENCIL);
170 ctx->Stencil.Function[0] = frontfunc;
171 ctx->Stencil.Function[1] = backfunc;
172 ctx->Stencil.Ref[0] = ctx->Stencil.Ref[1] = ref;
173 ctx->Stencil.ValueMask[0] = ctx->Stencil.ValueMask[1] = mask;
174 if (ctx->Driver.StencilFuncSeparate) {
175 ctx->Driver.StencilFuncSeparate(ctx, GL_FRONT,
176 frontfunc, ref, mask);
177 ctx->Driver.StencilFuncSeparate(ctx, GL_BACK,
178 backfunc, ref, mask);
179 }
180 }
181
182
183 /**
184 * Set the function and reference value for stencil testing.
185 *
186 * \param func test function.
187 * \param ref reference value.
188 * \param mask bitmask.
189 *
190 * \sa glStencilFunc().
191 *
192 * Verifies the parameters and updates the respective values in
193 * __GLcontextRec::Stencil. On change flushes the vertices and notifies the
194 * driver via the dd_function_table::StencilFunc callback.
195 */
196 void GLAPIENTRY
197 _mesa_StencilFunc( GLenum func, GLint ref, GLuint mask )
198 {
199 GET_CURRENT_CONTEXT(ctx);
200 const GLint stencilMax = (1 << ctx->DrawBuffer->Visual.stencilBits) - 1;
201 ASSERT_OUTSIDE_BEGIN_END(ctx);
202
203 if (!validate_stencil_func(ctx, func)) {
204 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilFunc(func)");
205 return;
206 }
207
208 ref = CLAMP( ref, 0, stencilMax );
209
210 if (ctx->Extensions.EXT_stencil_two_side) {
211 /* only set active face state */
212 const GLint face = ctx->Stencil.ActiveFace;
213 if (ctx->Stencil.Function[face] == func &&
214 ctx->Stencil.ValueMask[face] == mask &&
215 ctx->Stencil.Ref[face] == ref)
216 return;
217 FLUSH_VERTICES(ctx, _NEW_STENCIL);
218 ctx->Stencil.Function[face] = func;
219 ctx->Stencil.Ref[face] = ref;
220 ctx->Stencil.ValueMask[face] = mask;
221 if (ctx->Driver.StencilFuncSeparate) {
222 ctx->Driver.StencilFuncSeparate(ctx, face ? GL_BACK : GL_FRONT,
223 func, ref, mask);
224 }
225 }
226 else {
227 /* set both front and back state */
228 if (ctx->Stencil.Function[0] == func &&
229 ctx->Stencil.Function[1] == func &&
230 ctx->Stencil.ValueMask[0] == mask &&
231 ctx->Stencil.ValueMask[1] == mask &&
232 ctx->Stencil.Ref[0] == ref &&
233 ctx->Stencil.Ref[1] == ref)
234 return;
235 FLUSH_VERTICES(ctx, _NEW_STENCIL);
236 ctx->Stencil.Function[0] = ctx->Stencil.Function[1] = func;
237 ctx->Stencil.Ref[0] = ctx->Stencil.Ref[1] = ref;
238 ctx->Stencil.ValueMask[0] = ctx->Stencil.ValueMask[1] = mask;
239 if (ctx->Driver.StencilFuncSeparate) {
240 ctx->Driver.StencilFuncSeparate(ctx, GL_FRONT_AND_BACK,
241 func, ref, mask);
242 }
243 }
244 }
245
246
247 /**
248 * Set the stencil writing mask.
249 *
250 * \param mask bit-mask to enable/disable writing of individual bits in the
251 * stencil planes.
252 *
253 * \sa glStencilMask().
254 *
255 * Updates gl_stencil_attrib::WriteMask. On change flushes the vertices and
256 * notifies the driver via the dd_function_table::StencilMask callback.
257 */
258 void GLAPIENTRY
259 _mesa_StencilMask( GLuint mask )
260 {
261 GET_CURRENT_CONTEXT(ctx);
262 ASSERT_OUTSIDE_BEGIN_END(ctx);
263
264 if (ctx->Extensions.EXT_stencil_two_side) {
265 /* only set active face state */
266 const GLint face = ctx->Stencil.ActiveFace;
267 if (ctx->Stencil.WriteMask[face] == mask)
268 return;
269 FLUSH_VERTICES(ctx, _NEW_STENCIL);
270 ctx->Stencil.WriteMask[face] = mask;
271 if (ctx->Driver.StencilMaskSeparate) {
272 ctx->Driver.StencilMaskSeparate(ctx, face ? GL_BACK : GL_FRONT, mask);
273 }
274 }
275 else {
276 /* set both front and back state */
277 if (ctx->Stencil.WriteMask[0] == mask &&
278 ctx->Stencil.WriteMask[1] == mask)
279 return;
280 FLUSH_VERTICES(ctx, _NEW_STENCIL);
281 ctx->Stencil.WriteMask[0] = ctx->Stencil.WriteMask[1] = mask;
282 if (ctx->Driver.StencilMaskSeparate) {
283 ctx->Driver.StencilMaskSeparate(ctx, GL_FRONT_AND_BACK, mask);
284 }
285 }
286 }
287
288
289 /**
290 * Set the stencil test actions.
291 *
292 * \param fail action to take when stencil test fails.
293 * \param zfail action to take when stencil test passes, but depth test fails.
294 * \param zpass action to take when stencil test passes and the depth test
295 * passes (or depth testing is not enabled).
296 *
297 * \sa glStencilOp().
298 *
299 * Verifies the parameters and updates the respective fields in
300 * __GLcontextRec::Stencil. On change flushes the vertices and notifies the
301 * driver via the dd_function_table::StencilOp callback.
302 */
303 void GLAPIENTRY
304 _mesa_StencilOp(GLenum fail, GLenum zfail, GLenum zpass)
305 {
306 GET_CURRENT_CONTEXT(ctx);
307 ASSERT_OUTSIDE_BEGIN_END(ctx);
308
309 if (!validate_stencil_op(ctx, fail)) {
310 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOp(sfail)");
311 return;
312 }
313 if (!validate_stencil_op(ctx, zfail)) {
314 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOp(zfail)");
315 return;
316 }
317 if (!validate_stencil_op(ctx, zpass)) {
318 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOp(zpass)");
319 return;
320 }
321
322 if (ctx->Extensions.EXT_stencil_two_side) {
323 /* only set active face state */
324 const GLint face = ctx->Stencil.ActiveFace;
325 if (ctx->Stencil.ZFailFunc[face] == zfail &&
326 ctx->Stencil.ZPassFunc[face] == zpass &&
327 ctx->Stencil.FailFunc[face] == fail)
328 return;
329 FLUSH_VERTICES(ctx, _NEW_STENCIL);
330 ctx->Stencil.ZFailFunc[face] = zfail;
331 ctx->Stencil.ZPassFunc[face] = zpass;
332 ctx->Stencil.FailFunc[face] = fail;
333 if (ctx->Driver.StencilOpSeparate) {
334 ctx->Driver.StencilOpSeparate(ctx, face ? GL_BACK : GL_FRONT,
335 fail, zfail, zpass);
336 }
337 }
338 else {
339 /* set both front and back state */
340 if (ctx->Stencil.ZFailFunc[0] == zfail &&
341 ctx->Stencil.ZFailFunc[1] == zfail &&
342 ctx->Stencil.ZPassFunc[0] == zpass &&
343 ctx->Stencil.ZPassFunc[1] == zpass &&
344 ctx->Stencil.FailFunc[0] == fail &&
345 ctx->Stencil.FailFunc[1] == fail)
346 return;
347 FLUSH_VERTICES(ctx, _NEW_STENCIL);
348 ctx->Stencil.ZFailFunc[0] = ctx->Stencil.ZFailFunc[1] = zfail;
349 ctx->Stencil.ZPassFunc[0] = ctx->Stencil.ZPassFunc[1] = zpass;
350 ctx->Stencil.FailFunc[0] = ctx->Stencil.FailFunc[1] = fail;
351 if (ctx->Driver.StencilOpSeparate) {
352 ctx->Driver.StencilOpSeparate(ctx, GL_FRONT_AND_BACK,
353 fail, zfail, zpass);
354 }
355 }
356 }
357
358
359
360 #if _HAVE_FULL_GL
361 /* GL_EXT_stencil_two_side */
362 void GLAPIENTRY
363 _mesa_ActiveStencilFaceEXT(GLenum face)
364 {
365 GET_CURRENT_CONTEXT(ctx);
366 ASSERT_OUTSIDE_BEGIN_END(ctx);
367
368 if (!ctx->Extensions.EXT_stencil_two_side) {
369 _mesa_error(ctx, GL_INVALID_OPERATION, "glActiveStencilFaceEXT");
370 return;
371 }
372
373 if (face == GL_FRONT || face == GL_BACK) {
374 FLUSH_VERTICES(ctx, _NEW_STENCIL);
375 ctx->Stencil.ActiveFace = (face == GL_FRONT) ? 0 : 1;
376 }
377 else {
378 _mesa_error(ctx, GL_INVALID_ENUM, "glActiveStencilFaceEXT(face)");
379 }
380 }
381 #endif
382
383
384
385 /**
386 * OpenGL 2.0 function.
387 * \todo Make StencilOp() call this function. And eventually remove the
388 * ctx->Driver.StencilOp function and use ctx->Driver.StencilOpSeparate
389 * instead.
390 */
391 void GLAPIENTRY
392 _mesa_StencilOpSeparate(GLenum face, GLenum sfail, GLenum zfail, GLenum zpass)
393 {
394 GLboolean set = GL_FALSE;
395 GET_CURRENT_CONTEXT(ctx);
396 ASSERT_OUTSIDE_BEGIN_END(ctx);
397
398 if (!validate_stencil_op(ctx, sfail)) {
399 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(sfail)");
400 return;
401 }
402 if (!validate_stencil_op(ctx, zfail)) {
403 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(zfail)");
404 return;
405 }
406 if (!validate_stencil_op(ctx, zpass)) {
407 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(zpass)");
408 return;
409 }
410 if (face != GL_FRONT && face != GL_BACK && face != GL_FRONT_AND_BACK) {
411 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilOpSeparate(face)");
412 return;
413 }
414
415 if (face != GL_BACK) {
416 /* set front */
417 if (ctx->Stencil.ZFailFunc[0] != zfail ||
418 ctx->Stencil.ZPassFunc[0] != zpass ||
419 ctx->Stencil.FailFunc[0] != sfail){
420 FLUSH_VERTICES(ctx, _NEW_STENCIL);
421 ctx->Stencil.ZFailFunc[0] = zfail;
422 ctx->Stencil.ZPassFunc[0] = zpass;
423 ctx->Stencil.FailFunc[0] = sfail;
424 set = GL_TRUE;
425 }
426 }
427 if (face != GL_FRONT) {
428 /* set back */
429 if (ctx->Stencil.ZFailFunc[1] != zfail ||
430 ctx->Stencil.ZPassFunc[1] != zpass ||
431 ctx->Stencil.FailFunc[1] != sfail) {
432 FLUSH_VERTICES(ctx, _NEW_STENCIL);
433 ctx->Stencil.ZFailFunc[1] = zfail;
434 ctx->Stencil.ZPassFunc[1] = zpass;
435 ctx->Stencil.FailFunc[1] = sfail;
436 set = GL_TRUE;
437 }
438 }
439 if (set && ctx->Driver.StencilOpSeparate) {
440 ctx->Driver.StencilOpSeparate(ctx, face, sfail, zfail, zpass);
441 }
442 }
443
444
445 /* OpenGL 2.0 */
446 void GLAPIENTRY
447 _mesa_StencilFuncSeparate(GLenum face, GLenum func, GLint ref, GLuint mask)
448 {
449 GET_CURRENT_CONTEXT(ctx);
450 const GLint stencilMax = (1 << ctx->DrawBuffer->Visual.stencilBits) - 1;
451 ASSERT_OUTSIDE_BEGIN_END(ctx);
452
453 if (face != GL_FRONT && face != GL_BACK && face != GL_FRONT_AND_BACK) {
454 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilFuncSeparate(face)");
455 return;
456 }
457 if (!validate_stencil_func(ctx, func)) {
458 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilFuncSeparate(func)");
459 return;
460 }
461
462 ref = CLAMP(ref, 0, stencilMax);
463
464 FLUSH_VERTICES(ctx, _NEW_STENCIL);
465
466 if (face != GL_BACK) {
467 /* set front */
468 ctx->Stencil.Function[0] = func;
469 ctx->Stencil.Ref[0] = ref;
470 ctx->Stencil.ValueMask[0] = mask;
471 }
472 if (face != GL_FRONT) {
473 /* set back */
474 ctx->Stencil.Function[1] = func;
475 ctx->Stencil.Ref[1] = ref;
476 ctx->Stencil.ValueMask[1] = mask;
477 }
478 if (ctx->Driver.StencilFuncSeparate) {
479 ctx->Driver.StencilFuncSeparate(ctx, face, func, ref, mask);
480 }
481 }
482
483
484 /* OpenGL 2.0 */
485 void GLAPIENTRY
486 _mesa_StencilMaskSeparate(GLenum face, GLuint mask)
487 {
488 GET_CURRENT_CONTEXT(ctx);
489 ASSERT_OUTSIDE_BEGIN_END(ctx);
490
491 if (face != GL_FRONT && face != GL_BACK && face != GL_FRONT_AND_BACK) {
492 _mesa_error(ctx, GL_INVALID_ENUM, "glStencilaMaskSeparate(face)");
493 return;
494 }
495
496 FLUSH_VERTICES(ctx, _NEW_STENCIL);
497
498 if (face != GL_BACK) {
499 ctx->Stencil.WriteMask[0] = mask;
500 }
501 if (face != GL_FRONT) {
502 ctx->Stencil.WriteMask[1] = mask;
503 }
504 if (ctx->Driver.StencilMaskSeparate) {
505 ctx->Driver.StencilMaskSeparate(ctx, face, mask);
506 }
507 }
508
509
510 /**
511 * Update derived stencil state.
512 */
513 void
514 _mesa_update_stencil(GLcontext *ctx)
515 {
516 if (ctx->Extensions.EXT_stencil_two_side) {
517 ctx->Stencil._TestTwoSide = ctx->Stencil.TestTwoSide;
518 }
519 else {
520 ctx->Stencil._TestTwoSide =
521 (ctx->Stencil.Function[0] != ctx->Stencil.Function[1] ||
522 ctx->Stencil.FailFunc[0] != ctx->Stencil.FailFunc[1] ||
523 ctx->Stencil.ZPassFunc[0] != ctx->Stencil.ZPassFunc[1] ||
524 ctx->Stencil.ZFailFunc[0] != ctx->Stencil.ZFailFunc[1] ||
525 ctx->Stencil.Ref[0] != ctx->Stencil.Ref[1] ||
526 ctx->Stencil.ValueMask[0] != ctx->Stencil.ValueMask[1] ||
527 ctx->Stencil.WriteMask[0] != ctx->Stencil.WriteMask[1]);
528 }
529 }
530
531
532 /**
533 * Initialize the context stipple state.
534 *
535 * \param ctx GL context.
536 *
537 * Initializes __GLcontextRec::Stencil attribute group.
538 */
539 void
540 _mesa_init_stencil(GLcontext *ctx)
541 {
542 ctx->Stencil.Enabled = GL_FALSE;
543 ctx->Stencil.TestTwoSide = GL_FALSE;
544 ctx->Stencil.ActiveFace = 0; /* 0 = GL_FRONT, 1 = GL_BACK */
545 ctx->Stencil.Function[0] = GL_ALWAYS;
546 ctx->Stencil.Function[1] = GL_ALWAYS;
547 ctx->Stencil.FailFunc[0] = GL_KEEP;
548 ctx->Stencil.FailFunc[1] = GL_KEEP;
549 ctx->Stencil.ZPassFunc[0] = GL_KEEP;
550 ctx->Stencil.ZPassFunc[1] = GL_KEEP;
551 ctx->Stencil.ZFailFunc[0] = GL_KEEP;
552 ctx->Stencil.ZFailFunc[1] = GL_KEEP;
553 ctx->Stencil.Ref[0] = 0;
554 ctx->Stencil.Ref[1] = 0;
555 ctx->Stencil.ValueMask[0] = ~0U;
556 ctx->Stencil.ValueMask[1] = ~0U;
557 ctx->Stencil.WriteMask[0] = ~0U;
558 ctx->Stencil.WriteMask[1] = ~0U;
559 ctx->Stencil.Clear = 0;
560 }