src/gallium/drivers/llvmpipe/lp_bld_depth.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * @file
  30  * Depth/stencil testing to LLVM IR translation.
  31  *
  32  * To be done accurately/efficiently the depth/stencil test must be done with
  33  * the same type/format of the depth/stencil buffer, which implies massaging
  34  * the incoming depths to fit into place. Using a more straightforward
  35  * type/format for depth/stencil values internally and only convert when
  36  * flushing would avoid this, but it would most likely result in depth fighting
  37  * artifacts.
  38  *
  39  * We are free to use a different pixel layout though. Since our basic
  40  * processing unit is a quad (2x2 pixel block) we store the depth/stencil
  41  * values tiled, a quad at time. That is, a depth buffer containing
  42  *
  43  *  Z11 Z12 Z13 Z14 ...
  44  *  Z21 Z22 Z23 Z24 ...
  45  *  Z31 Z32 Z33 Z34 ...
  46  *  Z41 Z42 Z43 Z44 ...
  47  *  ... ... ... ... ...
  48  *
  49  * will actually be stored in memory as
  50  *
  51  *  Z11 Z12 Z21 Z22 Z13 Z14 Z23 Z24 ...
  52  *  Z31 Z32 Z41 Z42 Z33 Z34 Z43 Z44 ...
  53  *  ... ... ... ... ... ... ... ... ...
  54  *
  55  *
  56  * Stencil test:
  57  * Two-sided stencil test is supported but probably not as efficient as
  58  * it could be.  Currently, we use if/then/else constructs to do the
  59  * operations for front vs. back-facing polygons.  We could probably do
  60  * both the front and back arithmetic then use a Select() instruction to
  61  * choose the result depending on polyon orientation.  We'd have to
  62  * measure performance both ways and see which is better.
  63  *
  64  * @author Jose Fonseca <jfonseca@vmware.com>
  65  */
  66
  67 #include "pipe/p_state.h"
  68 #include "util/u_format.h"
  69
  70 #include "gallivm/lp_bld_type.h"
  71 #include "gallivm/lp_bld_arit.h"
  72 #include "gallivm/lp_bld_const.h"
  73 #include "gallivm/lp_bld_logic.h"
  74 #include "gallivm/lp_bld_flow.h"
  75 #include "gallivm/lp_bld_intr.h"
  76 #include "gallivm/lp_bld_debug.h"
  77 #include "gallivm/lp_bld_swizzle.h"
  78
  79 #include "lp_bld_depth.h"
  80
  81
  82 /** Used to select fields from pipe_stencil_state */
  83 enum stencil_op {
  84    S_FAIL_OP,
  85    Z_FAIL_OP,
  86    Z_PASS_OP
  87 };
  88
  89
  90
  91 /**
  92  * Do the stencil test comparison (compare FB stencil values against ref value).
  93  * This will be used twice when generating two-sided stencil code.
  94  * \param stencil  the front/back stencil state
  95  * \param stencilRef  the stencil reference value, replicated as a vector
  96  * \param stencilVals  vector of stencil values from framebuffer
  97  * \return vector mask of pass/fail values (~0 or 0)
  98  */
  99 static LLVMValueRef
 100 lp_build_stencil_test_single(struct lp_build_context *bld,
 101                              const struct pipe_stencil_state *stencil,
 102                              LLVMValueRef stencilRef,
 103                              LLVMValueRef stencilVals)
 104 {
 105    const unsigned stencilMax = 255; /* XXX fix */
 106    struct lp_type type = bld->type;
 107    LLVMValueRef res;
 108
 109    assert(type.sign);
 110
 111    assert(stencil->enabled);
 112
 113    if (stencil->valuemask != stencilMax) {
 114       /* compute stencilRef = stencilRef & valuemask */
 115       LLVMValueRef valuemask = lp_build_const_int_vec(type, stencil->valuemask);
 116       stencilRef = LLVMBuildAnd(bld->builder, stencilRef, valuemask, "");
 117       /* compute stencilVals = stencilVals & valuemask */
 118       stencilVals = LLVMBuildAnd(bld->builder, stencilVals, valuemask, "");
 119    }
 120
 121    res = lp_build_cmp(bld, stencil->func, stencilRef, stencilVals);
 122
 123    return res;
 124 }
 125
 126
 127 /**
 128  * Do the one or two-sided stencil test comparison.
 129  * \sa lp_build_stencil_test_single
 130  * \param face  an integer indicating front (+) or back (-) facing polygon.
 131  *              If NULL, assume front-facing.
 132  */
 133 static LLVMValueRef
 134 lp_build_stencil_test(struct lp_build_context *bld,
 135                       const struct pipe_stencil_state stencil[2],
 136                       LLVMValueRef stencilRefs[2],
 137                       LLVMValueRef stencilVals,
 138                       LLVMValueRef face)
 139 {
 140    LLVMValueRef res;
 141
 142    assert(stencil[0].enabled);
 143
 144    if (stencil[1].enabled && face) {
 145       /* do two-sided test */
 146       struct lp_build_flow_context *flow_ctx;
 147       struct lp_build_if_state if_ctx;
 148       LLVMValueRef front_facing;
 149       LLVMValueRef zero = LLVMConstReal(LLVMFloatType(), 0.0);
 150       LLVMValueRef result = bld->undef;
 151
 152       flow_ctx = lp_build_flow_create(bld->builder);
 153       lp_build_flow_scope_begin(flow_ctx);
 154
 155       lp_build_flow_scope_declare(flow_ctx, &result);
 156
 157       /* front_facing = face > 0.0 */
 158       front_facing = LLVMBuildFCmp(bld->builder, LLVMRealUGT, face, zero, "");
 159
 160       lp_build_if(&if_ctx, flow_ctx, bld->builder, front_facing);
 161       {
 162          result = lp_build_stencil_test_single(bld, &stencil[0],
 163                                                stencilRefs[0], stencilVals);
 164       }
 165       lp_build_else(&if_ctx);
 166       {
 167          result = lp_build_stencil_test_single(bld, &stencil[1],
 168                                                stencilRefs[1], stencilVals);
 169       }
 170       lp_build_endif(&if_ctx);
 171
 172       lp_build_flow_scope_end(flow_ctx);
 173       lp_build_flow_destroy(flow_ctx);
 174
 175       res = result;
 176    }
 177    else {
 178       /* do single-side test */
 179       res = lp_build_stencil_test_single(bld, &stencil[0],
 180                                          stencilRefs[0], stencilVals);
 181    }
 182
 183    return res;
 184 }
 185
 186
 187 /**
 188  * Apply the stencil operator (add/sub/keep/etc) to the given vector
 189  * of stencil values.
 190  * \return  new stencil values vector
 191  */
 192 static LLVMValueRef
 193 lp_build_stencil_op_single(struct lp_build_context *bld,
 194                            const struct pipe_stencil_state *stencil,
 195                            enum stencil_op op,
 196                            LLVMValueRef stencilRef,
 197                            LLVMValueRef stencilVals,
 198                            LLVMValueRef mask)
 199
 200 {
 201    const unsigned stencilMax = 255; /* XXX fix */
 202    struct lp_type type = bld->type;
 203    LLVMValueRef res;
 204    LLVMValueRef max = lp_build_const_int_vec(type, stencilMax);
 205    unsigned stencil_op;
 206
 207    assert(type.sign);
 208
 209    switch (op) {
 210    case S_FAIL_OP:
 211       stencil_op = stencil->fail_op;
 212       break;
 213    case Z_FAIL_OP:
 214       stencil_op = stencil->zfail_op;
 215       break;
 216    case Z_PASS_OP:
 217       stencil_op = stencil->zpass_op;
 218       break;
 219    default:
 220       assert(0 && "Invalid stencil_op mode");
 221       stencil_op = PIPE_STENCIL_OP_KEEP;
 222    }
 223
 224    switch (stencil_op) {
 225    case PIPE_STENCIL_OP_KEEP:
 226       res = stencilVals;
 227       /* we can return early for this case */
 228       return res;
 229    case PIPE_STENCIL_OP_ZERO:
 230       res = bld->zero;
 231       break;
 232    case PIPE_STENCIL_OP_REPLACE:
 233       res = stencilRef;
 234       break;
 235    case PIPE_STENCIL_OP_INCR:
 236       res = lp_build_add(bld, stencilVals, bld->one);
 237       res = lp_build_min(bld, res, max);
 238       break;
 239    case PIPE_STENCIL_OP_DECR:
 240       res = lp_build_sub(bld, stencilVals, bld->one);
 241       res = lp_build_max(bld, res, bld->zero);
 242       break;
 243    case PIPE_STENCIL_OP_INCR_WRAP:
 244       res = lp_build_add(bld, stencilVals, bld->one);
 245       res = LLVMBuildAnd(bld->builder, res, max, "");
 246       break;
 247    case PIPE_STENCIL_OP_DECR_WRAP:
 248       res = lp_build_sub(bld, stencilVals, bld->one);
 249       res = LLVMBuildAnd(bld->builder, res, max, "");
 250       break;
 251    case PIPE_STENCIL_OP_INVERT:
 252       res = LLVMBuildNot(bld->builder, stencilVals, "");
 253       res = LLVMBuildAnd(bld->builder, res, max, "");
 254       break;
 255    default:
 256       assert(0 && "bad stencil op mode");
 257       res = NULL;
 258    }
 259
 260    if (stencil->writemask != stencilMax) {
 261       /* compute res = (res & mask) | (stencilVals & ~mask) */
 262       LLVMValueRef mask = lp_build_const_int_vec(type, stencil->writemask);
 263       LLVMValueRef cmask = LLVMBuildNot(bld->builder, mask, "notWritemask");
 264       LLVMValueRef t1 = LLVMBuildAnd(bld->builder, res, mask, "t1");
 265       LLVMValueRef t2 = LLVMBuildAnd(bld->builder, stencilVals, cmask, "t2");
 266       res = LLVMBuildOr(bld->builder, t1, t2, "t1_or_t2");
 267    }
 268
 269    /* only the update the vector elements enabled by 'mask' */
 270    res = lp_build_select(bld, mask, res, stencilVals);
 271
 272    return res;
 273 }
 274
 275
 276 /**
 277  * Do the one or two-sided stencil test op/update.
 278  */
 279 static LLVMValueRef
 280 lp_build_stencil_op(struct lp_build_context *bld,
 281                     const struct pipe_stencil_state stencil[2],
 282                     enum stencil_op op,
 283                     LLVMValueRef stencilRefs[2],
 284                     LLVMValueRef stencilVals,
 285                     LLVMValueRef mask,
 286                     LLVMValueRef face)
 287
 288 {
 289    assert(stencil[0].enabled);
 290
 291    if (stencil[1].enabled && face) {
 292       /* do two-sided op */
 293       struct lp_build_flow_context *flow_ctx;
 294       struct lp_build_if_state if_ctx;
 295       LLVMValueRef front_facing;
 296       LLVMValueRef zero = LLVMConstReal(LLVMFloatType(), 0.0);
 297       LLVMValueRef result = bld->undef;
 298
 299       flow_ctx = lp_build_flow_create(bld->builder);
 300       lp_build_flow_scope_begin(flow_ctx);
 301
 302       lp_build_flow_scope_declare(flow_ctx, &result);
 303
 304       /* front_facing = face > 0.0 */
 305       front_facing = LLVMBuildFCmp(bld->builder, LLVMRealUGT, face, zero, "");
 306
 307       lp_build_if(&if_ctx, flow_ctx, bld->builder, front_facing);
 308       {
 309          result = lp_build_stencil_op_single(bld, &stencil[0], op,
 310                                              stencilRefs[0], stencilVals, mask);
 311       }
 312       lp_build_else(&if_ctx);
 313       {
 314          result = lp_build_stencil_op_single(bld, &stencil[1], op,
 315                                              stencilRefs[1], stencilVals, mask);
 316       }
 317       lp_build_endif(&if_ctx);
 318
 319       lp_build_flow_scope_end(flow_ctx);
 320       lp_build_flow_destroy(flow_ctx);
 321
 322       return result;
 323    }
 324    else {
 325       /* do single-sided op */
 326       return lp_build_stencil_op_single(bld, &stencil[0], op,
 327                                         stencilRefs[0], stencilVals, mask);
 328    }
 329 }
 330
 331
 332
 333 /**
 334  * Return a type appropriate for depth/stencil testing.
 335  */
 336 struct lp_type
 337 lp_depth_type(const struct util_format_description *format_desc,
 338               unsigned length)
 339 {
 340    struct lp_type type;
 341    unsigned swizzle;
 342
 343    assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
 344    assert(format_desc->block.width == 1);
 345    assert(format_desc->block.height == 1);
 346
 347    swizzle = format_desc->swizzle[0];
 348    assert(swizzle < 4);
 349
 350    memset(&type, 0, sizeof type);
 351    type.width = format_desc->block.bits;
 352
 353    if(format_desc->channel[swizzle].type == UTIL_FORMAT_TYPE_FLOAT) {
 354       type.floating = TRUE;
 355       assert(swizzle == 0);
 356       assert(format_desc->channel[swizzle].size == format_desc->block.bits);
 357    }
 358    else if(format_desc->channel[swizzle].type == UTIL_FORMAT_TYPE_UNSIGNED) {
 359       assert(format_desc->block.bits <= 32);
 360       if(format_desc->channel[swizzle].normalized)
 361          type.norm = TRUE;
 362    }
 363    else
 364       assert(0);
 365
 366    assert(type.width <= length);
 367    type.length = length / type.width;
 368
 369    return type;
 370 }
 371
 372
 373 /**
 374  * Compute bitmask and bit shift to apply to the incoming fragment Z values
 375  * and the Z buffer values needed before doing the Z comparison.
 376  *
 377  * Note that we leave the Z bits in the position that we find them
 378  * in the Z buffer (typically 0xffffff00 or 0x00ffffff).  That lets us
 379  * get by with fewer bit twiddling steps.
 380  */
 381 static boolean
 382 get_z_shift_and_mask(const struct util_format_description *format_desc,
 383                      unsigned *shift, unsigned *mask)
 384 {
 385    const unsigned total_bits = format_desc->block.bits;
 386    unsigned z_swizzle;
 387    unsigned chan;
 388    unsigned padding_left, padding_right;
 389
 390    assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
 391    assert(format_desc->block.width == 1);
 392    assert(format_desc->block.height == 1);
 393
 394    z_swizzle = format_desc->swizzle[0];
 395
 396    if (z_swizzle == UTIL_FORMAT_SWIZZLE_NONE)
 397       return FALSE;
 398
 399    padding_right = 0;
 400    for (chan = 0; chan < z_swizzle; ++chan)
 401       padding_right += format_desc->channel[chan].size;
 402
 403    padding_left =
 404       total_bits - (padding_right + format_desc->channel[z_swizzle].size);
 405
 406    if (padding_left || padding_right) {
 407       unsigned long long mask_left = (1ULL << (total_bits - padding_left)) - 1;
 408       unsigned long long mask_right = (1ULL << (padding_right)) - 1;
 409       *mask = mask_left ^ mask_right;
 410    }
 411    else {
 412       *mask = 0xffffffff;
 413    }
 414
 415    *shift = padding_left;
 416
 417    return TRUE;
 418 }
 419
 420
 421 /**
 422  * Compute bitmask and bit shift to apply to the framebuffer pixel values
 423  * to put the stencil bits in the least significant position.
 424  * (i.e. 0x000000ff)
 425  */
 426 static boolean
 427 get_s_shift_and_mask(const struct util_format_description *format_desc,
 428                      unsigned *shift, unsigned *mask)
 429 {
 430    unsigned s_swizzle;
 431    unsigned chan, sz;
 432
 433    s_swizzle = format_desc->swizzle[1];
 434
 435    if (s_swizzle == UTIL_FORMAT_SWIZZLE_NONE)
 436       return FALSE;
 437
 438    *shift = 0;
 439    for (chan = 0; chan < s_swizzle; chan++)
 440       *shift += format_desc->channel[chan].size;
 441
 442    sz = format_desc->channel[s_swizzle].size;
 443    *mask = (1U << sz) - 1U;
 444
 445    return TRUE;
 446 }
 447
 448
 449 /**
 450  * Perform the occlusion test and increase the counter.
 451  * Test the depth mask. Add the number of channel which has none zero mask
 452  * into the occlusion counter. e.g. maskvalue is {-1, -1, -1, -1}.
 453  * The counter will add 4.
 454  *
 455  * \param type holds element type of the mask vector.
 456  * \param maskvalue is the depth test mask.
 457  * \param counter is a pointer of the uint32 counter.
 458  */
 459 static void
 460 lp_build_occlusion_count(LLVMBuilderRef builder,
 461                          struct lp_type type,
 462                          LLVMValueRef maskvalue,
 463                          LLVMValueRef counter)
 464 {
 465    LLVMValueRef countmask = lp_build_const_int_vec(type, 1);
 466    LLVMValueRef countv = LLVMBuildAnd(builder, maskvalue, countmask, "countv");
 467    LLVMTypeRef i8v16 = LLVMVectorType(LLVMInt8Type(), 16);
 468    LLVMValueRef counti = LLVMBuildBitCast(builder, countv, i8v16, "counti");
 469    LLVMValueRef maskarray[4] = {
 470       LLVMConstInt(LLVMInt32Type(), 0, 0),
 471       LLVMConstInt(LLVMInt32Type(), 4, 0),
 472       LLVMConstInt(LLVMInt32Type(), 8, 0),
 473       LLVMConstInt(LLVMInt32Type(), 12, 0),
 474    };
 475    LLVMValueRef shufflemask = LLVMConstVector(maskarray, 4);
 476    LLVMValueRef shufflev =  LLVMBuildShuffleVector(builder, counti, LLVMGetUndef(i8v16), shufflemask, "shufflev");
 477    LLVMValueRef shuffle = LLVMBuildBitCast(builder, shufflev, LLVMInt32Type(), "shuffle");
 478    LLVMValueRef count = lp_build_intrinsic_unary(builder, "llvm.ctpop.i32", LLVMInt32Type(), shuffle);
 479    LLVMValueRef orig = LLVMBuildLoad(builder, counter, "orig");
 480    LLVMValueRef incr = LLVMBuildAdd(builder, orig, count, "incr");
 481    LLVMBuildStore(builder, incr, counter);
 482 }
 483
 484
 485
 486 /**
 487  * Generate code for performing depth and/or stencil tests.
 488  * We operate on a vector of values (typically a 2x2 quad).
 489  *
 490  * \param depth  the depth test state
 491  * \param stencil  the front/back stencil state
 492  * \param type  the data type of the fragment depth/stencil values
 493  * \param format_desc  description of the depth/stencil surface
 494  * \param mask  the alive/dead pixel mask for the quad (vector)
 495  * \param stencil_refs  the front/back stencil ref values (scalar)
 496  * \param z_src  the incoming depth/stencil values (a 2x2 quad)
 497  * \param zs_dst_ptr  pointer to depth/stencil values in framebuffer
 498  * \param facing  contains float value indicating front/back facing polygon
 499  */
 500 void
 501 lp_build_depth_stencil_test(LLVMBuilderRef builder,
 502                             const struct pipe_depth_state *depth,
 503                             const struct pipe_stencil_state stencil[2],
 504                             struct lp_type type,
 505                             const struct util_format_description *format_desc,
 506                             struct lp_build_mask_context *mask,
 507                             LLVMValueRef stencil_refs[2],
 508                             LLVMValueRef z_src,
 509                             LLVMValueRef zs_dst_ptr,
 510                             LLVMValueRef face,
 511                             LLVMValueRef counter)
 512 {
 513    struct lp_build_context bld;
 514    struct lp_build_context sbld;
 515    struct lp_type s_type;
 516    LLVMValueRef zs_dst, z_dst = NULL;
 517    LLVMValueRef stencil_vals = NULL;
 518    LLVMValueRef z_bitmask = NULL, stencil_shift = NULL;
 519    LLVMValueRef z_pass = NULL, s_pass_mask = NULL;
 520    LLVMValueRef orig_mask = mask->value;
 521
 522    /* Sanity checking */
 523    {
 524       const unsigned z_swizzle = format_desc->swizzle[0];
 525       const unsigned s_swizzle = format_desc->swizzle[1];
 526
 527       assert(z_swizzle != UTIL_FORMAT_SWIZZLE_NONE ||
 528              s_swizzle != UTIL_FORMAT_SWIZZLE_NONE);
 529
 530       assert(depth->enabled || stencil[0].enabled);
 531
 532       assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
 533       assert(format_desc->block.width == 1);
 534       assert(format_desc->block.height == 1);
 535
 536       if (stencil[0].enabled) {
 537          assert(format_desc->format == PIPE_FORMAT_Z24_UNORM_S8_USCALED ||
 538                 format_desc->format == PIPE_FORMAT_S8_USCALED_Z24_UNORM);
 539       }
 540
 541       assert(z_swizzle < 4);
 542       assert(format_desc->block.bits == type.width);
 543       if (type.floating) {
 544          assert(z_swizzle == 0);
 545          assert(format_desc->channel[z_swizzle].type ==
 546                 UTIL_FORMAT_TYPE_FLOAT);
 547          assert(format_desc->channel[z_swizzle].size ==
 548                 format_desc->block.bits);
 549       }
 550       else {
 551          assert(format_desc->channel[z_swizzle].type ==
 552                 UTIL_FORMAT_TYPE_UNSIGNED);
 553          assert(format_desc->channel[z_swizzle].normalized);
 554          assert(!type.fixed);
 555          assert(!type.sign);
 556          assert(type.norm);
 557       }
 558    }
 559
 560
 561    /* Setup build context for Z vals */
 562    lp_build_context_init(&bld, builder, type);
 563
 564    /* Setup build context for stencil vals */
 565    s_type = lp_type_int_vec(type.width);
 566    lp_build_context_init(&sbld, builder, s_type);
 567
 568    /* Load current z/stencil value from z/stencil buffer */
 569    zs_dst = LLVMBuildLoad(builder, zs_dst_ptr, "");
 570
 571    lp_build_name(zs_dst, "zsbufval");
 572
 573
 574    /* Compute and apply the Z/stencil bitmasks and shifts.
 575     */
 576    {
 577       unsigned z_shift, z_mask;
 578       unsigned s_shift, s_mask;
 579
 580       if (get_z_shift_and_mask(format_desc, &z_shift, &z_mask)) {
 581          if (z_shift) {
 582             LLVMValueRef shift = lp_build_const_int_vec(type, z_shift);
 583             z_src = LLVMBuildLShr(builder, z_src, shift, "");
 584          }
 585
 586          if (z_mask != 0xffffffff) {
 587             LLVMValueRef mask = lp_build_const_int_vec(type, z_mask);
 588             z_src = LLVMBuildAnd(builder, z_src, mask, "");
 589             z_dst = LLVMBuildAnd(builder, zs_dst, mask, "");
 590             z_bitmask = mask;  /* used below */
 591          }
 592          else {
 593             z_dst = zs_dst;
 594          }
 595
 596          lp_build_name(z_dst, "zsbuf.z");
 597       }
 598
 599       if (get_s_shift_and_mask(format_desc, &s_shift, &s_mask)) {
 600          if (s_shift) {
 601             LLVMValueRef shift = lp_build_const_int_vec(type, s_shift);
 602             stencil_vals = LLVMBuildLShr(builder, zs_dst, shift, "");
 603             stencil_shift = shift;  /* used below */
 604          }
 605          else {
 606             stencil_vals = zs_dst;
 607          }
 608
 609          if (s_mask != 0xffffffff) {
 610             LLVMValueRef mask = lp_build_const_int_vec(type, s_mask);
 611             stencil_vals = LLVMBuildAnd(builder, stencil_vals, mask, "");
 612          }
 613
 614          lp_build_name(stencil_vals, "stencil");
 615       }
 616    }
 617
 618
 619    if (stencil[0].enabled) {
 620       /* convert scalar stencil refs into vectors */
 621       stencil_refs[0] = lp_build_broadcast_scalar(&bld, stencil_refs[0]);
 622       stencil_refs[1] = lp_build_broadcast_scalar(&bld, stencil_refs[1]);
 623
 624       s_pass_mask = lp_build_stencil_test(&sbld, stencil,
 625                                           stencil_refs, stencil_vals, face);
 626
 627       /* apply stencil-fail operator */
 628       {
 629          LLVMValueRef s_fail_mask = lp_build_andc(&bld, orig_mask, s_pass_mask);
 630          stencil_vals = lp_build_stencil_op(&sbld, stencil, S_FAIL_OP,
 631                                             stencil_refs, stencil_vals,
 632                                             s_fail_mask, face);
 633       }
 634    }
 635
 636    if (depth->enabled) {
 637       /* compare src Z to dst Z, returning 'pass' mask */
 638       z_pass = lp_build_cmp(&bld, depth->func, z_src, z_dst);
 639
 640       if (!stencil[0].enabled) {
 641          /* We can potentially skip all remaining operations here, but only
 642           * if stencil is disabled because we still need to update the stencil
 643           * buffer values.  Don't need to update Z buffer values.
 644           */
 645          lp_build_mask_update(mask, z_pass);
 646       }
 647
 648       if (depth->writemask) {
 649          LLVMValueRef zselectmask = mask->value;
 650
 651          /* mask off bits that failed Z test */
 652          zselectmask = LLVMBuildAnd(builder, zselectmask, z_pass, "");
 653
 654          /* mask off bits that failed stencil test */
 655          if (s_pass_mask) {
 656             zselectmask = LLVMBuildAnd(builder, zselectmask, s_pass_mask, "");
 657          }
 658
 659          /* if combined Z/stencil format, mask off the stencil bits */
 660          if (z_bitmask) {
 661             zselectmask = LLVMBuildAnd(builder, zselectmask, z_bitmask, "");
 662          }
 663
 664          /* Mix the old and new Z buffer values.
 665           * z_dst[i] = zselectmask[i] ? z_src[i] : z_dst[i]
 666           */
 667          z_dst = lp_build_select(&bld, zselectmask, z_src, z_dst);
 668       }
 669
 670       if (stencil[0].enabled) {
 671          /* update stencil buffer values according to z pass/fail result */
 672          LLVMValueRef z_fail_mask, z_pass_mask;
 673
 674          /* apply Z-fail operator */
 675          z_fail_mask = lp_build_andc(&bld, orig_mask, z_pass);
 676          stencil_vals = lp_build_stencil_op(&sbld, stencil, Z_FAIL_OP,
 677                                             stencil_refs, stencil_vals,
 678                                             z_fail_mask, face);
 679
 680          /* apply Z-pass operator */
 681          z_pass_mask = LLVMBuildAnd(bld.builder, orig_mask, z_pass, "");
 682          stencil_vals = lp_build_stencil_op(&sbld, stencil, Z_PASS_OP,
 683                                             stencil_refs, stencil_vals,
 684                                             z_pass_mask, face);
 685       }
 686    }
 687    else {
 688       /* No depth test: apply Z-pass operator to stencil buffer values which
 689        * passed the stencil test.
 690        */
 691       s_pass_mask = LLVMBuildAnd(bld.builder, orig_mask, s_pass_mask, "");
 692       stencil_vals = lp_build_stencil_op(&sbld, stencil, Z_PASS_OP,
 693                                          stencil_refs, stencil_vals,
 694                                          s_pass_mask, face);
 695    }
 696
 697    /* The Z bits are already in the right place but we may need to shift the
 698     * stencil bits before ORing Z with Stencil to make the final pixel value.
 699     */
 700    if (stencil_vals && stencil_shift)
 701       stencil_vals = LLVMBuildShl(bld.builder, stencil_vals,
 702                                   stencil_shift, "");
 703
 704    /* Finally, merge/store the z/stencil values */
 705    if ((depth->enabled && depth->writemask) ||
 706        (stencil[0].enabled && stencil[0].writemask)) {
 707
 708       if (z_dst && stencil_vals)
 709          zs_dst = LLVMBuildOr(bld.builder, z_dst, stencil_vals, "");
 710       else if (z_dst)
 711          zs_dst = z_dst;
 712       else
 713          zs_dst = stencil_vals;
 714
 715       LLVMBuildStore(builder, zs_dst, zs_dst_ptr);
 716    }
 717
 718    if (s_pass_mask)
 719       lp_build_mask_update(mask, s_pass_mask);
 720
 721    if (depth->enabled && stencil[0].enabled)
 722       lp_build_mask_update(mask, z_pass);
 723
 724    if (counter)
 725       lp_build_occlusion_count(builder, type, mask->value, counter);
 726 }