src/gallium/auxiliary/gallivm/lp_bld_tgsi_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * Copyright 2007-2008 VMware, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28
  29 /**
  30  * @file
  31  * TGSI to LLVM IR translation -- SoA.
  32  *
  33  * @author Jose Fonseca <jfonseca@vmware.com>
  34  *
  35  * Based on tgsi_sse2.c code written by Michal Krol, Keith Whitwell,
  36  * Brian Paul, and others.
  37  */
  38
  39 #include "pipe/p_config.h"
  40 #include "pipe/p_shader_tokens.h"
  41 #include "util/u_debug.h"
  42 #include "util/u_math.h"
  43 #include "util/u_memory.h"
  44 #include "tgsi/tgsi_dump.h"
  45 #include "tgsi/tgsi_exec.h"
  46 #include "tgsi/tgsi_info.h"
  47 #include "tgsi/tgsi_parse.h"
  48 #include "tgsi/tgsi_util.h"
  49 #include "tgsi/tgsi_scan.h"
  50 #include "tgsi/tgsi_strings.h"
  51 #include "lp_bld_tgsi_action.h"
  52 #include "lp_bld_type.h"
  53 #include "lp_bld_const.h"
  54 #include "lp_bld_arit.h"
  55 #include "lp_bld_bitarit.h"
  56 #include "lp_bld_gather.h"
  57 #include "lp_bld_init.h"
  58 #include "lp_bld_logic.h"
  59 #include "lp_bld_swizzle.h"
  60 #include "lp_bld_flow.h"
  61 #include "lp_bld_quad.h"
  62 #include "lp_bld_tgsi.h"
  63 #include "lp_bld_limits.h"
  64 #include "lp_bld_debug.h"
  65 #include "lp_bld_printf.h"
  66 #include "lp_bld_sample.h"
  67 #include "lp_bld_struct.h"
  68
  69 /* SM 4.0 says that subroutines can nest 32 deep and
  70  * we need one more for our main function */
  71 #define LP_MAX_NUM_FUNCS 33
  72
  73 #define DUMP_GS_EMITS 0
  74
  75 /*
  76  * If non-zero, the generated LLVM IR will print intermediate results on every TGSI
  77  * instruction.
  78  *
  79  * TODO:
  80  * - take execution masks in consideration
  81  * - debug control-flow instructions
  82  */
  83 #define DEBUG_EXECUTION 0
  84
  85
  86 /*
  87  * Emit code to print a register value.
  88  */
  89 static void
  90 emit_dump_reg(struct gallivm_state *gallivm,
  91               unsigned file,
  92               unsigned index,
  93               unsigned chan,
  94               LLVMValueRef value)
  95 {
  96    char buf[32];
  97
  98    util_snprintf(buf, sizeof buf, "    %s[%u].%c = ",
  99                  tgsi_file_name(file),
 100                  index, "xyzw"[chan]);
 101
 102    lp_build_print_value(gallivm, buf, value);
 103 }
 104
 105 /*
 106  * Return the context for the current function.
 107  * (always 'main', if shader doesn't do any function calls)
 108  */
 109 static inline struct function_ctx *
 110 func_ctx(struct lp_exec_mask *mask)
 111 {
 112    assert(mask->function_stack_size > 0);
 113    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 114    return &mask->function_stack[mask->function_stack_size - 1];
 115 }
 116
 117 /*
 118  * Returns true if we're in a loop.
 119  * It's global, meaning that it returns true even if there's
 120  * no loop inside the current function, but we were inside
 121  * a loop inside another function, from which this one was called.
 122  */
 123 static inline boolean
 124 mask_has_loop(struct lp_exec_mask *mask)
 125 {
 126    int i;
 127    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 128       const struct function_ctx *ctx = &mask->function_stack[i];
 129       if (ctx->loop_stack_size > 0)
 130          return TRUE;
 131    }
 132    return FALSE;
 133 }
 134
 135 /*
 136  * Returns true if we're inside a switch statement.
 137  * It's global, meaning that it returns true even if there's
 138  * no switch in the current function, but we were inside
 139  * a switch inside another function, from which this one was called.
 140  */
 141 static inline boolean
 142 mask_has_switch(struct lp_exec_mask *mask)
 143 {
 144    int i;
 145    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 146       const struct function_ctx *ctx = &mask->function_stack[i];
 147       if (ctx->switch_stack_size > 0)
 148          return TRUE;
 149    }
 150    return FALSE;
 151 }
 152
 153 /*
 154  * Returns true if we're inside a conditional.
 155  * It's global, meaning that it returns true even if there's
 156  * no conditional in the current function, but we were inside
 157  * a conditional inside another function, from which this one was called.
 158  */
 159 static inline boolean
 160 mask_has_cond(struct lp_exec_mask *mask)
 161 {
 162    int i;
 163    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 164       const struct function_ctx *ctx = &mask->function_stack[i];
 165       if (ctx->cond_stack_size > 0)
 166          return TRUE;
 167    }
 168    return FALSE;
 169 }
 170
 171
 172 /*
 173  * Initialize a function context at the specified index.
 174  */
 175 static void
 176 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
 177 {
 178    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 179    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 180    struct function_ctx *ctx =  &mask->function_stack[function_idx];
 181
 182    ctx->cond_stack_size = 0;
 183    ctx->loop_stack_size = 0;
 184    ctx->switch_stack_size = 0;
 185
 186    if (function_idx == 0) {
 187       ctx->ret_mask = mask->ret_mask;
 188    }
 189
 190    ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
 191                                        int_type, "looplimiter");
 192    LLVMBuildStore(
 193       builder,
 194       LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
 195       ctx->loop_limiter);
 196 }
 197
 198 static void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
 199 {
 200    mask->bld = bld;
 201    mask->has_mask = FALSE;
 202    mask->ret_in_main = FALSE;
 203    /* For the main function */
 204    mask->function_stack_size = 1;
 205
 206    mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
 207    mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
 208          mask->cond_mask = mask->switch_mask =
 209          LLVMConstAllOnes(mask->int_vec_type);
 210
 211    mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
 212                                  sizeof(mask->function_stack[0]));
 213    lp_exec_mask_function_init(mask, 0);
 214 }
 215
 216 static void
 217 lp_exec_mask_fini(struct lp_exec_mask *mask)
 218 {
 219    FREE(mask->function_stack);
 220 }
 221
 222 static void lp_exec_mask_update(struct lp_exec_mask *mask)
 223 {
 224    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 225    boolean has_loop_mask = mask_has_loop(mask);
 226    boolean has_cond_mask = mask_has_cond(mask);
 227    boolean has_switch_mask = mask_has_switch(mask);
 228    boolean has_ret_mask = mask->function_stack_size > 1 ||
 229          mask->ret_in_main;
 230
 231    if (has_loop_mask) {
 232       /*for loops we need to update the entire mask at runtime */
 233       LLVMValueRef tmp;
 234       assert(mask->break_mask);
 235       tmp = LLVMBuildAnd(builder,
 236                          mask->cont_mask,
 237                          mask->break_mask,
 238                          "maskcb");
 239       mask->exec_mask = LLVMBuildAnd(builder,
 240                                      mask->cond_mask,
 241                                      tmp,
 242                                      "maskfull");
 243    } else
 244       mask->exec_mask = mask->cond_mask;
 245
 246    if (has_switch_mask) {
 247       mask->exec_mask = LLVMBuildAnd(builder,
 248                                      mask->exec_mask,
 249                                      mask->switch_mask,
 250                                      "switchmask");
 251    }
 252
 253    if (has_ret_mask) {
 254       mask->exec_mask = LLVMBuildAnd(builder,
 255                                      mask->exec_mask,
 256                                      mask->ret_mask,
 257                                      "callmask");
 258    }
 259
 260    mask->has_mask = (has_cond_mask ||
 261                      has_loop_mask ||
 262                      has_switch_mask ||
 263                      has_ret_mask);
 264 }
 265
 266 static void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
 267                                    LLVMValueRef val)
 268 {
 269    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 270    struct function_ctx *ctx = func_ctx(mask);
 271
 272    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
 273       ctx->cond_stack_size++;
 274       return;
 275    }
 276    if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
 277       assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
 278    }
 279    ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
 280    assert(LLVMTypeOf(val) == mask->int_vec_type);
 281    mask->cond_mask = LLVMBuildAnd(builder,
 282                                   mask->cond_mask,
 283                                   val,
 284                                   "");
 285    lp_exec_mask_update(mask);
 286 }
 287
 288 static void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
 289 {
 290    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 291    struct function_ctx *ctx = func_ctx(mask);
 292    LLVMValueRef prev_mask;
 293    LLVMValueRef inv_mask;
 294
 295    assert(ctx->cond_stack_size);
 296    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 297       return;
 298    prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
 299    if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
 300       assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
 301    }
 302
 303    inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");
 304
 305    mask->cond_mask = LLVMBuildAnd(builder,
 306                                   inv_mask,
 307                                   prev_mask, "");
 308    lp_exec_mask_update(mask);
 309 }
 310
 311 static void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
 312 {
 313    struct function_ctx *ctx = func_ctx(mask);
 314    assert(ctx->cond_stack_size);
 315    --ctx->cond_stack_size;
 316    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 317       return;
 318    mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
 319    lp_exec_mask_update(mask);
 320 }
 321
 322 static void lp_exec_bgnloop(struct lp_exec_mask *mask)
 323 {
 324    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 325    struct function_ctx *ctx = func_ctx(mask);
 326
 327    if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
 328       ++ctx->loop_stack_size;
 329       return;
 330    }
 331
 332    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 333       ctx->break_type;
 334    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;
 335
 336    ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
 337    ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
 338    ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
 339    ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
 340    ++ctx->loop_stack_size;
 341
 342    ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
 343    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 344
 345    ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");
 346
 347    LLVMBuildBr(builder, ctx->loop_block);
 348    LLVMPositionBuilderAtEnd(builder, ctx->loop_block);
 349
 350    mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
 351
 352    lp_exec_mask_update(mask);
 353 }
 354
 355 static void lp_exec_break(struct lp_exec_mask *mask,
 356                           struct lp_build_tgsi_context * bld_base)
 357 {
 358    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 359    struct function_ctx *ctx = func_ctx(mask);
 360
 361    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 362       LLVMValueRef exec_mask = LLVMBuildNot(builder,
 363                                             mask->exec_mask,
 364                                             "break");
 365
 366       mask->break_mask = LLVMBuildAnd(builder,
 367                                       mask->break_mask,
 368                                       exec_mask, "break_full");
 369    }
 370    else {
 371       unsigned opcode = bld_base->instructions[bld_base->pc + 1].Instruction.Opcode;
 372       boolean break_always = (opcode == TGSI_OPCODE_ENDSWITCH ||
 373                               opcode == TGSI_OPCODE_CASE);
 374
 375
 376       if (ctx->switch_in_default) {
 377          /*
 378           * stop default execution but only if this is an unconditional switch.
 379           * (The condition here is not perfect since dead code after break is
 380           * allowed but should be sufficient since false negatives are just
 381           * unoptimized - so we don't have to pre-evaluate that).
 382           */
 383          if(break_always && ctx->switch_pc) {
 384             bld_base->pc = ctx->switch_pc;
 385             return;
 386          }
 387       }
 388
 389       if (break_always) {
 390          mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
 391       }
 392       else {
 393          LLVMValueRef exec_mask = LLVMBuildNot(builder,
 394                                                mask->exec_mask,
 395                                                "break");
 396          mask->switch_mask = LLVMBuildAnd(builder,
 397                                           mask->switch_mask,
 398                                           exec_mask, "break_switch");
 399       }
 400    }
 401
 402    lp_exec_mask_update(mask);
 403 }
 404
 405 static void lp_exec_break_condition(struct lp_exec_mask *mask,
 406                                     LLVMValueRef cond)
 407 {
 408    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 409    struct function_ctx *ctx = func_ctx(mask);
 410    LLVMValueRef cond_mask = LLVMBuildAnd(builder,
 411                                          mask->exec_mask,
 412                                          cond, "cond_mask");
 413    cond_mask = LLVMBuildNot(builder, cond_mask, "break_cond");
 414
 415    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 416       mask->break_mask = LLVMBuildAnd(builder,
 417                                       mask->break_mask,
 418                                       cond_mask, "breakc_full");
 419    }
 420    else {
 421       mask->switch_mask = LLVMBuildAnd(builder,
 422                                        mask->switch_mask,
 423                                        cond_mask, "breakc_switch");
 424    }
 425
 426    lp_exec_mask_update(mask);
 427 }
 428
 429 static void lp_exec_continue(struct lp_exec_mask *mask)
 430 {
 431    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 432    LLVMValueRef exec_mask = LLVMBuildNot(builder,
 433                                          mask->exec_mask,
 434                                          "");
 435
 436    mask->cont_mask = LLVMBuildAnd(builder,
 437                                   mask->cont_mask,
 438                                   exec_mask, "");
 439
 440    lp_exec_mask_update(mask);
 441 }
 442
 443
 444 static void lp_exec_endloop(struct gallivm_state *gallivm,
 445                             struct lp_exec_mask *mask)
 446 {
 447    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 448    struct function_ctx *ctx = func_ctx(mask);
 449    LLVMBasicBlockRef endloop;
 450    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 451    LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
 452                                                mask->bld->type.width *
 453                                                mask->bld->type.length);
 454    LLVMValueRef i1cond, i2cond, icond, limiter;
 455
 456    assert(mask->break_mask);
 457
 458
 459    assert(ctx->loop_stack_size);
 460    if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 461       --ctx->loop_stack_size;
 462       return;
 463    }
 464
 465    /*
 466     * Restore the cont_mask, but don't pop
 467     */
 468    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
 469    lp_exec_mask_update(mask);
 470
 471    /*
 472     * Unlike the continue mask, the break_mask must be preserved across loop
 473     * iterations
 474     */
 475    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 476
 477    /* Decrement the loop limiter */
 478    limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");
 479
 480    limiter = LLVMBuildSub(
 481       builder,
 482       limiter,
 483       LLVMConstInt(int_type, 1, false),
 484       "");
 485
 486    LLVMBuildStore(builder, limiter, ctx->loop_limiter);
 487
 488    /* i1cond = (mask != 0) */
 489    i1cond = LLVMBuildICmp(
 490       builder,
 491       LLVMIntNE,
 492       LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
 493       LLVMConstNull(reg_type), "i1cond");
 494
 495    /* i2cond = (looplimiter > 0) */
 496    i2cond = LLVMBuildICmp(
 497       builder,
 498       LLVMIntSGT,
 499       limiter,
 500       LLVMConstNull(int_type), "i2cond");
 501
 502    /* if( i1cond && i2cond ) */
 503    icond = LLVMBuildAnd(builder, i1cond, i2cond, "");
 504
 505    endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");
 506
 507    LLVMBuildCondBr(builder,
 508                    icond, ctx->loop_block, endloop);
 509
 510    LLVMPositionBuilderAtEnd(builder, endloop);
 511
 512    assert(ctx->loop_stack_size);
 513    --ctx->loop_stack_size;
 514    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
 515    mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
 516    ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
 517    ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
 518    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
 519          ctx->switch_stack_size];
 520
 521    lp_exec_mask_update(mask);
 522 }
 523
 524 static void lp_exec_switch(struct lp_exec_mask *mask,
 525                            LLVMValueRef switchval)
 526 {
 527    struct function_ctx *ctx = func_ctx(mask);
 528
 529    if (ctx->switch_stack_size >= LP_MAX_TGSI_NESTING ||
 530        ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 531       ctx->switch_stack_size++;
 532       return;
 533    }
 534
 535    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 536       ctx->break_type;
 537    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_SWITCH;
 538
 539    ctx->switch_stack[ctx->switch_stack_size].switch_mask = mask->switch_mask;
 540    ctx->switch_stack[ctx->switch_stack_size].switch_val = ctx->switch_val;
 541    ctx->switch_stack[ctx->switch_stack_size].switch_mask_default = ctx->switch_mask_default;
 542    ctx->switch_stack[ctx->switch_stack_size].switch_in_default = ctx->switch_in_default;
 543    ctx->switch_stack[ctx->switch_stack_size].switch_pc = ctx->switch_pc;
 544    ctx->switch_stack_size++;
 545
 546    mask->switch_mask = LLVMConstNull(mask->int_vec_type);
 547    ctx->switch_val = switchval;
 548    ctx->switch_mask_default = LLVMConstNull(mask->int_vec_type);
 549    ctx->switch_in_default = false;
 550    ctx->switch_pc = 0;
 551
 552    lp_exec_mask_update(mask);
 553 }
 554
 555 static void lp_exec_endswitch(struct lp_exec_mask *mask,
 556                               struct lp_build_tgsi_context * bld_base)
 557 {
 558    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 559    struct function_ctx *ctx = func_ctx(mask);
 560
 561    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 562       ctx->switch_stack_size--;
 563       return;
 564    }
 565
 566    /* check if there's deferred default if so do it now */
 567    if (ctx->switch_pc && !ctx->switch_in_default) {
 568       LLVMValueRef prevmask, defaultmask;
 569       unsigned tmp_pc;
 570       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 571       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 572       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 573       ctx->switch_in_default = true;
 574
 575       lp_exec_mask_update(mask);
 576
 577       assert(bld_base->instructions[ctx->switch_pc - 1].Instruction.Opcode ==
 578              TGSI_OPCODE_DEFAULT);
 579
 580       tmp_pc = bld_base->pc;
 581       bld_base->pc = ctx->switch_pc;
 582       /*
 583        * re-purpose switch_pc to point to here again, since we stop execution of
 584        * the deferred default after next break.
 585        */
 586       ctx->switch_pc = tmp_pc - 1;
 587
 588       return;
 589    }
 590
 591    else if (ctx->switch_pc && ctx->switch_in_default) {
 592       assert(bld_base->pc == ctx->switch_pc + 1);
 593    }
 594
 595    ctx->switch_stack_size--;
 596    mask->switch_mask = ctx->switch_stack[ctx->switch_stack_size].switch_mask;
 597    ctx->switch_val = ctx->switch_stack[ctx->switch_stack_size].switch_val;
 598    ctx->switch_mask_default = ctx->switch_stack[ctx->switch_stack_size].switch_mask_default;
 599    ctx->switch_in_default = ctx->switch_stack[ctx->switch_stack_size].switch_in_default;
 600    ctx->switch_pc = ctx->switch_stack[ctx->switch_stack_size].switch_pc;
 601
 602    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size];
 603
 604    lp_exec_mask_update(mask);
 605 }
 606
 607 static void lp_exec_case(struct lp_exec_mask *mask,
 608                          LLVMValueRef caseval)
 609 {
 610    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 611    struct function_ctx *ctx = func_ctx(mask);
 612
 613    LLVMValueRef casemask, prevmask;
 614
 615    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 616       return;
 617    }
 618
 619    /* skipping case mask evaluation here is NOT optional (not in all cases anyway). */
 620    if (!ctx->switch_in_default) {
 621       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 622       casemask = lp_build_cmp(mask->bld, PIPE_FUNC_EQUAL, caseval, ctx->switch_val);
 623       ctx->switch_mask_default = LLVMBuildOr(builder, casemask,
 624                                              ctx->switch_mask_default, "sw_default_mask");
 625       casemask = LLVMBuildOr(builder, casemask, mask->switch_mask, "");
 626       mask->switch_mask = LLVMBuildAnd(builder, casemask, prevmask, "sw_mask");
 627
 628       lp_exec_mask_update(mask);
 629    }
 630 }
 631
 632 /*
 633  * Analyse default statement in a switch.
 634  * \return true if default is last statement, false otherwise
 635  * \param default_pc_start contains pc of instruction to jump to
 636  *                         if default wasn't last but there's no
 637  *                         fallthrough into default.
 638  */
 639 static boolean default_analyse_is_last(struct lp_exec_mask *mask,
 640                                        struct lp_build_tgsi_context * bld_base,
 641                                        int *default_pc_start)
 642 {
 643    unsigned pc = bld_base->pc;
 644    struct function_ctx *ctx = func_ctx(mask);
 645    unsigned curr_switch_stack = ctx->switch_stack_size;
 646
 647    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 648       return false;
 649    }
 650
 651    /* skip over case statements which are together with default */
 652    while (bld_base->instructions[pc].Instruction.Opcode == TGSI_OPCODE_CASE) {
 653       pc++;
 654    }
 655
 656    while (pc != -1 && pc < bld_base->num_instructions) {
 657       unsigned opcode = bld_base->instructions[pc].Instruction.Opcode;
 658       switch (opcode) {
 659       case TGSI_OPCODE_CASE:
 660          if (curr_switch_stack == ctx->switch_stack_size) {
 661             *default_pc_start = pc - 1;
 662             return false;
 663          }
 664          break;
 665       case TGSI_OPCODE_SWITCH:
 666          curr_switch_stack++;
 667          break;
 668       case TGSI_OPCODE_ENDSWITCH:
 669          if (curr_switch_stack == ctx->switch_stack_size) {
 670             *default_pc_start = pc - 1;
 671             return true;
 672          }
 673          curr_switch_stack--;
 674          break;
 675       }
 676       pc++;
 677    }
 678    /* should never arrive here */
 679    assert(0);
 680    return true;
 681 }
 682
 683 static void lp_exec_default(struct lp_exec_mask *mask,
 684                             struct lp_build_tgsi_context * bld_base)
 685 {
 686    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 687    struct function_ctx *ctx = func_ctx(mask);
 688
 689    int default_exec_pc;
 690    boolean default_is_last;
 691
 692    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 693       return;
 694    }
 695
 696    /*
 697     * This is a messy opcode, because it may not be always at the end and
 698     * there can be fallthrough in and out of it.
 699     */
 700
 701    default_is_last = default_analyse_is_last(mask, bld_base, &default_exec_pc);
 702    /*
 703     * If it is last statement in switch (note that case statements appearing
 704     * "at the same time" as default don't change that) everything is just fine,
 705     * update switch mask and go on. This means we can handle default with
 706     * fallthrough INTO it without overhead, if it is last.
 707     */
 708    if (default_is_last) {
 709       LLVMValueRef prevmask, defaultmask;
 710       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 711       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 712       defaultmask = LLVMBuildOr(builder, defaultmask, mask->switch_mask, "");
 713       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 714       ctx->switch_in_default = true;
 715
 716       lp_exec_mask_update(mask);
 717    }
 718    else {
 719       /*
 720        * Technically, "case" immediately before default isn't really a
 721        * fallthrough, however we still have to count them as such as we
 722        * already have updated the masks.
 723        * If that happens in practice could add a switch optimizer pass
 724        * which just gets rid of all case statements appearing together with
 725        * default (or could do switch analysis at switch start time instead).
 726        */
 727       unsigned opcode = bld_base->instructions[bld_base->pc - 1].Instruction.Opcode;
 728       boolean ft_into = (opcode != TGSI_OPCODE_BRK &&
 729                          opcode != TGSI_OPCODE_SWITCH);
 730       /*
 731        * If it is not last statement and there was no fallthrough into it,
 732        * we record the PC and continue execution at next case (again, those
 733        * case encountered at the same time don't count). At endswitch
 734        * time, we update switchmask, and go back executing the code we skipped
 735        * until the next break (possibly re-executing some code with changed mask
 736        * if there was a fallthrough out of default).
 737        * Finally, if it is not last statement and there was a fallthrough into it,
 738        * do the same as with the former case, except instead of skipping the code
 739        * just execute it without updating the mask, then go back and re-execute.
 740        */
 741       ctx->switch_pc = bld_base->pc;
 742       if (!ft_into) {
 743          bld_base->pc = default_exec_pc;
 744       }
 745    }
 746 }
 747
 748
 749 /* stores val into an address pointed to by dst_ptr.
 750  * mask->exec_mask is used to figure out which bits of val
 751  * should be stored into the address
 752  * (0 means don't store this bit, 1 means do store).
 753  */
 754 static void lp_exec_mask_store(struct lp_exec_mask *mask,
 755                                struct lp_build_context *bld_store,
 756                                LLVMValueRef pred,
 757                                LLVMValueRef val,
 758                                LLVMValueRef dst_ptr)
 759 {
 760    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 761
 762    assert(lp_check_value(bld_store->type, val));
 763    assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
 764    assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val));
 765
 766    /* Mix the predicate and execution mask */
 767    if (mask->has_mask) {
 768       if (pred) {
 769          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
 770       } else {
 771          pred = mask->exec_mask;
 772       }
 773    }
 774
 775    if (pred) {
 776       LLVMValueRef res, dst;
 777
 778       dst = LLVMBuildLoad(builder, dst_ptr, "");
 779       res = lp_build_select(bld_store, pred, val, dst);
 780       LLVMBuildStore(builder, res, dst_ptr);
 781    } else
 782       LLVMBuildStore(builder, val, dst_ptr);
 783 }
 784
 785 static void lp_exec_mask_call(struct lp_exec_mask *mask,
 786                               int func,
 787                               int *pc)
 788 {
 789    if (mask->function_stack_size >= LP_MAX_NUM_FUNCS) {
 790       return;
 791    }
 792
 793    lp_exec_mask_function_init(mask, mask->function_stack_size);
 794    mask->function_stack[mask->function_stack_size].pc = *pc;
 795    mask->function_stack[mask->function_stack_size].ret_mask = mask->ret_mask;
 796    mask->function_stack_size++;
 797    *pc = func;
 798 }
 799
 800 static void lp_exec_mask_ret(struct lp_exec_mask *mask, int *pc)
 801 {
 802    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 803    struct function_ctx *ctx = func_ctx(mask);
 804    LLVMValueRef exec_mask;
 805
 806    if (ctx->cond_stack_size == 0 &&
 807        ctx->loop_stack_size == 0 &&
 808        ctx->switch_stack_size == 0 &&
 809        mask->function_stack_size == 1) {
 810       /* returning from main() */
 811       *pc = -1;
 812       return;
 813    }
 814
 815    if (mask->function_stack_size == 1) {
 816       /*
 817        * This requires special handling since we need to ensure
 818        * we don't drop the mask even if we have no call stack
 819        * (e.g. after a ret in a if clause after the endif)
 820        */
 821       mask->ret_in_main = TRUE;
 822    }
 823
 824    exec_mask = LLVMBuildNot(builder,
 825                             mask->exec_mask,
 826                             "ret");
 827
 828    mask->ret_mask = LLVMBuildAnd(builder,
 829                                  mask->ret_mask,
 830                                  exec_mask, "ret_full");
 831
 832    lp_exec_mask_update(mask);
 833 }
 834
 835 static void lp_exec_mask_bgnsub(struct lp_exec_mask *mask)
 836 {
 837 }
 838
 839 static void lp_exec_mask_endsub(struct lp_exec_mask *mask, int *pc)
 840 {
 841    struct function_ctx *ctx;
 842
 843    assert(mask->function_stack_size > 1);
 844    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 845
 846    ctx = func_ctx(mask);
 847    mask->function_stack_size--;
 848
 849    *pc = ctx->pc;
 850    mask->ret_mask = ctx->ret_mask;
 851
 852    lp_exec_mask_update(mask);
 853 }
 854
 855
 856 static LLVMValueRef
 857 get_file_ptr(struct lp_build_tgsi_soa_context *bld,
 858              unsigned file,
 859              unsigned index,
 860              unsigned chan)
 861 {
 862    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 863    LLVMValueRef (*array_of_vars)[TGSI_NUM_CHANNELS];
 864    LLVMValueRef var_of_array;
 865
 866    switch (file) {
 867    case TGSI_FILE_TEMPORARY:
 868       array_of_vars = bld->temps;
 869       var_of_array = bld->temps_array;
 870       break;
 871    case TGSI_FILE_OUTPUT:
 872       array_of_vars = bld->outputs;
 873       var_of_array = bld->outputs_array;
 874       break;
 875    default:
 876       assert(0);
 877       return NULL;
 878    }
 879
 880    assert(chan < 4);
 881
 882    if (bld->indirect_files & (1 << file)) {
 883       LLVMValueRef lindex = lp_build_const_int32(bld->bld_base.base.gallivm, index * 4 + chan);
 884       return LLVMBuildGEP(builder, var_of_array, &lindex, 1, "");
 885    }
 886    else {
 887       assert(index <= bld->bld_base.info->file_max[file]);
 888       return array_of_vars[index][chan];
 889    }
 890 }
 891
 892
 893 /**
 894  * Return pointer to a temporary register channel (src or dest).
 895  * Note that indirect addressing cannot be handled here.
 896  * \param index  which temporary register
 897  * \param chan  which channel of the temp register.
 898  */
 899 LLVMValueRef
 900 lp_get_temp_ptr_soa(struct lp_build_tgsi_soa_context *bld,
 901              unsigned index,
 902              unsigned chan)
 903 {
 904    return get_file_ptr(bld, TGSI_FILE_TEMPORARY, index, chan);
 905 }
 906
 907 /**
 908  * Return pointer to a output register channel (src or dest).
 909  * Note that indirect addressing cannot be handled here.
 910  * \param index  which output register
 911  * \param chan  which channel of the output register.
 912  */
 913 LLVMValueRef
 914 lp_get_output_ptr(struct lp_build_tgsi_soa_context *bld,
 915                unsigned index,
 916                unsigned chan)
 917 {
 918    return get_file_ptr(bld, TGSI_FILE_OUTPUT, index, chan);
 919 }
 920
 921 /*
 922  * If we have indirect addressing in outputs copy our alloca array
 923  * to the outputs slots specified by the caller to make sure
 924  * our outputs are delivered consistently via the same interface.
 925  */
 926 static void
 927 gather_outputs(struct lp_build_tgsi_soa_context * bld)
 928 {
 929    if ((bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
 930       unsigned index, chan;
 931       assert(bld->bld_base.info->num_outputs <=
 932              bld->bld_base.info->file_max[TGSI_FILE_OUTPUT] + 1);
 933       for (index = 0; index < bld->bld_base.info->num_outputs; ++index) {
 934          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 935             bld->outputs[index][chan] = lp_get_output_ptr(bld, index, chan);
 936          }
 937       }
 938    }
 939 }
 940
 941 /**
 942  * Gather vector.
 943  * XXX the lp_build_gather() function should be capable of doing this
 944  * with a little work.
 945  */
 946 static LLVMValueRef
 947 build_gather(struct lp_build_tgsi_context *bld_base,
 948              LLVMValueRef base_ptr,
 949              LLVMValueRef indexes,
 950              LLVMValueRef overflow_mask,
 951              LLVMValueRef indexes2)
 952 {
 953    struct gallivm_state *gallivm = bld_base->base.gallivm;
 954    LLVMBuilderRef builder = gallivm->builder;
 955    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 956    struct lp_build_context *bld = &bld_base->base;
 957    LLVMValueRef res;
 958    unsigned i;
 959
 960    if (indexes2)
 961       res = LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2));
 962    else
 963       res = bld->undef;
 964    /*
 965     * overflow_mask is a vector telling us which channels
 966     * in the vector overflowed. We use the overflow behavior for
 967     * constant buffers which is defined as:
 968     * Out of bounds access to constant buffer returns 0 in all
 969     * components. Out of bounds behavior is always with respect
 970     * to the size of the buffer bound at that slot.
 971     */
 972
 973    if (overflow_mask) {
 974       /*
 975        * We avoid per-element control flow here (also due to llvm going crazy,
 976        * though I suspect it's better anyway since overflow is likely rare).
 977        * Note that since we still fetch from buffers even if num_elements was
 978        * zero (in this case we'll fetch from index zero) the jit func callers
 979        * MUST provide valid fake constant buffers of size 4x32 (the values do
 980        * not matter), otherwise we'd still need (not per element though)
 981        * control flow.
 982        */
 983       indexes = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes);
 984       if (indexes2)
 985          indexes2 = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes2);
 986    }
 987
 988    /*
 989     * Loop over elements of index_vec, load scalar value, insert it into 'res'.
 990     */
 991    for (i = 0; i < bld->type.length * (indexes2 ? 2 : 1); i++) {
 992       LLVMValueRef si, di;
 993       LLVMValueRef index;
 994       LLVMValueRef scalar_ptr, scalar;
 995
 996       di = lp_build_const_int32(bld->gallivm, i);
 997       if (indexes2)
 998          si = lp_build_const_int32(bld->gallivm, i >> 1);
 999       else
1000          si = di;
1001
1002       if (indexes2 && (i & 1)) {
1003          index = LLVMBuildExtractElement(builder,
1004                                          indexes2, si, "");
1005       } else {
1006          index = LLVMBuildExtractElement(builder,
1007                                          indexes, si, "");
1008       }
1009       scalar_ptr = LLVMBuildGEP(builder, base_ptr,
1010                                 &index, 1, "gather_ptr");
1011       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1012
1013       res = LLVMBuildInsertElement(builder, res, scalar, di, "");
1014    }
1015
1016    if (overflow_mask) {
1017       if (indexes2) {
1018          res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1019          overflow_mask = LLVMBuildSExt(builder, overflow_mask,
1020                                        bld_base->dbl_bld.int_vec_type, "");
1021          res = lp_build_select(&bld_base->dbl_bld, overflow_mask,
1022                                bld_base->dbl_bld.zero, res);
1023       } else
1024          res = lp_build_select(bld, overflow_mask, bld->zero, res);
1025    }
1026
1027    return res;
1028 }
1029
1030
1031 /**
1032  * Scatter/store vector.
1033  */
1034 static void
1035 emit_mask_scatter(struct lp_build_tgsi_soa_context *bld,
1036                   LLVMValueRef base_ptr,
1037                   LLVMValueRef indexes,
1038                   LLVMValueRef values,
1039                   struct lp_exec_mask *mask,
1040                   LLVMValueRef pred)
1041 {
1042    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1043    LLVMBuilderRef builder = gallivm->builder;
1044    unsigned i;
1045
1046    /* Mix the predicate and execution mask */
1047    if (mask->has_mask) {
1048       if (pred) {
1049          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
1050       }
1051       else {
1052          pred = mask->exec_mask;
1053       }
1054    }
1055
1056    /*
1057     * Loop over elements of index_vec, store scalar value.
1058     */
1059    for (i = 0; i < bld->bld_base.base.type.length; i++) {
1060       LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1061       LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
1062       LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
1063       LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
1064       LLVMValueRef scalar_pred = pred ?
1065          LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
1066
1067       if (0)
1068          lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
1069                          ii, val, index, scalar_ptr);
1070
1071       if (scalar_pred) {
1072          LLVMValueRef real_val, dst_val;
1073          dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
1074          real_val = lp_build_select(&bld->elem_bld, scalar_pred, val, dst_val);
1075          LLVMBuildStore(builder, real_val, scalar_ptr);
1076       }
1077       else {
1078          LLVMBuildStore(builder, val, scalar_ptr);
1079       }
1080    }
1081 }
1082
1083
1084 /**
1085  * Read the current value of the ADDR register, convert the floats to
1086  * ints, add the base index and return the vector of offsets.
1087  * The offsets will be used to index into the constant buffer or
1088  * temporary register file.
1089  */
1090 static LLVMValueRef
1091 get_indirect_index(struct lp_build_tgsi_soa_context *bld,
1092                    unsigned reg_file, unsigned reg_index,
1093                    const struct tgsi_ind_register *indirect_reg)
1094 {
1095    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1096    struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
1097    /* always use X component of address register */
1098    unsigned swizzle = indirect_reg->Swizzle;
1099    LLVMValueRef base;
1100    LLVMValueRef rel;
1101    LLVMValueRef max_index;
1102    LLVMValueRef index;
1103
1104    assert(bld->indirect_files & (1 << reg_file));
1105
1106    base = lp_build_const_int_vec(bld->bld_base.base.gallivm, uint_bld->type, reg_index);
1107
1108    assert(swizzle < 4);
1109    switch (indirect_reg->File) {
1110    case TGSI_FILE_ADDRESS:
1111       rel = LLVMBuildLoad(builder,
1112                           bld->addr[indirect_reg->Index][swizzle],
1113                           "load addr reg");
1114       /* ADDR LLVM values already have LLVM integer type. */
1115       break;
1116    case TGSI_FILE_TEMPORARY:
1117       rel = lp_get_temp_ptr_soa(bld, indirect_reg->Index, swizzle);
1118       rel = LLVMBuildLoad(builder, rel, "load temp reg");
1119       /* TEMP LLVM values always have LLVM float type, but for indirection, the
1120        * value actually stored is expected to be an integer */
1121       rel = LLVMBuildBitCast(builder, rel, uint_bld->vec_type, "");
1122       break;
1123    default:
1124       assert(0);
1125       rel = uint_bld->zero;
1126    }
1127
1128    index = lp_build_add(uint_bld, base, rel);
1129
1130    /*
1131     * emit_fetch_constant handles constant buffer overflow so this code
1132     * is pointless for them.
1133     * Furthermore the D3D10 spec in section 6.5 says:
1134     * If the constant buffer bound to a slot is larger than the size
1135     * declared in the shader for that slot, implementations are allowed
1136     * to return incorrect data (not necessarily 0) for indices that are
1137     * larger than the declared size but smaller than the buffer size.
1138     */
1139    if (reg_file != TGSI_FILE_CONSTANT) {
1140       max_index = lp_build_const_int_vec(bld->bld_base.base.gallivm,
1141                                          uint_bld->type,
1142                                          bld->bld_base.info->file_max[reg_file]);
1143
1144       assert(!uint_bld->type.sign);
1145       index = lp_build_min(uint_bld, index, max_index);
1146    }
1147
1148    return index;
1149 }
1150
1151 static struct lp_build_context *
1152 stype_to_fetch(struct lp_build_tgsi_context * bld_base,
1153                enum tgsi_opcode_type stype)
1154 {
1155    struct lp_build_context *bld_fetch;
1156
1157    switch (stype) {
1158    case TGSI_TYPE_FLOAT:
1159    case TGSI_TYPE_UNTYPED:
1160       bld_fetch = &bld_base->base;
1161       break;
1162    case TGSI_TYPE_UNSIGNED:
1163       bld_fetch = &bld_base->uint_bld;
1164       break;
1165    case TGSI_TYPE_SIGNED:
1166       bld_fetch = &bld_base->int_bld;
1167       break;
1168    case TGSI_TYPE_DOUBLE:
1169       bld_fetch = &bld_base->dbl_bld;
1170       break;
1171    case TGSI_TYPE_VOID:
1172    default:
1173       assert(0);
1174       bld_fetch = NULL;
1175       break;
1176    }
1177    return bld_fetch;
1178 }
1179
1180 static LLVMValueRef
1181 get_soa_array_offsets(struct lp_build_context *uint_bld,
1182                       LLVMValueRef indirect_index,
1183                       unsigned chan_index,
1184                       boolean need_perelement_offset)
1185 {
1186    struct gallivm_state *gallivm = uint_bld->gallivm;
1187    LLVMValueRef chan_vec =
1188       lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
1189    LLVMValueRef length_vec =
1190       lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
1191    LLVMValueRef index_vec;
1192
1193    /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
1194    index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1195    index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
1196    index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
1197
1198    if (need_perelement_offset) {
1199       LLVMValueRef pixel_offsets;
1200       int i;
1201      /* build pixel offset vector: {0, 1, 2, 3, ...} */
1202       pixel_offsets = uint_bld->undef;
1203       for (i = 0; i < uint_bld->type.length; i++) {
1204          LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1205          pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
1206                                                 ii, ii, "");
1207       }
1208       index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
1209    }
1210    return index_vec;
1211 }
1212
1213 static LLVMValueRef
1214 emit_fetch_constant(
1215    struct lp_build_tgsi_context * bld_base,
1216    const struct tgsi_full_src_register * reg,
1217    enum tgsi_opcode_type stype,
1218    unsigned swizzle)
1219 {
1220    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1221    struct gallivm_state *gallivm = bld_base->base.gallivm;
1222    LLVMBuilderRef builder = gallivm->builder;
1223    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1224    unsigned dimension = 0;
1225    LLVMValueRef consts_ptr;
1226    LLVMValueRef num_consts;
1227    LLVMValueRef res;
1228
1229    /* XXX: Handle fetching xyzw components as a vector */
1230    assert(swizzle != ~0);
1231
1232    if (reg->Register.Dimension) {
1233       assert(!reg->Dimension.Indirect);
1234       dimension = reg->Dimension.Index;
1235       assert(dimension < LP_MAX_TGSI_CONST_BUFFERS);
1236    }
1237
1238    consts_ptr = bld->consts[dimension];
1239    num_consts = bld->consts_sizes[dimension];
1240
1241    if (reg->Register.Indirect) {
1242       LLVMValueRef indirect_index;
1243       LLVMValueRef swizzle_vec =
1244          lp_build_const_int_vec(gallivm, uint_bld->type, swizzle);
1245       LLVMValueRef index_vec;  /* index into the const buffer */
1246       LLVMValueRef overflow_mask;
1247       LLVMValueRef index_vec2 = NULL;
1248
1249       indirect_index = get_indirect_index(bld,
1250                                           reg->Register.File,
1251                                           reg->Register.Index,
1252                                           &reg->Indirect);
1253
1254       /* All fetches are from the same constant buffer, so
1255        * we need to propagate the size to a vector to do a
1256        * vector comparison */
1257       num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
1258       /* Construct a boolean vector telling us which channels
1259        * overflow the bound constant buffer */
1260       overflow_mask = lp_build_compare(gallivm, uint_bld->type, PIPE_FUNC_GEQUAL,
1261                                        indirect_index, num_consts);
1262
1263       /* index_vec = indirect_index * 4 + swizzle */
1264       index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1265       index_vec = lp_build_add(uint_bld, index_vec, swizzle_vec);
1266
1267       if (stype == TGSI_TYPE_DOUBLE) {
1268          LLVMValueRef swizzle_vec2;
1269          swizzle_vec2 = lp_build_const_int_vec(gallivm, uint_bld->type, swizzle + 1);
1270          index_vec2 = lp_build_shl_imm(uint_bld, indirect_index, 2);
1271          index_vec2 = lp_build_add(uint_bld, index_vec2, swizzle_vec2);
1272       }
1273       /* Gather values from the constant buffer */
1274       res = build_gather(bld_base, consts_ptr, index_vec, overflow_mask, index_vec2);
1275    }
1276    else {
1277       LLVMValueRef index;  /* index into the const buffer */
1278       LLVMValueRef scalar, scalar_ptr;
1279       struct lp_build_context *bld_broad = &bld_base->base;
1280       index = lp_build_const_int32(gallivm, reg->Register.Index * 4 + swizzle);
1281
1282       scalar_ptr = LLVMBuildGEP(builder, consts_ptr,
1283                                 &index, 1, "");
1284       if (stype == TGSI_TYPE_DOUBLE) {
1285          LLVMTypeRef dptr_type = LLVMPointerType(LLVMDoubleTypeInContext(gallivm->context), 0);
1286          scalar_ptr = LLVMBuildBitCast(builder, scalar_ptr, dptr_type, "");
1287          bld_broad = &bld_base->dbl_bld;
1288       }
1289       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1290       res = lp_build_broadcast_scalar(bld_broad, scalar);
1291    }
1292
1293    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED || stype == TGSI_TYPE_DOUBLE) {
1294       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1295       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1296    }
1297
1298    return res;
1299 }
1300
1301 /**
1302  * Fetch double values from two separate channels.
1303  * Doubles are stored split across two channels, like xy and zw.
1304  * This function creates a set of 16 floats,
1305  * extracts the values from the two channels,
1306  * puts them in the correct place, then casts to 8 doubles.
1307  */
1308 static LLVMValueRef
1309 emit_fetch_double(
1310    struct lp_build_tgsi_context * bld_base,
1311    enum tgsi_opcode_type stype,
1312    LLVMValueRef input,
1313    LLVMValueRef input2)
1314 {
1315    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1316    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1317    LLVMBuilderRef builder = gallivm->builder;
1318    LLVMValueRef res;
1319    struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1320    int i;
1321    LLVMValueRef shuffles[16];
1322    int len = bld_base->base.type.length * 2;
1323    assert(len <= 16);
1324
1325    for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
1326       shuffles[i] = lp_build_const_int32(gallivm, i / 2);
1327       shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
1328    }
1329    res = LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
1330
1331    return LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1332 }
1333
1334 static LLVMValueRef
1335 emit_fetch_immediate(
1336    struct lp_build_tgsi_context * bld_base,
1337    const struct tgsi_full_src_register * reg,
1338    enum tgsi_opcode_type stype,
1339    unsigned swizzle)
1340 {
1341    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1342    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1343    LLVMBuilderRef builder = gallivm->builder;
1344    LLVMValueRef res = NULL;
1345
1346    if (bld->use_immediates_array || reg->Register.Indirect) {
1347       LLVMValueRef imms_array;
1348       LLVMTypeRef fptr_type;
1349
1350       /* cast imms_array pointer to float* */
1351       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1352       imms_array = LLVMBuildBitCast(builder, bld->imms_array, fptr_type, "");
1353
1354       if (reg->Register.Indirect) {
1355          LLVMValueRef indirect_index;
1356          LLVMValueRef index_vec;  /* index into the immediate register array */
1357          LLVMValueRef index_vec2 = NULL;
1358          indirect_index = get_indirect_index(bld,
1359                                              reg->Register.File,
1360                                              reg->Register.Index,
1361                                              &reg->Indirect);
1362          /*
1363           * Unlike for other reg classes, adding pixel offsets is unnecessary -
1364           * immediates are stored as full vectors (FIXME??? - might be better
1365           * to store them the same as constants) but all elements are the same
1366           * in any case.
1367           */
1368          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1369                                            indirect_index,
1370                                            swizzle,
1371                                            FALSE);
1372          if (stype == TGSI_TYPE_DOUBLE)
1373             index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
1374                                               indirect_index,
1375                                               swizzle + 1,
1376                                               FALSE);
1377          /* Gather values from the immediate register array */
1378          res = build_gather(bld_base, imms_array, index_vec, NULL, index_vec2);
1379       } else {
1380          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1381                                         reg->Register.Index * 4 + swizzle);
1382          LLVMValueRef imms_ptr =  LLVMBuildGEP(builder,
1383                                                 bld->imms_array, &lindex, 1, "");
1384          res = LLVMBuildLoad(builder, imms_ptr, "");
1385
1386          if (stype == TGSI_TYPE_DOUBLE) {
1387             LLVMValueRef lindex1;
1388             LLVMValueRef imms_ptr2;
1389             LLVMValueRef res2;
1390
1391             lindex1 = lp_build_const_int32(gallivm,
1392                                            reg->Register.Index * 4 + swizzle + 1);
1393             imms_ptr2 = LLVMBuildGEP(builder,
1394                                       bld->imms_array, &lindex1, 1, "");
1395             res2 = LLVMBuildLoad(builder, imms_ptr2, "");
1396             res = emit_fetch_double(bld_base, stype, res, res2);
1397          }
1398       }
1399    }
1400    else {
1401       res = bld->immediates[reg->Register.Index][swizzle];
1402       if (stype == TGSI_TYPE_DOUBLE)
1403          res = emit_fetch_double(bld_base, stype, res, bld->immediates[reg->Register.Index][swizzle + 1]);
1404    }
1405
1406    if (stype == TGSI_TYPE_UNSIGNED) {
1407       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1408    } else if (stype == TGSI_TYPE_SIGNED) {
1409       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1410    } else if (stype == TGSI_TYPE_DOUBLE) {
1411       res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1412    }
1413    return res;
1414 }
1415
1416 static LLVMValueRef
1417 emit_fetch_input(
1418    struct lp_build_tgsi_context * bld_base,
1419    const struct tgsi_full_src_register * reg,
1420    enum tgsi_opcode_type stype,
1421    unsigned swizzle)
1422 {
1423    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1424    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1425    LLVMBuilderRef builder = gallivm->builder;
1426    LLVMValueRef res;
1427
1428    if (reg->Register.Indirect) {
1429       LLVMValueRef indirect_index;
1430       LLVMValueRef index_vec;  /* index into the input reg array */
1431       LLVMValueRef index_vec2 = NULL;
1432       LLVMValueRef inputs_array;
1433       LLVMTypeRef fptr_type;
1434
1435       indirect_index = get_indirect_index(bld,
1436                                           reg->Register.File,
1437                                           reg->Register.Index,
1438                                           &reg->Indirect);
1439
1440       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1441                                         indirect_index,
1442                                         swizzle,
1443                                         TRUE);
1444       if (stype == TGSI_TYPE_DOUBLE) {
1445          index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
1446                                            indirect_index,
1447                                            swizzle + 1,
1448                                            TRUE);
1449       }
1450       /* cast inputs_array pointer to float* */
1451       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1452       inputs_array = LLVMBuildBitCast(builder, bld->inputs_array, fptr_type, "");
1453
1454       /* Gather values from the input register array */
1455       res = build_gather(bld_base, inputs_array, index_vec, NULL, index_vec2);
1456    } else {
1457       if (bld->indirect_files & (1 << TGSI_FILE_INPUT)) {
1458          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1459                                         reg->Register.Index * 4 + swizzle);
1460          LLVMValueRef input_ptr = LLVMBuildGEP(builder,
1461                                                bld->inputs_array, &lindex, 1, "");
1462
1463          res = LLVMBuildLoad(builder, input_ptr, "");
1464          if (stype == TGSI_TYPE_DOUBLE) {
1465             LLVMValueRef lindex1;
1466             LLVMValueRef input_ptr2;
1467             LLVMValueRef res2;
1468
1469             lindex1 = lp_build_const_int32(gallivm,
1470                                            reg->Register.Index * 4 + swizzle + 1);
1471             input_ptr2 = LLVMBuildGEP(builder,
1472                                       bld->inputs_array, &lindex1, 1, "");
1473             res2 = LLVMBuildLoad(builder, input_ptr2, "");
1474             res = emit_fetch_double(bld_base, stype, res, res2);
1475          }
1476       }
1477       else {
1478          res = bld->inputs[reg->Register.Index][swizzle];
1479          if (stype == TGSI_TYPE_DOUBLE)
1480             res = emit_fetch_double(bld_base, stype, res, bld->inputs[reg->Register.Index][swizzle + 1]);
1481       }
1482    }
1483
1484    assert(res);
1485
1486    if (stype == TGSI_TYPE_UNSIGNED) {
1487       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1488    } else if (stype == TGSI_TYPE_SIGNED) {
1489       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1490    } else if (stype == TGSI_TYPE_DOUBLE) {
1491       res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1492    }
1493
1494    return res;
1495 }
1496
1497
1498 static LLVMValueRef
1499 emit_fetch_gs_input(
1500    struct lp_build_tgsi_context * bld_base,
1501    const struct tgsi_full_src_register * reg,
1502    enum tgsi_opcode_type stype,
1503    unsigned swizzle)
1504 {
1505    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1506    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1507    const struct tgsi_shader_info *info = bld->bld_base.info;
1508    LLVMBuilderRef builder = gallivm->builder;
1509    LLVMValueRef attrib_index = NULL;
1510    LLVMValueRef vertex_index = NULL;
1511    LLVMValueRef swizzle_index = lp_build_const_int32(gallivm, swizzle);
1512    LLVMValueRef res;
1513
1514    if (info->input_semantic_name[reg->Register.Index] == TGSI_SEMANTIC_PRIMID) {
1515       /* This is really a system value not a regular input */
1516       assert(!reg->Register.Indirect);
1517       assert(!reg->Dimension.Indirect);
1518       res = bld->system_values.prim_id;
1519       if (stype != TGSI_TYPE_UNSIGNED && stype != TGSI_TYPE_SIGNED) {
1520          res = LLVMBuildBitCast(builder, res, bld_base->base.vec_type, "");
1521       }
1522       return res;
1523    }
1524
1525    if (reg->Register.Indirect) {
1526       attrib_index = get_indirect_index(bld,
1527                                         reg->Register.File,
1528                                         reg->Register.Index,
1529                                         &reg->Indirect);
1530    } else {
1531       attrib_index = lp_build_const_int32(gallivm, reg->Register.Index);
1532    }
1533
1534    if (reg->Dimension.Indirect) {
1535       vertex_index = get_indirect_index(bld,
1536                                         reg->Register.File,
1537                                         reg->Dimension.Index,
1538                                         &reg->DimIndirect);
1539    } else {
1540       vertex_index = lp_build_const_int32(gallivm, reg->Dimension.Index);
1541    }
1542
1543    res = bld->gs_iface->fetch_input(bld->gs_iface, bld_base,
1544                                     reg->Dimension.Indirect,
1545                                     vertex_index,
1546                                     reg->Register.Indirect,
1547                                     attrib_index,
1548                                     swizzle_index);
1549
1550    assert(res);
1551
1552    if (stype == TGSI_TYPE_UNSIGNED) {
1553       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1554    } else if (stype == TGSI_TYPE_SIGNED) {
1555       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1556    } else if (stype == TGSI_TYPE_DOUBLE) {
1557       res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1558    }
1559
1560    return res;
1561 }
1562
1563 static LLVMValueRef
1564 emit_fetch_temporary(
1565    struct lp_build_tgsi_context * bld_base,
1566    const struct tgsi_full_src_register * reg,
1567    enum tgsi_opcode_type stype,
1568    unsigned swizzle)
1569 {
1570    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1571    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1572    LLVMBuilderRef builder = gallivm->builder;
1573    LLVMValueRef res;
1574
1575    if (reg->Register.Indirect) {
1576       LLVMValueRef indirect_index;
1577       LLVMValueRef index_vec, index_vec2 = NULL;  /* index into the temp reg array */
1578       LLVMValueRef temps_array;
1579       LLVMTypeRef fptr_type;
1580
1581       indirect_index = get_indirect_index(bld,
1582                                           reg->Register.File,
1583                                           reg->Register.Index,
1584                                           &reg->Indirect);
1585
1586       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1587                                         indirect_index,
1588                                         swizzle,
1589                                         TRUE);
1590       if (stype == TGSI_TYPE_DOUBLE) {
1591                index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
1592                                                   indirect_index,
1593                                                   swizzle + 1,
1594                                                   TRUE);
1595       }
1596
1597       /* cast temps_array pointer to float* */
1598       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1599       temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1600
1601       /* Gather values from the temporary register array */
1602       res = build_gather(bld_base, temps_array, index_vec, NULL, index_vec2);
1603    }
1604    else {
1605       LLVMValueRef temp_ptr;
1606       temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle);
1607       res = LLVMBuildLoad(builder, temp_ptr, "");
1608
1609       if (stype == TGSI_TYPE_DOUBLE) {
1610          LLVMValueRef temp_ptr2, res2;
1611
1612          temp_ptr2 = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle + 1);
1613          res2 = LLVMBuildLoad(builder, temp_ptr2, "");
1614          res = emit_fetch_double(bld_base, stype, res, res2);
1615       }
1616    }
1617
1618    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED || stype == TGSI_TYPE_DOUBLE) {
1619       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1620       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1621    }
1622
1623    return res;
1624 }
1625
1626 static LLVMValueRef
1627 emit_fetch_system_value(
1628    struct lp_build_tgsi_context * bld_base,
1629    const struct tgsi_full_src_register * reg,
1630    enum tgsi_opcode_type stype,
1631    unsigned swizzle)
1632 {
1633    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1634    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1635    const struct tgsi_shader_info *info = bld->bld_base.info;
1636    LLVMBuilderRef builder = gallivm->builder;
1637    LLVMValueRef res;
1638    enum tgsi_opcode_type atype; // Actual type of the value
1639
1640    assert(!reg->Register.Indirect);
1641
1642    switch (info->system_value_semantic_name[reg->Register.Index]) {
1643    case TGSI_SEMANTIC_INSTANCEID:
1644       res = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
1645       atype = TGSI_TYPE_UNSIGNED;
1646       break;
1647
1648    case TGSI_SEMANTIC_VERTEXID:
1649       res = bld->system_values.vertex_id;
1650       atype = TGSI_TYPE_UNSIGNED;
1651       break;
1652
1653    case TGSI_SEMANTIC_VERTEXID_NOBASE:
1654       res = bld->system_values.vertex_id_nobase;
1655       atype = TGSI_TYPE_UNSIGNED;
1656       break;
1657
1658    case TGSI_SEMANTIC_BASEVERTEX:
1659       res = bld->system_values.basevertex;
1660       atype = TGSI_TYPE_UNSIGNED;
1661       break;
1662
1663    case TGSI_SEMANTIC_PRIMID:
1664       res = bld->system_values.prim_id;
1665       atype = TGSI_TYPE_UNSIGNED;
1666       break;
1667
1668    case TGSI_SEMANTIC_INVOCATIONID:
1669       res = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.invocation_id);
1670       atype = TGSI_TYPE_UNSIGNED;
1671       break;
1672
1673    default:
1674       assert(!"unexpected semantic in emit_fetch_system_value");
1675       res = bld_base->base.zero;
1676       atype = TGSI_TYPE_FLOAT;
1677       break;
1678    }
1679
1680    if (atype != stype) {
1681       if (stype == TGSI_TYPE_FLOAT) {
1682          res = LLVMBuildBitCast(builder, res, bld_base->base.vec_type, "");
1683       } else if (stype == TGSI_TYPE_UNSIGNED) {
1684          res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1685       } else if (stype == TGSI_TYPE_SIGNED) {
1686          res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1687       }
1688    }
1689
1690    return res;
1691 }
1692
1693 /**
1694  * Register fetch with derivatives.
1695  */
1696 static void
1697 emit_fetch_deriv(
1698    struct lp_build_tgsi_soa_context *bld,
1699    LLVMValueRef src,
1700    LLVMValueRef *res,
1701    LLVMValueRef *ddx,
1702    LLVMValueRef *ddy)
1703 {
1704    if(res)
1705       *res = src;
1706
1707    /* TODO: use interpolation coeffs for inputs */
1708
1709    if(ddx)
1710       *ddx = lp_build_ddx(&bld->bld_base.base, src);
1711
1712    if(ddy)
1713       *ddy = lp_build_ddy(&bld->bld_base.base, src);
1714 }
1715
1716
1717 /**
1718  * Predicate.
1719  */
1720 static void
1721 emit_fetch_predicate(
1722    struct lp_build_tgsi_soa_context *bld,
1723    const struct tgsi_full_instruction *inst,
1724    LLVMValueRef *pred)
1725 {
1726    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1727    unsigned index;
1728    unsigned char swizzles[4];
1729    LLVMValueRef unswizzled[4] = {NULL, NULL, NULL, NULL};
1730    LLVMValueRef value;
1731    unsigned chan;
1732
1733    if (!inst->Instruction.Predicate) {
1734       TGSI_FOR_EACH_CHANNEL( chan ) {
1735          pred[chan] = NULL;
1736       }
1737       return;
1738    }
1739
1740    swizzles[0] = inst->Predicate.SwizzleX;
1741    swizzles[1] = inst->Predicate.SwizzleY;
1742    swizzles[2] = inst->Predicate.SwizzleZ;
1743    swizzles[3] = inst->Predicate.SwizzleW;
1744
1745    index = inst->Predicate.Index;
1746    assert(index < LP_MAX_TGSI_PREDS);
1747
1748    TGSI_FOR_EACH_CHANNEL( chan ) {
1749       unsigned swizzle = swizzles[chan];
1750
1751       /*
1752        * Only fetch the predicate register channels that are actually listed
1753        * in the swizzles
1754        */
1755       if (!unswizzled[swizzle]) {
1756          value = LLVMBuildLoad(builder,
1757                                bld->preds[index][swizzle], "");
1758
1759          /*
1760           * Convert the value to an integer mask.
1761           *
1762           * TODO: Short-circuit this comparison -- a D3D setp_xx instructions
1763           * is needlessly causing two comparisons due to storing the intermediate
1764           * result as float vector instead of an integer mask vector.
1765           */
1766          value = lp_build_compare(bld->bld_base.base.gallivm,
1767                                   bld->bld_base.base.type,
1768                                   PIPE_FUNC_NOTEQUAL,
1769                                   value,
1770                                   bld->bld_base.base.zero);
1771          if (inst->Predicate.Negate) {
1772             value = LLVMBuildNot(builder, value, "");
1773          }
1774
1775          unswizzled[swizzle] = value;
1776       } else {
1777          value = unswizzled[swizzle];
1778       }
1779
1780       pred[chan] = value;
1781    }
1782 }
1783
1784 /**
1785  * store an array of 8 doubles into two arrays of 8 floats
1786  * i.e.
1787  * value is d0, d1, d2, d3 etc.
1788  * each double has high and low pieces x, y
1789  * so gets stored into the separate channels as:
1790  * chan_ptr = d0.x, d1.x, d2.x, d3.x
1791  * chan_ptr2 = d0.y, d1.y, d2.y, d3.y
1792  */
1793 static void
1794 emit_store_double_chan(struct lp_build_tgsi_context *bld_base,
1795                        int dtype,
1796                        LLVMValueRef chan_ptr, LLVMValueRef chan_ptr2,
1797                        LLVMValueRef pred,
1798                        LLVMValueRef value)
1799 {
1800    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1801    struct gallivm_state *gallivm = bld_base->base.gallivm;
1802    LLVMBuilderRef builder = gallivm->builder;
1803    struct lp_build_context *float_bld = &bld_base->base;
1804    int i;
1805    LLVMValueRef temp, temp2;
1806    LLVMValueRef shuffles[8];
1807    LLVMValueRef shuffles2[8];
1808
1809    for (i = 0; i < bld_base->base.type.length; i++) {
1810       shuffles[i] = lp_build_const_int32(gallivm, i * 2);
1811       shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
1812    }
1813
1814    temp = LLVMBuildShuffleVector(builder, value,
1815                                  LLVMGetUndef(LLVMTypeOf(value)),
1816                                  LLVMConstVector(shuffles,
1817                                                  bld_base->base.type.length),
1818                                  "");
1819    temp2 = LLVMBuildShuffleVector(builder, value,
1820                                   LLVMGetUndef(LLVMTypeOf(value)),
1821                                   LLVMConstVector(shuffles2,
1822                                                   bld_base->base.type.length),
1823                                   "");
1824
1825    lp_exec_mask_store(&bld->exec_mask, float_bld, pred, temp, chan_ptr);
1826    lp_exec_mask_store(&bld->exec_mask, float_bld, pred, temp2, chan_ptr2);
1827 }
1828
1829 /**
1830  * Register store.
1831  */
1832 static void
1833 emit_store_chan(
1834    struct lp_build_tgsi_context *bld_base,
1835    const struct tgsi_full_instruction *inst,
1836    unsigned index,
1837    unsigned chan_index,
1838    LLVMValueRef pred,
1839    LLVMValueRef value)
1840 {
1841    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1842    struct gallivm_state *gallivm = bld_base->base.gallivm;
1843    LLVMBuilderRef builder = gallivm->builder;
1844    const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1845    struct lp_build_context *float_bld = &bld_base->base;
1846    struct lp_build_context *int_bld = &bld_base->int_bld;
1847    LLVMValueRef indirect_index = NULL;
1848    enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode);
1849
1850    /*
1851     * Apply saturation.
1852     *
1853     * It is always assumed to be float.
1854     */
1855    if (inst->Instruction.Saturate) {
1856       assert(dtype == TGSI_TYPE_FLOAT ||
1857              dtype == TGSI_TYPE_UNTYPED);
1858       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1859       value = lp_build_clamp_zero_one_nanzero(float_bld, value);
1860    }
1861
1862    if (reg->Register.Indirect) {
1863       /*
1864        * Currently the mesa/st doesn't generate indirect stores
1865        * to doubles, it normally uses MOV to do indirect stores.
1866        */
1867       assert(dtype != TGSI_TYPE_DOUBLE);
1868       indirect_index = get_indirect_index(bld,
1869                                           reg->Register.File,
1870                                           reg->Register.Index,
1871                                           &reg->Indirect);
1872    } else {
1873       assert(reg->Register.Index <=
1874                              bld_base->info->file_max[reg->Register.File]);
1875    }
1876
1877    if (DEBUG_EXECUTION) {
1878       emit_dump_reg(gallivm, reg->Register.File, reg->Register.Index, chan_index, value);
1879    }
1880
1881    switch( reg->Register.File ) {
1882    case TGSI_FILE_OUTPUT:
1883       /* Outputs are always stored as floats */
1884       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1885
1886       if (reg->Register.Indirect) {
1887          LLVMValueRef index_vec;  /* indexes into the output registers */
1888          LLVMValueRef outputs_array;
1889          LLVMTypeRef fptr_type;
1890
1891          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1892                                            indirect_index,
1893                                            chan_index,
1894                                            TRUE);
1895
1896          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1897          outputs_array = LLVMBuildBitCast(builder, bld->outputs_array, fptr_type, "");
1898
1899          /* Scatter store values into output registers */
1900          emit_mask_scatter(bld, outputs_array, index_vec, value,
1901                            &bld->exec_mask, pred);
1902       }
1903       else {
1904          LLVMValueRef out_ptr = lp_get_output_ptr(bld, reg->Register.Index,
1905                                                   chan_index);
1906
1907          if (dtype == TGSI_TYPE_DOUBLE) {
1908             LLVMValueRef out_ptr2 = lp_get_output_ptr(bld, reg->Register.Index,
1909                                                       chan_index + 1);
1910             emit_store_double_chan(bld_base, dtype, out_ptr, out_ptr2,
1911                                    pred, value);
1912          } else
1913             lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, out_ptr);
1914       }
1915       break;
1916
1917    case TGSI_FILE_TEMPORARY:
1918       /* Temporaries are always stored as floats */
1919       if (dtype != TGSI_TYPE_DOUBLE)
1920          value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1921       else
1922          value = LLVMBuildBitCast(builder, value,  LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2), "");
1923
1924       if (reg->Register.Indirect) {
1925          LLVMValueRef index_vec;  /* indexes into the temp registers */
1926          LLVMValueRef temps_array;
1927          LLVMTypeRef fptr_type;
1928
1929          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1930                                            indirect_index,
1931                                            chan_index,
1932                                            TRUE);
1933
1934          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1935          temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1936
1937          /* Scatter store values into temp registers */
1938          emit_mask_scatter(bld, temps_array, index_vec, value,
1939                            &bld->exec_mask, pred);
1940       }
1941       else {
1942          LLVMValueRef temp_ptr;
1943          temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, chan_index);
1944
1945          if (dtype == TGSI_TYPE_DOUBLE) {
1946             LLVMValueRef temp_ptr2 = lp_get_temp_ptr_soa(bld,
1947                                                          reg->Register.Index,
1948                                                          chan_index + 1);
1949             emit_store_double_chan(bld_base, dtype, temp_ptr, temp_ptr2,
1950                                    pred, value);
1951          }
1952          else
1953             lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, temp_ptr);
1954       }
1955       break;
1956
1957    case TGSI_FILE_ADDRESS:
1958       assert(dtype == TGSI_TYPE_SIGNED);
1959       assert(LLVMTypeOf(value) == int_bld->vec_type);
1960       value = LLVMBuildBitCast(builder, value, int_bld->vec_type, "");
1961       lp_exec_mask_store(&bld->exec_mask, int_bld, pred, value,
1962                          bld->addr[reg->Register.Index][chan_index]);
1963       break;
1964
1965    case TGSI_FILE_PREDICATE:
1966       assert(LLVMTypeOf(value) == float_bld->vec_type);
1967       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1968       lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value,
1969                          bld->preds[reg->Register.Index][chan_index]);
1970       break;
1971
1972    default:
1973       assert( 0 );
1974    }
1975
1976    (void)dtype;
1977 }
1978
1979 /*
1980  * Called at the beginning of the translation of each TGSI instruction, to
1981  * emit some debug code.
1982  */
1983 static void
1984 emit_debug(
1985    struct lp_build_tgsi_context * bld_base,
1986    const struct tgsi_full_instruction * inst,
1987    const struct tgsi_opcode_info * info)
1988
1989 {
1990    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1991
1992    if (DEBUG_EXECUTION) {
1993       /*
1994        * Dump the TGSI instruction.
1995        */
1996
1997       struct gallivm_state *gallivm = bld_base->base.gallivm;
1998       char buf[512];
1999       buf[0] = '$';
2000       buf[1] = ' ';
2001       tgsi_dump_instruction_str(inst, bld_base->pc, &buf[2], sizeof buf - 2);
2002       lp_build_printf(gallivm, buf);
2003
2004       /* Dump the execution mask.
2005        */
2006       if (bld->exec_mask.has_mask) {
2007          lp_build_print_value(gallivm, "    mask = ", bld->exec_mask.exec_mask);
2008       }
2009    }
2010 }
2011
2012 static void
2013 emit_store(
2014    struct lp_build_tgsi_context * bld_base,
2015    const struct tgsi_full_instruction * inst,
2016    const struct tgsi_opcode_info * info,
2017    LLVMValueRef dst[4])
2018
2019 {
2020    unsigned chan_index;
2021    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2022    enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode);
2023    if(info->num_dst) {
2024       LLVMValueRef pred[TGSI_NUM_CHANNELS];
2025
2026       emit_fetch_predicate( bld, inst, pred );
2027
2028       TGSI_FOR_EACH_DST0_ENABLED_CHANNEL( inst, chan_index ) {
2029
2030          if (dtype == TGSI_TYPE_DOUBLE && (chan_index == 1 || chan_index == 3))
2031              continue;
2032          emit_store_chan(bld_base, inst, 0, chan_index, pred[chan_index], dst[chan_index]);
2033       }
2034    }
2035 }
2036
2037 static unsigned
2038 tgsi_to_pipe_tex_target(unsigned tgsi_target)
2039 {
2040    switch (tgsi_target) {
2041    case TGSI_TEXTURE_BUFFER:
2042       return PIPE_BUFFER;
2043    case TGSI_TEXTURE_1D:
2044    case TGSI_TEXTURE_SHADOW1D:
2045       return PIPE_TEXTURE_1D;
2046    case TGSI_TEXTURE_2D:
2047    case TGSI_TEXTURE_SHADOW2D:
2048    case TGSI_TEXTURE_2D_MSAA:
2049       return PIPE_TEXTURE_2D;
2050    case TGSI_TEXTURE_3D:
2051       return PIPE_TEXTURE_3D;
2052    case TGSI_TEXTURE_CUBE:
2053    case TGSI_TEXTURE_SHADOWCUBE:
2054       return PIPE_TEXTURE_CUBE;
2055    case TGSI_TEXTURE_RECT:
2056    case TGSI_TEXTURE_SHADOWRECT:
2057       return PIPE_TEXTURE_RECT;
2058    case TGSI_TEXTURE_1D_ARRAY:
2059    case TGSI_TEXTURE_SHADOW1D_ARRAY:
2060       return PIPE_TEXTURE_1D_ARRAY;
2061    case TGSI_TEXTURE_2D_ARRAY:
2062    case TGSI_TEXTURE_SHADOW2D_ARRAY:
2063    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2064       return PIPE_TEXTURE_2D_ARRAY;
2065    case TGSI_TEXTURE_CUBE_ARRAY:
2066    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2067       return PIPE_TEXTURE_CUBE_ARRAY;
2068    default:
2069       assert(0);
2070       return PIPE_BUFFER;
2071    }
2072 }
2073
2074
2075 static enum lp_sampler_lod_property
2076 lp_build_lod_property(
2077    struct lp_build_tgsi_context *bld_base,
2078    const struct tgsi_full_instruction *inst,
2079    unsigned src_op)
2080 {
2081    const struct tgsi_full_src_register *reg = &inst->Src[src_op];
2082    enum lp_sampler_lod_property lod_property;
2083
2084    /*
2085     * Not much we can do here. We could try catching inputs declared
2086     * with constant interpolation but not sure it's worth it - since for
2087     * TEX opcodes as well as FETCH/LD the lod comes from same reg as
2088     * the coords, so it could only work for SAMPLE/TXQ/SVIEWINFO), just
2089     * like the constant/immediate recognition below.
2090     * What seems to be of more value would be to recognize temps holding
2091     * broadcasted scalars but no way we can do it.
2092     * Tried asking llvm but without any success (using LLVMIsConstant
2093     * even though this isn't exactly what we'd need), even as simple as
2094     * IMM[0] UINT32 (0,-1,0,0)
2095     * MOV TEMP[0] IMM[0].yyyy
2096     * SVIEWINFO TEMP[1], TEMP[0].xxxx, SVIEWINFO[0]
2097     * doesn't work.
2098     * This means there's ZERO chance this will ever catch a scalar lod
2099     * with traditional tex opcodes as well as texel fetches, since the lod
2100     * comes from the same reg as coords (except some test shaders using
2101     * constant coords maybe).
2102     * There's at least hope for sample opcodes as well as size queries.
2103     */
2104    if (reg->Register.File == TGSI_FILE_CONSTANT ||
2105        reg->Register.File == TGSI_FILE_IMMEDIATE) {
2106       lod_property = LP_SAMPLER_LOD_SCALAR;
2107    }
2108    else if (bld_base->info->processor == TGSI_PROCESSOR_FRAGMENT) {
2109       if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2110          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2111       }
2112       else {
2113          lod_property = LP_SAMPLER_LOD_PER_QUAD;
2114       }
2115    }
2116    else {
2117       /* never use scalar (per-quad) lod the results are just too wrong. */
2118       lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2119    }
2120    return lod_property;
2121 }
2122
2123
2124 /**
2125  * High-level instruction translators.
2126  */
2127
2128 static void
2129 emit_tex( struct lp_build_tgsi_soa_context *bld,
2130           const struct tgsi_full_instruction *inst,
2131           enum lp_build_tex_modifier modifier,
2132           LLVMValueRef *texel,
2133           unsigned sampler_reg,
2134           enum lp_sampler_op_type sampler_op)
2135 {
2136    unsigned unit = inst->Src[sampler_reg].Register.Index;
2137    LLVMValueRef oow = NULL;
2138    LLVMValueRef lod = NULL;
2139    LLVMValueRef coords[5];
2140    LLVMValueRef offsets[3] = { NULL };
2141    struct lp_derivatives derivs;
2142    struct lp_sampler_params params;
2143    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2144    unsigned num_derivs, num_offsets, i;
2145    unsigned shadow_coord = 0;
2146    unsigned layer_coord = 0;
2147    unsigned sample_key = sampler_op << LP_SAMPLER_OP_TYPE_SHIFT;
2148
2149    memset(&params, 0, sizeof(params));
2150
2151    if (!bld->sampler) {
2152       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2153       for (i = 0; i < 4; i++) {
2154          texel[i] = bld->bld_base.base.undef;
2155       }
2156       return;
2157    }
2158
2159    switch (inst->Texture.Texture) {
2160    case TGSI_TEXTURE_1D_ARRAY:
2161       layer_coord = 1;
2162       /* fallthrough */
2163    case TGSI_TEXTURE_1D:
2164       num_offsets = 1;
2165       num_derivs = 1;
2166       break;
2167    case TGSI_TEXTURE_2D_ARRAY:
2168       layer_coord = 2;
2169       /* fallthrough */
2170    case TGSI_TEXTURE_2D:
2171    case TGSI_TEXTURE_RECT:
2172       num_offsets = 2;
2173       num_derivs = 2;
2174       break;
2175    case TGSI_TEXTURE_SHADOW1D_ARRAY:
2176       layer_coord = 1;
2177       /* fallthrough */
2178    case TGSI_TEXTURE_SHADOW1D:
2179       shadow_coord = 2;
2180       num_offsets = 1;
2181       num_derivs = 1;
2182       break;
2183    case TGSI_TEXTURE_SHADOW2D_ARRAY:
2184       layer_coord = 2;
2185       shadow_coord = 3;
2186       num_offsets = 2;
2187       num_derivs = 2;
2188       break;
2189    case TGSI_TEXTURE_SHADOW2D:
2190    case TGSI_TEXTURE_SHADOWRECT:
2191       shadow_coord = 2;
2192       num_offsets = 2;
2193       num_derivs = 2;
2194       break;
2195    case TGSI_TEXTURE_CUBE:
2196       num_offsets = 2;
2197       num_derivs = 3;
2198       break;
2199    case TGSI_TEXTURE_3D:
2200       num_offsets = 3;
2201       num_derivs = 3;
2202       break;
2203    case TGSI_TEXTURE_SHADOWCUBE:
2204       shadow_coord = 3;
2205       num_offsets = 2;
2206       num_derivs = 3;
2207       break;
2208    case TGSI_TEXTURE_CUBE_ARRAY:
2209       num_offsets = 2;
2210       num_derivs = 3;
2211       layer_coord = 3;
2212       break;
2213    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2214       num_offsets = 2;
2215       num_derivs = 3;
2216       layer_coord = 3;
2217       shadow_coord = 4; /* shadow coord special different reg */
2218       break;
2219    case TGSI_TEXTURE_2D_MSAA:
2220    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2221    default:
2222       assert(0);
2223       return;
2224    }
2225
2226    /* Note lod and especially projected are illegal in a LOT of cases */
2227    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2228        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2229       if (inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
2230           inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY) {
2231          /* note that shadow cube array with bias/explicit lod does not exist */
2232          lod = lp_build_emit_fetch(&bld->bld_base, inst, 1, 0);
2233       }
2234       else {
2235          lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2236       }
2237       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2238          sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
2239       }
2240       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2241          sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2242       }
2243       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2244    }
2245
2246    if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED) {
2247       oow = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2248       oow = lp_build_rcp(&bld->bld_base.base, oow);
2249    }
2250
2251    for (i = 0; i < num_derivs; i++) {
2252       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2253       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2254          coords[i] = lp_build_mul(&bld->bld_base.base, coords[i], oow);
2255    }
2256    for (i = num_derivs; i < 5; i++) {
2257       coords[i] = bld->bld_base.base.undef;
2258    }
2259
2260    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2261    if (layer_coord) {
2262       if (layer_coord == 3) {
2263          coords[3] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2264       }
2265       else {
2266          coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2267       }
2268       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2269          coords[2] = lp_build_mul(&bld->bld_base.base, coords[2], oow);
2270    }
2271    /* Shadow coord occupies always 5th slot. */
2272    if (shadow_coord) {
2273       sample_key |= LP_SAMPLER_SHADOW;
2274       if (shadow_coord == 4) {
2275          coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 1, 0);
2276       }
2277       else {
2278          coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 0, shadow_coord);
2279       }
2280       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2281          coords[4] = lp_build_mul(&bld->bld_base.base, coords[4], oow);
2282    }
2283
2284    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2285       unsigned dim;
2286       sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
2287       for (dim = 0; dim < num_derivs; ++dim) {
2288          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 1, dim);
2289          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 2, dim);
2290       }
2291       params.derivs = &derivs;
2292       /*
2293        * could also check all src regs if constant but I doubt such
2294        * cases exist in practice.
2295        */
2296       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2297          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2298             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2299          }
2300          else {
2301             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2302          }
2303       }
2304       else {
2305          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2306       }
2307    }
2308    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
2309
2310    /* we don't handle the 4 offset version of tg4 */
2311    if (inst->Texture.NumOffsets == 1) {
2312       unsigned dim;
2313       sample_key |= LP_SAMPLER_OFFSETS;
2314       for (dim = 0; dim < num_offsets; dim++) {
2315          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2316       }
2317    }
2318
2319    params.type = bld->bld_base.base.type;
2320    params.sample_key = sample_key;
2321    params.texture_index = unit;
2322    params.sampler_index = unit;
2323    params.context_ptr = bld->context_ptr;
2324    params.coords = coords;
2325    params.offsets = offsets;
2326    params.lod = lod;
2327    params.texel = texel;
2328
2329    bld->sampler->emit_tex_sample(bld->sampler,
2330                                  bld->bld_base.base.gallivm,
2331                                  &params);
2332 }
2333
2334 static void
2335 emit_sample(struct lp_build_tgsi_soa_context *bld,
2336             const struct tgsi_full_instruction *inst,
2337             enum lp_build_tex_modifier modifier,
2338             boolean compare,
2339             LLVMValueRef *texel)
2340 {
2341    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2342    unsigned texture_unit, sampler_unit;
2343    LLVMValueRef lod = NULL;
2344    LLVMValueRef coords[5];
2345    LLVMValueRef offsets[3] = { NULL };
2346    struct lp_derivatives derivs;
2347    struct lp_sampler_params params;
2348    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2349
2350    unsigned num_offsets, num_derivs, i;
2351    unsigned layer_coord = 0;
2352    unsigned sample_key = LP_SAMPLER_OP_TEXTURE << LP_SAMPLER_OP_TYPE_SHIFT;
2353
2354    memset(&params, 0, sizeof(params));
2355
2356    if (!bld->sampler) {
2357       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2358       for (i = 0; i < 4; i++) {
2359          texel[i] = bld->bld_base.base.undef;
2360       }
2361       return;
2362    }
2363
2364    /*
2365     * unlike old-style tex opcodes the texture/sampler indices
2366     * always come from src1 and src2 respectively.
2367     */
2368    texture_unit = inst->Src[1].Register.Index;
2369    sampler_unit = inst->Src[2].Register.Index;
2370
2371    /*
2372     * Note inst->Texture.Texture will contain the number of offsets,
2373     * however the target information is NOT there and comes from the
2374     * declared sampler views instead.
2375     */
2376    switch (bld->sv[texture_unit].Resource) {
2377    case TGSI_TEXTURE_1D:
2378       num_offsets = 1;
2379       num_derivs = 1;
2380       break;
2381    case TGSI_TEXTURE_1D_ARRAY:
2382       layer_coord = 1;
2383       num_offsets = 1;
2384       num_derivs = 1;
2385       break;
2386    case TGSI_TEXTURE_2D:
2387    case TGSI_TEXTURE_RECT:
2388       num_offsets = 2;
2389       num_derivs = 2;
2390       break;
2391    case TGSI_TEXTURE_2D_ARRAY:
2392       layer_coord = 2;
2393       num_offsets = 2;
2394       num_derivs = 2;
2395       break;
2396    case TGSI_TEXTURE_CUBE:
2397       num_offsets = 2;
2398       num_derivs = 3;
2399       break;
2400    case TGSI_TEXTURE_3D:
2401       num_offsets = 3;
2402       num_derivs = 3;
2403       break;
2404    case TGSI_TEXTURE_CUBE_ARRAY:
2405       layer_coord = 3;
2406       num_offsets = 2;
2407       num_derivs = 3;
2408       break;
2409    default:
2410       assert(0);
2411       return;
2412    }
2413
2414    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2415        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2416       lod = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2417       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2418          sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
2419       }
2420       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2421          sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2422       }
2423       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2424    }
2425    else if (modifier == LP_BLD_TEX_MODIFIER_LOD_ZERO) {
2426       /* XXX might be better to explicitly pass the level zero information */
2427       sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2428       lod = lp_build_const_vec(gallivm, bld->bld_base.base.type, 0.0F);
2429    }
2430
2431    for (i = 0; i < num_derivs; i++) {
2432       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2433    }
2434    for (i = num_derivs; i < 5; i++) {
2435       coords[i] = bld->bld_base.base.undef;
2436    }
2437
2438    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2439    if (layer_coord) {
2440       if (layer_coord == 3)
2441          coords[3] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2442       else
2443          coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2444    }
2445    /* Shadow coord occupies always 5th slot. */
2446    if (compare) {
2447       sample_key |= LP_SAMPLER_SHADOW;
2448       coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2449    }
2450
2451    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2452       unsigned dim;
2453       sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
2454       for (dim = 0; dim < num_derivs; ++dim) {
2455          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 3, dim);
2456          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 4, dim);
2457       }
2458       params.derivs = &derivs;
2459       /*
2460        * could also check all src regs if constant but I doubt such
2461        * cases exist in practice.
2462        */
2463       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2464          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2465             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2466          }
2467          else {
2468             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2469          }
2470       }
2471       else {
2472          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2473       }
2474    }
2475
2476    /* some advanced gather instructions (txgo) would require 4 offsets */
2477    if (inst->Texture.NumOffsets == 1) {
2478       unsigned dim;
2479       sample_key |= LP_SAMPLER_OFFSETS;
2480       for (dim = 0; dim < num_offsets; dim++) {
2481          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2482       }
2483    }
2484    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
2485
2486    params.type = bld->bld_base.base.type;
2487    params.sample_key = sample_key;
2488    params.texture_index = texture_unit;
2489    params.sampler_index = sampler_unit;
2490    params.context_ptr = bld->context_ptr;
2491    params.coords = coords;
2492    params.offsets = offsets;
2493    params.lod = lod;
2494    params.texel = texel;
2495
2496    bld->sampler->emit_tex_sample(bld->sampler,
2497                                  bld->bld_base.base.gallivm,
2498                                  &params);
2499
2500    if (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2501        inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2502        inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2503        inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA) {
2504       unsigned char swizzles[4];
2505       swizzles[0] = inst->Src[1].Register.SwizzleX;
2506       swizzles[1] = inst->Src[1].Register.SwizzleY;
2507       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2508       swizzles[3] = inst->Src[1].Register.SwizzleW;
2509
2510       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2511    }
2512 }
2513
2514 static void
2515 emit_fetch_texels( struct lp_build_tgsi_soa_context *bld,
2516                    const struct tgsi_full_instruction *inst,
2517                    LLVMValueRef *texel,
2518                    boolean is_samplei)
2519 {
2520    unsigned unit, target;
2521    LLVMValueRef coord_undef = LLVMGetUndef(bld->bld_base.base.int_vec_type);
2522    LLVMValueRef explicit_lod = NULL;
2523    LLVMValueRef coords[5];
2524    LLVMValueRef offsets[3] = { NULL };
2525    struct lp_sampler_params params;
2526    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2527    unsigned dims, i;
2528    unsigned layer_coord = 0;
2529    unsigned sample_key = LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
2530
2531    memset(&params, 0, sizeof(params));
2532
2533    if (!bld->sampler) {
2534       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2535       for (i = 0; i < 4; i++) {
2536          texel[i] = coord_undef;
2537       }
2538       return;
2539    }
2540
2541    unit = inst->Src[1].Register.Index;
2542
2543    if (is_samplei) {
2544       target = bld->sv[unit].Resource;
2545    }
2546    else {
2547       target = inst->Texture.Texture;
2548    }
2549
2550    switch (target) {
2551    case TGSI_TEXTURE_1D:
2552    case TGSI_TEXTURE_BUFFER:
2553       dims = 1;
2554       break;
2555    case TGSI_TEXTURE_1D_ARRAY:
2556       layer_coord = 1;
2557       dims = 1;
2558       break;
2559    case TGSI_TEXTURE_2D:
2560    case TGSI_TEXTURE_RECT:
2561    case TGSI_TEXTURE_2D_MSAA:
2562       dims = 2;
2563       break;
2564    case TGSI_TEXTURE_2D_ARRAY:
2565    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2566       layer_coord = 2;
2567       dims = 2;
2568       break;
2569    case TGSI_TEXTURE_3D:
2570       dims = 3;
2571       break;
2572    default:
2573       assert(0);
2574       return;
2575    }
2576
2577    /* always have lod except for buffers and msaa targets ? */
2578    if (target != TGSI_TEXTURE_BUFFER &&
2579        target != TGSI_TEXTURE_2D_MSAA &&
2580        target != TGSI_TEXTURE_2D_ARRAY_MSAA) {
2581       sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2582       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2583       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2584    }
2585    /* XXX: for real msaa support, the w component would be the sample index. */
2586
2587    for (i = 0; i < dims; i++) {
2588       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2589    }
2590    /* never use more than 3 coords here but emit_fetch_texel copies all 5 anyway */
2591    for (i = dims; i < 5; i++) {
2592       coords[i] = coord_undef;
2593    }
2594    if (layer_coord)
2595       coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2596
2597    if (inst->Texture.NumOffsets == 1) {
2598       unsigned dim;
2599       sample_key |= LP_SAMPLER_OFFSETS;
2600       for (dim = 0; dim < dims; dim++) {
2601          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2602       }
2603    }
2604    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
2605
2606    params.type = bld->bld_base.base.type;
2607    params.sample_key = sample_key;
2608    params.texture_index = unit;
2609    params.sampler_index = unit;
2610    params.context_ptr = bld->context_ptr;
2611    params.coords = coords;
2612    params.offsets = offsets;
2613    params.derivs = NULL;
2614    params.lod = explicit_lod;
2615    params.texel = texel;
2616
2617    bld->sampler->emit_tex_sample(bld->sampler,
2618                                  bld->bld_base.base.gallivm,
2619                                  &params);
2620
2621    if (is_samplei &&
2622        (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2623         inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2624         inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2625         inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA)) {
2626       unsigned char swizzles[4];
2627       swizzles[0] = inst->Src[1].Register.SwizzleX;
2628       swizzles[1] = inst->Src[1].Register.SwizzleY;
2629       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2630       swizzles[3] = inst->Src[1].Register.SwizzleW;
2631
2632       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2633    }
2634 }
2635
2636 static void
2637 emit_size_query( struct lp_build_tgsi_soa_context *bld,
2638                  const struct tgsi_full_instruction *inst,
2639                  LLVMValueRef *sizes_out,
2640                  boolean is_sviewinfo)
2641 {
2642    LLVMValueRef explicit_lod;
2643    enum lp_sampler_lod_property lod_property;
2644    unsigned has_lod;
2645    unsigned i;
2646    unsigned unit = inst->Src[1].Register.Index;
2647    unsigned target, pipe_target;
2648
2649    if (is_sviewinfo) {
2650       target = bld->sv[unit].Resource;
2651    }
2652    else {
2653       target = inst->Texture.Texture;
2654    }
2655    switch (target) {
2656    case TGSI_TEXTURE_BUFFER:
2657    case TGSI_TEXTURE_RECT:
2658    case TGSI_TEXTURE_SHADOWRECT:
2659       has_lod = 0;
2660       break;
2661    default:
2662       has_lod = 1;
2663       break;
2664    }
2665
2666    if (!bld->sampler) {
2667       _debug_printf("warning: found texture query instruction but no sampler generator supplied\n");
2668       for (i = 0; i < 4; i++)
2669          sizes_out[i] = bld->bld_base.int_bld.undef;
2670       return;
2671    }
2672
2673    if (has_lod) {
2674       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 0);
2675       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2676    }
2677    else {
2678       explicit_lod = NULL;
2679       lod_property = LP_SAMPLER_LOD_SCALAR;
2680    }
2681
2682
2683    pipe_target = tgsi_to_pipe_tex_target(target);
2684
2685    bld->sampler->emit_size_query(bld->sampler,
2686                                  bld->bld_base.base.gallivm,
2687                                  bld->bld_base.int_bld.type,
2688                                  unit, pipe_target,
2689                                  bld->context_ptr,
2690                                  TRUE,
2691                                  lod_property,
2692                                  explicit_lod,
2693                                  sizes_out);
2694 }
2695
2696 static boolean
2697 near_end_of_shader(struct lp_build_tgsi_soa_context *bld,
2698                    int pc)
2699 {
2700    int i;
2701
2702    for (i = 0; i < 5; i++) {
2703       unsigned opcode;
2704
2705       if (pc + i >= bld->bld_base.info->num_instructions)
2706          return TRUE;
2707
2708       opcode = bld->bld_base.instructions[pc + i].Instruction.Opcode;
2709
2710       if (opcode == TGSI_OPCODE_END)
2711          return TRUE;
2712
2713       if (opcode == TGSI_OPCODE_TEX ||
2714          opcode == TGSI_OPCODE_TXP ||
2715          opcode == TGSI_OPCODE_TXD ||
2716          opcode == TGSI_OPCODE_TXB ||
2717          opcode == TGSI_OPCODE_TXL ||
2718          opcode == TGSI_OPCODE_TXF ||
2719          opcode == TGSI_OPCODE_TXQ ||
2720          opcode == TGSI_OPCODE_TEX2 ||
2721          opcode == TGSI_OPCODE_TXB2 ||
2722          opcode == TGSI_OPCODE_TXL2 ||
2723          opcode == TGSI_OPCODE_SAMPLE ||
2724          opcode == TGSI_OPCODE_SAMPLE_B ||
2725          opcode == TGSI_OPCODE_SAMPLE_C ||
2726          opcode == TGSI_OPCODE_SAMPLE_C_LZ ||
2727          opcode == TGSI_OPCODE_SAMPLE_D ||
2728          opcode == TGSI_OPCODE_SAMPLE_I ||
2729          opcode == TGSI_OPCODE_SAMPLE_L ||
2730          opcode == TGSI_OPCODE_SVIEWINFO ||
2731          opcode == TGSI_OPCODE_CAL ||
2732          opcode == TGSI_OPCODE_CALLNZ ||
2733          opcode == TGSI_OPCODE_IF ||
2734          opcode == TGSI_OPCODE_UIF ||
2735          opcode == TGSI_OPCODE_BGNLOOP ||
2736          opcode == TGSI_OPCODE_SWITCH)
2737          return FALSE;
2738    }
2739
2740    return TRUE;
2741 }
2742
2743
2744
2745 /**
2746  * Kill fragment if any of the src register values are negative.
2747  */
2748 static void
2749 emit_kill_if(
2750    struct lp_build_tgsi_soa_context *bld,
2751    const struct tgsi_full_instruction *inst,
2752    int pc)
2753 {
2754    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2755    const struct tgsi_full_src_register *reg = &inst->Src[0];
2756    LLVMValueRef terms[TGSI_NUM_CHANNELS];
2757    LLVMValueRef mask;
2758    unsigned chan_index;
2759
2760    memset(&terms, 0, sizeof terms);
2761
2762    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2763       unsigned swizzle;
2764
2765       /* Unswizzle channel */
2766       swizzle = tgsi_util_get_full_src_register_swizzle( reg, chan_index );
2767
2768       /* Check if the component has not been already tested. */
2769       assert(swizzle < TGSI_NUM_CHANNELS);
2770       if( !terms[swizzle] )
2771          /* TODO: change the comparison operator instead of setting the sign */
2772          terms[swizzle] =  lp_build_emit_fetch(&bld->bld_base, inst, 0, chan_index );
2773    }
2774
2775    mask = NULL;
2776    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2777       if(terms[chan_index]) {
2778          LLVMValueRef chan_mask;
2779
2780          /*
2781           * If term < 0 then mask = 0 else mask = ~0.
2782           */
2783          chan_mask = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GEQUAL, terms[chan_index], bld->bld_base.base.zero);
2784
2785          if(mask)
2786             mask = LLVMBuildAnd(builder, mask, chan_mask, "");
2787          else
2788             mask = chan_mask;
2789       }
2790    }
2791
2792    if (bld->exec_mask.has_mask) {
2793       LLVMValueRef invmask;
2794       invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2795       mask = LLVMBuildOr(builder, mask, invmask, "");
2796    }
2797
2798    lp_build_mask_update(bld->mask, mask);
2799    if (!near_end_of_shader(bld, pc))
2800       lp_build_mask_check(bld->mask);
2801 }
2802
2803
2804 /**
2805  * Unconditional fragment kill.
2806  * The only predication is the execution mask which will apply if
2807  * we're inside a loop or conditional.
2808  */
2809 static void
2810 emit_kill(struct lp_build_tgsi_soa_context *bld,
2811           int pc)
2812 {
2813    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2814    LLVMValueRef mask;
2815
2816    /* For those channels which are "alive", disable fragment shader
2817     * execution.
2818     */
2819    if (bld->exec_mask.has_mask) {
2820       mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2821    }
2822    else {
2823       LLVMValueRef zero = LLVMConstNull(bld->bld_base.base.int_vec_type);
2824       mask = zero;
2825    }
2826
2827    lp_build_mask_update(bld->mask, mask);
2828
2829    if (!near_end_of_shader(bld, pc))
2830       lp_build_mask_check(bld->mask);
2831 }
2832
2833
2834 /**
2835  * Emit code which will dump the value of all the temporary registers
2836  * to stdout.
2837  */
2838 static void
2839 emit_dump_file(struct lp_build_tgsi_soa_context *bld,
2840                unsigned file)
2841 {
2842    const struct tgsi_shader_info *info = bld->bld_base.info;
2843    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2844    LLVMBuilderRef builder = gallivm->builder;
2845    LLVMValueRef reg_ptr;
2846    int index;
2847    int max_index = info->file_max[file];
2848
2849    /*
2850     * Some register files, particularly constants, can be very large,
2851     * and dumping everything could make this unusably slow.
2852     */
2853    max_index = MIN2(max_index, 32);
2854
2855    for (index = 0; index <= max_index; index++) {
2856       LLVMValueRef res;
2857       unsigned mask;
2858       int chan;
2859
2860       if (index < 8 * sizeof(unsigned) &&
2861           (info->file_mask[file] & (1 << index)) == 0)  {
2862          /* This was not declared.*/
2863          continue;
2864       }
2865
2866       if (file == TGSI_FILE_INPUT) {
2867          mask = info->input_usage_mask[index];
2868       } else {
2869          mask = TGSI_WRITEMASK_XYZW;
2870       }
2871
2872       for (chan = 0; chan < 4; chan++) {
2873          if ((mask & (1 << chan)) == 0) {
2874             /* This channel is not used.*/
2875             continue;
2876          }
2877
2878          if (file == TGSI_FILE_CONSTANT) {
2879             struct tgsi_full_src_register reg;
2880             memset(&reg, 0, sizeof reg);
2881             reg.Register.File = file;
2882             reg.Register.Index = index;
2883             reg.Register.SwizzleX = 0;
2884             reg.Register.SwizzleY = 1;
2885             reg.Register.SwizzleZ = 2;
2886             reg.Register.SwizzleW = 3;
2887
2888             res = bld->bld_base.emit_fetch_funcs[file](&bld->bld_base, &reg, TGSI_TYPE_FLOAT, chan);
2889             if (!res) {
2890                continue;
2891             }
2892          } else if (file == TGSI_FILE_INPUT) {
2893             res = bld->inputs[index][chan];
2894             if (!res) {
2895                continue;
2896             }
2897          } else if (file == TGSI_FILE_TEMPORARY) {
2898             reg_ptr = lp_get_temp_ptr_soa(bld, index, chan);
2899             assert(reg_ptr);
2900             res = LLVMBuildLoad(builder, reg_ptr, "");
2901          } else if (file == TGSI_FILE_OUTPUT) {
2902             reg_ptr = lp_get_output_ptr(bld, index, chan);
2903             assert(reg_ptr);
2904             res = LLVMBuildLoad(builder, reg_ptr, "");
2905          } else {
2906             assert(0);
2907             continue;
2908          }
2909
2910          emit_dump_reg(gallivm, file, index, chan, res);
2911       }
2912    }
2913 }
2914
2915
2916
2917 void
2918 lp_emit_declaration_soa(
2919    struct lp_build_tgsi_context *bld_base,
2920    const struct tgsi_full_declaration *decl)
2921 {
2922    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2923    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2924    LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
2925    const unsigned first = decl->Range.First;
2926    const unsigned last = decl->Range.Last;
2927    unsigned idx, i;
2928
2929    assert(last <= bld->bld_base.info->file_max[decl->Declaration.File]);
2930
2931    switch (decl->Declaration.File) {
2932    case TGSI_FILE_TEMPORARY:
2933       if (!(bld->indirect_files & (1 << TGSI_FILE_TEMPORARY))) {
2934          assert(last < LP_MAX_INLINED_TEMPS);
2935          for (idx = first; idx <= last; ++idx) {
2936             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2937                bld->temps[idx][i] = lp_build_alloca(gallivm, vec_type, "temp");
2938          }
2939       }
2940       break;
2941
2942    case TGSI_FILE_OUTPUT:
2943       if (!(bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
2944          for (idx = first; idx <= last; ++idx) {
2945             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2946                bld->outputs[idx][i] = lp_build_alloca(gallivm,
2947                                                       vec_type, "output");
2948          }
2949       }
2950       break;
2951
2952    case TGSI_FILE_ADDRESS:
2953       /* ADDR registers are only allocated with an integer LLVM IR type,
2954        * as they are guaranteed to always have integers.
2955        * XXX: Not sure if this exception is worthwhile (or the whole idea of
2956        * an ADDR register for that matter).
2957        */
2958       assert(last < LP_MAX_TGSI_ADDRS);
2959       for (idx = first; idx <= last; ++idx) {
2960          assert(idx < LP_MAX_TGSI_ADDRS);
2961          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2962             bld->addr[idx][i] = lp_build_alloca(gallivm, bld_base->base.int_vec_type, "addr");
2963       }
2964       break;
2965
2966    case TGSI_FILE_PREDICATE:
2967       assert(last < LP_MAX_TGSI_PREDS);
2968       for (idx = first; idx <= last; ++idx) {
2969          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2970             bld->preds[idx][i] = lp_build_alloca(gallivm, vec_type,
2971                                                  "predicate");
2972       }
2973       break;
2974
2975    case TGSI_FILE_SAMPLER_VIEW:
2976       /*
2977        * The target stored here MUST match whatever there actually
2978        * is in the set sampler views (what about return type?).
2979        */
2980       assert(last < PIPE_MAX_SHADER_SAMPLER_VIEWS);
2981       for (idx = first; idx <= last; ++idx) {
2982          bld->sv[idx] = decl->SamplerView;
2983       }
2984       break;
2985
2986    case TGSI_FILE_CONSTANT:
2987    {
2988       /*
2989        * We could trivially fetch the per-buffer pointer when fetching the
2990        * constant, relying on llvm to figure out it's always the same pointer
2991        * anyway. However, doing so results in a huge (more than factor of 10)
2992        * slowdown in llvm compilation times for some (but not all) shaders
2993        * (more specifically, the IR optimization spends way more time in
2994        * DominatorTree::dominates). At least with llvm versions 3.1, 3.3.
2995        */
2996       unsigned idx2D = decl->Dim.Index2D;
2997       LLVMValueRef index2D = lp_build_const_int32(gallivm, idx2D);
2998       assert(idx2D < LP_MAX_TGSI_CONST_BUFFERS);
2999       bld->consts[idx2D] =
3000          lp_build_array_get(gallivm, bld->consts_ptr, index2D);
3001       bld->consts_sizes[idx2D] =
3002          lp_build_array_get(gallivm, bld->const_sizes_ptr, index2D);
3003    }
3004       break;
3005
3006    default:
3007       /* don't need to declare other vars */
3008       break;
3009    }
3010 }
3011
3012
3013 void lp_emit_immediate_soa(
3014    struct lp_build_tgsi_context *bld_base,
3015    const struct tgsi_full_immediate *imm)
3016 {
3017    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
3018    struct gallivm_state * gallivm = bld_base->base.gallivm;
3019    LLVMValueRef imms[4];
3020    unsigned i;
3021    const uint size = imm->Immediate.NrTokens - 1;
3022    assert(size <= 4);
3023    switch (imm->Immediate.DataType) {
3024    case TGSI_IMM_FLOAT32:
3025       for( i = 0; i < size; ++i )
3026          imms[i] =
3027                lp_build_const_vec(gallivm, bld_base->base.type, imm->u[i].Float);
3028
3029       break;
3030    case TGSI_IMM_FLOAT64:
3031    case TGSI_IMM_UINT32:
3032       for( i = 0; i < size; ++i ) {
3033          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->uint_bld.type, imm->u[i].Uint);
3034          imms[i] = LLVMConstBitCast(tmp, bld_base->base.vec_type);
3035       }
3036
3037       break;
3038    case TGSI_IMM_INT32:
3039       for( i = 0; i < size; ++i ) {
3040          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->int_bld.type, imm->u[i].Int);
3041          imms[i] = LLVMConstBitCast(tmp, bld_base->base.vec_type);
3042       }
3043
3044       break;
3045    }
3046    for( i = size; i < 4; ++i )
3047       imms[i] = bld_base->base.undef;
3048
3049    if (bld->use_immediates_array) {
3050       unsigned index = bld->num_immediates;
3051       struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
3052       LLVMBuilderRef builder = gallivm->builder;
3053
3054       assert(bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE));
3055       for (i = 0; i < 4; ++i ) {
3056          LLVMValueRef lindex = lp_build_const_int32(
3057                   bld->bld_base.base.gallivm, index * 4 + i);
3058          LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
3059                                              bld->imms_array, &lindex, 1, "");
3060          LLVMBuildStore(builder, imms[i], imm_ptr);
3061       }
3062    } else {
3063       /* simply copy the immediate values into the next immediates[] slot */
3064       unsigned i;
3065       const uint size = imm->Immediate.NrTokens - 1;
3066       assert(size <= 4);
3067       assert(bld->num_immediates < LP_MAX_INLINED_IMMEDIATES);
3068
3069       for(i = 0; i < 4; ++i )
3070          bld->immediates[bld->num_immediates][i] = imms[i];
3071
3072       if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
3073          unsigned index = bld->num_immediates;
3074          struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
3075          LLVMBuilderRef builder = gallivm->builder;
3076          for (i = 0; i < 4; ++i ) {
3077             LLVMValueRef lindex = lp_build_const_int32(
3078                      bld->bld_base.base.gallivm, index * 4 + i);
3079             LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
3080                                                 bld->imms_array, &lindex, 1, "");
3081             LLVMBuildStore(builder,
3082                            bld->immediates[index][i],
3083                            imm_ptr);
3084          }
3085       }
3086    }
3087
3088    bld->num_immediates++;
3089 }
3090
3091 static void
3092 ddx_emit(
3093    const struct lp_build_tgsi_action * action,
3094    struct lp_build_tgsi_context * bld_base,
3095    struct lp_build_emit_data * emit_data)
3096 {
3097    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3098
3099    emit_fetch_deriv(bld, emit_data->args[0], NULL,
3100                     &emit_data->output[emit_data->chan], NULL);
3101 }
3102
3103 static void
3104 ddy_emit(
3105    const struct lp_build_tgsi_action * action,
3106    struct lp_build_tgsi_context * bld_base,
3107    struct lp_build_emit_data * emit_data)
3108 {
3109    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3110
3111    emit_fetch_deriv(bld, emit_data->args[0], NULL, NULL,
3112                     &emit_data->output[emit_data->chan]);
3113 }
3114
3115 static void
3116 kill_emit(
3117    const struct lp_build_tgsi_action * action,
3118    struct lp_build_tgsi_context * bld_base,
3119    struct lp_build_emit_data * emit_data)
3120 {
3121    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3122
3123    emit_kill(bld, bld_base->pc - 1);
3124 }
3125
3126 static void
3127 kill_if_emit(
3128    const struct lp_build_tgsi_action * action,
3129    struct lp_build_tgsi_context * bld_base,
3130    struct lp_build_emit_data * emit_data)
3131 {
3132    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3133
3134    emit_kill_if(bld, emit_data->inst, bld_base->pc - 1);
3135 }
3136
3137 static void
3138 tex_emit(
3139    const struct lp_build_tgsi_action * action,
3140    struct lp_build_tgsi_context * bld_base,
3141    struct lp_build_emit_data * emit_data)
3142 {
3143    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3144
3145    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3146             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3147 }
3148
3149 static void
3150 tex2_emit(
3151    const struct lp_build_tgsi_action * action,
3152    struct lp_build_tgsi_context * bld_base,
3153    struct lp_build_emit_data * emit_data)
3154 {
3155    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3156
3157    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3158             emit_data->output, 2, LP_SAMPLER_OP_TEXTURE);
3159 }
3160
3161 static void
3162 txb_emit(
3163    const struct lp_build_tgsi_action * action,
3164    struct lp_build_tgsi_context * bld_base,
3165    struct lp_build_emit_data * emit_data)
3166 {
3167    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3168
3169    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
3170             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3171 }
3172
3173 static void
3174 txb2_emit(
3175    const struct lp_build_tgsi_action * action,
3176    struct lp_build_tgsi_context * bld_base,
3177    struct lp_build_emit_data * emit_data)
3178 {
3179    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3180
3181    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
3182             emit_data->output, 2, LP_SAMPLER_OP_TEXTURE);
3183 }
3184
3185 static void
3186 txd_emit(
3187    const struct lp_build_tgsi_action * action,
3188    struct lp_build_tgsi_context * bld_base,
3189    struct lp_build_emit_data * emit_data)
3190 {
3191    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3192
3193    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
3194             emit_data->output, 3, LP_SAMPLER_OP_TEXTURE);
3195 }
3196
3197 static void
3198 txl_emit(
3199    const struct lp_build_tgsi_action * action,
3200    struct lp_build_tgsi_context * bld_base,
3201    struct lp_build_emit_data * emit_data)
3202 {
3203    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3204
3205    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3206             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3207 }
3208
3209 static void
3210 txl2_emit(
3211    const struct lp_build_tgsi_action * action,
3212    struct lp_build_tgsi_context * bld_base,
3213    struct lp_build_emit_data * emit_data)
3214 {
3215    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3216
3217    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3218             emit_data->output, 2, LP_SAMPLER_OP_TEXTURE);
3219 }
3220
3221 static void
3222 txp_emit(
3223    const struct lp_build_tgsi_action * action,
3224    struct lp_build_tgsi_context * bld_base,
3225    struct lp_build_emit_data * emit_data)
3226 {
3227    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3228
3229    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_PROJECTED,
3230             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3231 }
3232
3233 static void
3234 tg4_emit(
3235    const struct lp_build_tgsi_action * action,
3236    struct lp_build_tgsi_context * bld_base,
3237    struct lp_build_emit_data * emit_data)
3238 {
3239    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3240
3241    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3242             emit_data->output, 2, LP_SAMPLER_OP_GATHER);
3243 }
3244
3245 static void
3246 txq_emit(
3247    const struct lp_build_tgsi_action * action,
3248    struct lp_build_tgsi_context * bld_base,
3249    struct lp_build_emit_data * emit_data)
3250 {
3251    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3252
3253    emit_size_query(bld, emit_data->inst, emit_data->output, FALSE);
3254 }
3255
3256 static void
3257 txf_emit(
3258    const struct lp_build_tgsi_action * action,
3259    struct lp_build_tgsi_context * bld_base,
3260    struct lp_build_emit_data * emit_data)
3261 {
3262    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3263
3264    emit_fetch_texels(bld, emit_data->inst, emit_data->output, FALSE);
3265 }
3266
3267 static void
3268 sample_i_emit(
3269    const struct lp_build_tgsi_action * action,
3270    struct lp_build_tgsi_context * bld_base,
3271    struct lp_build_emit_data * emit_data)
3272 {
3273    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3274
3275    emit_fetch_texels(bld, emit_data->inst, emit_data->output, TRUE);
3276 }
3277
3278 static void
3279 sample_emit(
3280    const struct lp_build_tgsi_action * action,
3281    struct lp_build_tgsi_context * bld_base,
3282    struct lp_build_emit_data * emit_data)
3283 {
3284    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3285
3286    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3287                FALSE, emit_data->output);
3288 }
3289
3290 static void
3291 sample_b_emit(
3292    const struct lp_build_tgsi_action * action,
3293    struct lp_build_tgsi_context * bld_base,
3294    struct lp_build_emit_data * emit_data)
3295 {
3296    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3297
3298    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
3299                FALSE, emit_data->output);
3300 }
3301
3302 static void
3303 sample_c_emit(
3304    const struct lp_build_tgsi_action * action,
3305    struct lp_build_tgsi_context * bld_base,
3306    struct lp_build_emit_data * emit_data)
3307 {
3308    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3309
3310    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3311                TRUE, emit_data->output);
3312 }
3313
3314 static void
3315 sample_c_lz_emit(
3316    const struct lp_build_tgsi_action * action,
3317    struct lp_build_tgsi_context * bld_base,
3318    struct lp_build_emit_data * emit_data)
3319 {
3320    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3321
3322    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_ZERO,
3323                TRUE, emit_data->output);
3324 }
3325
3326 static void
3327 sample_d_emit(
3328    const struct lp_build_tgsi_action * action,
3329    struct lp_build_tgsi_context * bld_base,
3330    struct lp_build_emit_data * emit_data)
3331 {
3332    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3333
3334    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
3335                FALSE, emit_data->output);
3336 }
3337
3338 static void
3339 sample_l_emit(
3340    const struct lp_build_tgsi_action * action,
3341    struct lp_build_tgsi_context * bld_base,
3342    struct lp_build_emit_data * emit_data)
3343 {
3344    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3345
3346    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3347                FALSE, emit_data->output);
3348 }
3349
3350 static void
3351 sviewinfo_emit(
3352    const struct lp_build_tgsi_action * action,
3353    struct lp_build_tgsi_context * bld_base,
3354    struct lp_build_emit_data * emit_data)
3355 {
3356    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3357
3358    emit_size_query(bld, emit_data->inst, emit_data->output, TRUE);
3359 }
3360
3361 static LLVMValueRef
3362 mask_vec(struct lp_build_tgsi_context *bld_base)
3363 {
3364    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3365    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3366    struct lp_exec_mask *exec_mask = &bld->exec_mask;
3367
3368    if (!exec_mask->has_mask) {
3369       return lp_build_mask_value(bld->mask);
3370    }
3371    return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
3372                        exec_mask->exec_mask, "");
3373 }
3374
3375 static void
3376 increment_vec_ptr_by_mask(struct lp_build_tgsi_context * bld_base,
3377                           LLVMValueRef ptr,
3378                           LLVMValueRef mask)
3379 {
3380    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3381    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3382
3383    current_vec = LLVMBuildSub(builder, current_vec, mask, "");
3384
3385    LLVMBuildStore(builder, current_vec, ptr);
3386 }
3387
3388 static void
3389 clear_uint_vec_ptr_from_mask(struct lp_build_tgsi_context * bld_base,
3390                              LLVMValueRef ptr,
3391                              LLVMValueRef mask)
3392 {
3393    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3394    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3395
3396    current_vec = lp_build_select(&bld_base->uint_bld,
3397                                  mask,
3398                                  bld_base->uint_bld.zero,
3399                                  current_vec);
3400
3401    LLVMBuildStore(builder, current_vec, ptr);
3402 }
3403
3404 static LLVMValueRef
3405 clamp_mask_to_max_output_vertices(struct lp_build_tgsi_soa_context * bld,
3406                                   LLVMValueRef current_mask_vec,
3407                                   LLVMValueRef total_emitted_vertices_vec)
3408 {
3409    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3410    struct lp_build_context *int_bld = &bld->bld_base.int_bld;
3411    LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
3412                                         total_emitted_vertices_vec,
3413                                         bld->max_output_vertices_vec);
3414
3415    return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
3416 }
3417
3418 static void
3419 emit_vertex(
3420    const struct lp_build_tgsi_action * action,
3421    struct lp_build_tgsi_context * bld_base,
3422    struct lp_build_emit_data * emit_data)
3423 {
3424    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3425    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3426
3427    if (bld->gs_iface->emit_vertex) {
3428       LLVMValueRef mask = mask_vec(bld_base);
3429       LLVMValueRef total_emitted_vertices_vec =
3430          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3431       mask = clamp_mask_to_max_output_vertices(bld, mask,
3432                                                total_emitted_vertices_vec);
3433       gather_outputs(bld);
3434       bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base,
3435                                  bld->outputs,
3436                                  total_emitted_vertices_vec);
3437       increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr,
3438                                 mask);
3439       increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr,
3440                                 mask);
3441 #if DUMP_GS_EMITS
3442       lp_build_print_value(bld->bld_base.base.gallivm,
3443                            " +++ emit vertex masked ones = ",
3444                            mask);
3445       lp_build_print_value(bld->bld_base.base.gallivm,
3446                            " +++ emit vertex emitted = ",
3447                            total_emitted_vertices_vec);
3448 #endif
3449    }
3450 }
3451
3452
3453 static void
3454 end_primitive_masked(struct lp_build_tgsi_context * bld_base,
3455                      LLVMValueRef mask)
3456 {
3457    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3458    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3459
3460    if (bld->gs_iface->end_primitive) {
3461       struct lp_build_context *uint_bld = &bld_base->uint_bld;
3462       LLVMValueRef emitted_vertices_vec =
3463          LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr, "");
3464       LLVMValueRef emitted_prims_vec =
3465          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3466
3467       LLVMValueRef emitted_mask = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3468                                                emitted_vertices_vec,
3469                                                uint_bld->zero);
3470       /* We need to combine the current execution mask with the mask
3471          telling us which, if any, execution slots actually have
3472          unemitted primitives, this way we make sure that end_primitives
3473          executes only on the paths that have unflushed vertices */
3474       mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
3475
3476       bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base,
3477                                    emitted_vertices_vec,
3478                                    emitted_prims_vec);
3479
3480 #if DUMP_GS_EMITS
3481       lp_build_print_value(bld->bld_base.base.gallivm,
3482                            " +++ end prim masked ones = ",
3483                            mask);
3484       lp_build_print_value(bld->bld_base.base.gallivm,
3485                            " +++ end prim emitted verts1 = ",
3486                            emitted_vertices_vec);
3487       lp_build_print_value(bld->bld_base.base.gallivm,
3488                            " +++ end prim emitted prims1 = ",
3489                            LLVMBuildLoad(builder,
3490                                          bld->emitted_prims_vec_ptr, ""));
3491 #endif
3492       increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr,
3493                                 mask);
3494       clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr,
3495                                    mask);
3496 #if DUMP_GS_EMITS
3497       lp_build_print_value(bld->bld_base.base.gallivm,
3498                            " +++ end prim emitted verts2 = ",
3499                            LLVMBuildLoad(builder,
3500                                          bld->emitted_vertices_vec_ptr, ""));
3501 #endif
3502    }
3503
3504 }
3505
3506 static void
3507 end_primitive(
3508    const struct lp_build_tgsi_action * action,
3509    struct lp_build_tgsi_context * bld_base,
3510    struct lp_build_emit_data * emit_data)
3511 {
3512    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3513
3514    if (bld->gs_iface->end_primitive) {
3515       LLVMValueRef mask = mask_vec(bld_base);
3516       end_primitive_masked(bld_base, mask);
3517    }
3518 }
3519
3520 static void
3521 cal_emit(
3522    const struct lp_build_tgsi_action * action,
3523    struct lp_build_tgsi_context * bld_base,
3524    struct lp_build_emit_data * emit_data)
3525 {
3526    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3527
3528    lp_exec_mask_call(&bld->exec_mask, emit_data->inst->Label.Label,
3529                      &bld_base->pc);
3530 }
3531
3532 static void
3533 ret_emit(
3534    const struct lp_build_tgsi_action * action,
3535    struct lp_build_tgsi_context * bld_base,
3536    struct lp_build_emit_data * emit_data)
3537 {
3538    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3539
3540    lp_exec_mask_ret(&bld->exec_mask, &bld_base->pc);
3541 }
3542
3543 static void
3544 brk_emit(
3545    const struct lp_build_tgsi_action * action,
3546    struct lp_build_tgsi_context * bld_base,
3547    struct lp_build_emit_data * emit_data)
3548 {
3549    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3550
3551    lp_exec_break(&bld->exec_mask, bld_base);
3552 }
3553
3554 static void
3555 breakc_emit(
3556    const struct lp_build_tgsi_action * action,
3557    struct lp_build_tgsi_context * bld_base,
3558    struct lp_build_emit_data * emit_data)
3559 {
3560    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3561    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3562    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3563    LLVMValueRef unsigned_cond =
3564       LLVMBuildBitCast(builder, emit_data->args[0], uint_bld->vec_type, "");
3565    LLVMValueRef cond = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3566                                     unsigned_cond,
3567                                     uint_bld->zero);
3568
3569    lp_exec_break_condition(&bld->exec_mask, cond);
3570 }
3571
3572 static void
3573 if_emit(
3574    const struct lp_build_tgsi_action * action,
3575    struct lp_build_tgsi_context * bld_base,
3576    struct lp_build_emit_data * emit_data)
3577 {
3578    LLVMValueRef tmp;
3579    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3580
3581    tmp = lp_build_cmp(&bld_base->base, PIPE_FUNC_NOTEQUAL,
3582                       emit_data->args[0], bld->bld_base.base.zero);
3583    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3584 }
3585
3586 static void
3587 uif_emit(
3588    const struct lp_build_tgsi_action * action,
3589    struct lp_build_tgsi_context * bld_base,
3590    struct lp_build_emit_data * emit_data)
3591 {
3592    LLVMValueRef tmp;
3593    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3594    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3595
3596    tmp = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3597                       emit_data->args[0], uint_bld->zero);
3598    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3599 }
3600
3601 static void
3602 case_emit(
3603    const struct lp_build_tgsi_action * action,
3604    struct lp_build_tgsi_context * bld_base,
3605    struct lp_build_emit_data * emit_data)
3606 {
3607    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3608
3609    lp_exec_case(&bld->exec_mask, emit_data->args[0]);
3610 }
3611
3612 static void
3613 default_emit(
3614    const struct lp_build_tgsi_action * action,
3615    struct lp_build_tgsi_context * bld_base,
3616    struct lp_build_emit_data * emit_data)
3617 {
3618    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3619
3620    lp_exec_default(&bld->exec_mask, bld_base);
3621 }
3622
3623 static void
3624 switch_emit(
3625    const struct lp_build_tgsi_action * action,
3626    struct lp_build_tgsi_context * bld_base,
3627    struct lp_build_emit_data * emit_data)
3628 {
3629    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3630
3631    lp_exec_switch(&bld->exec_mask, emit_data->args[0]);
3632 }
3633
3634 static void
3635 endswitch_emit(
3636    const struct lp_build_tgsi_action * action,
3637    struct lp_build_tgsi_context * bld_base,
3638    struct lp_build_emit_data * emit_data)
3639 {
3640    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3641
3642    lp_exec_endswitch(&bld->exec_mask, bld_base);
3643 }
3644
3645 static void
3646 bgnloop_emit(
3647    const struct lp_build_tgsi_action * action,
3648    struct lp_build_tgsi_context * bld_base,
3649    struct lp_build_emit_data * emit_data)
3650 {
3651    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3652
3653    lp_exec_bgnloop(&bld->exec_mask);
3654 }
3655
3656 static void
3657 bgnsub_emit(
3658    const struct lp_build_tgsi_action * action,
3659    struct lp_build_tgsi_context * bld_base,
3660    struct lp_build_emit_data * emit_data)
3661 {
3662    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3663
3664    lp_exec_mask_bgnsub(&bld->exec_mask);
3665 }
3666
3667 static void
3668 else_emit(
3669    const struct lp_build_tgsi_action * action,
3670    struct lp_build_tgsi_context * bld_base,
3671    struct lp_build_emit_data * emit_data)
3672 {
3673    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3674
3675    lp_exec_mask_cond_invert(&bld->exec_mask);
3676 }
3677
3678 static void
3679 endif_emit(
3680    const struct lp_build_tgsi_action * action,
3681    struct lp_build_tgsi_context * bld_base,
3682    struct lp_build_emit_data * emit_data)
3683 {
3684    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3685
3686    lp_exec_mask_cond_pop(&bld->exec_mask);
3687 }
3688
3689 static void
3690 endloop_emit(
3691    const struct lp_build_tgsi_action * action,
3692    struct lp_build_tgsi_context * bld_base,
3693    struct lp_build_emit_data * emit_data)
3694 {
3695    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3696
3697    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
3698 }
3699
3700 static void
3701 endsub_emit(
3702    const struct lp_build_tgsi_action * action,
3703    struct lp_build_tgsi_context * bld_base,
3704    struct lp_build_emit_data * emit_data)
3705 {
3706    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3707
3708    lp_exec_mask_endsub(&bld->exec_mask, &bld_base->pc);
3709 }
3710
3711 static void
3712 cont_emit(
3713    const struct lp_build_tgsi_action * action,
3714    struct lp_build_tgsi_context * bld_base,
3715    struct lp_build_emit_data * emit_data)
3716 {
3717    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3718
3719    lp_exec_continue(&bld->exec_mask);
3720 }
3721
3722 static void emit_prologue(struct lp_build_tgsi_context * bld_base)
3723 {
3724    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3725    struct gallivm_state * gallivm = bld_base->base.gallivm;
3726
3727    if (bld->indirect_files & (1 << TGSI_FILE_TEMPORARY)) {
3728       LLVMValueRef array_size =
3729          lp_build_const_int32(gallivm,
3730                          bld_base->info->file_max[TGSI_FILE_TEMPORARY] * 4 + 4);
3731       bld->temps_array = lp_build_array_alloca(gallivm,
3732                                               bld_base->base.vec_type, array_size,
3733                                               "temp_array");
3734    }
3735
3736    if (bld->indirect_files & (1 << TGSI_FILE_OUTPUT)) {
3737       LLVMValueRef array_size =
3738          lp_build_const_int32(gallivm,
3739                             bld_base->info->file_max[TGSI_FILE_OUTPUT] * 4 + 4);
3740       bld->outputs_array = lp_build_array_alloca(gallivm,
3741                                                 bld_base->base.vec_type, array_size,
3742                                                 "output_array");
3743    }
3744
3745    if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
3746       LLVMValueRef array_size =
3747          lp_build_const_int32(gallivm,
3748                          bld_base->info->file_max[TGSI_FILE_IMMEDIATE] * 4 + 4);
3749       bld->imms_array = lp_build_array_alloca(gallivm,
3750                                               bld_base->base.vec_type, array_size,
3751                                               "imms_array");
3752    }
3753
3754    /* If we have indirect addressing in inputs we need to copy them into
3755     * our alloca array to be able to iterate over them */
3756    if (bld->indirect_files & (1 << TGSI_FILE_INPUT) && !bld->gs_iface) {
3757       unsigned index, chan;
3758       LLVMTypeRef vec_type = bld_base->base.vec_type;
3759       LLVMValueRef array_size = lp_build_const_int32(gallivm,
3760             bld_base->info->file_max[TGSI_FILE_INPUT]*4 + 4);
3761       bld->inputs_array = lp_build_array_alloca(gallivm,
3762                                                vec_type, array_size,
3763                                                "input_array");
3764
3765       assert(bld_base->info->num_inputs
3766                         <= bld_base->info->file_max[TGSI_FILE_INPUT] + 1);
3767
3768       for (index = 0; index < bld_base->info->num_inputs; ++index) {
3769          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3770             LLVMValueRef lindex =
3771                lp_build_const_int32(gallivm, index * 4 + chan);
3772             LLVMValueRef input_ptr =
3773                LLVMBuildGEP(gallivm->builder, bld->inputs_array,
3774                             &lindex, 1, "");
3775             LLVMValueRef value = bld->inputs[index][chan];
3776             if (value)
3777                LLVMBuildStore(gallivm->builder, value, input_ptr);
3778          }
3779       }
3780    }
3781
3782    if (bld->gs_iface) {
3783       struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
3784       bld->emitted_prims_vec_ptr =
3785          lp_build_alloca(gallivm,
3786                          uint_bld->vec_type,
3787                          "emitted_prims_ptr");
3788       bld->emitted_vertices_vec_ptr =
3789          lp_build_alloca(gallivm,
3790                          uint_bld->vec_type,
3791                          "emitted_vertices_ptr");
3792       bld->total_emitted_vertices_vec_ptr =
3793          lp_build_alloca(gallivm,
3794                          uint_bld->vec_type,
3795                          "total_emitted_vertices_ptr");
3796
3797       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3798                      bld->emitted_prims_vec_ptr);
3799       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3800                      bld->emitted_vertices_vec_ptr);
3801       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3802                      bld->total_emitted_vertices_vec_ptr);
3803    }
3804
3805    if (DEBUG_EXECUTION) {
3806       lp_build_printf(gallivm, "\n");
3807       emit_dump_file(bld, TGSI_FILE_CONSTANT);
3808       if (!bld->gs_iface)
3809          emit_dump_file(bld, TGSI_FILE_INPUT);
3810    }
3811 }
3812
3813 static void emit_epilogue(struct lp_build_tgsi_context * bld_base)
3814 {
3815    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3816    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3817
3818    if (DEBUG_EXECUTION) {
3819       /* for debugging */
3820       if (0) {
3821          emit_dump_file(bld, TGSI_FILE_TEMPORARY);
3822       }
3823       emit_dump_file(bld, TGSI_FILE_OUTPUT);
3824       lp_build_printf(bld_base->base.gallivm, "\n");
3825    }
3826
3827    /* If we have indirect addressing in outputs we need to copy our alloca array
3828     * to the outputs slots specified by the caller */
3829    if (bld->gs_iface) {
3830       LLVMValueRef total_emitted_vertices_vec;
3831       LLVMValueRef emitted_prims_vec;
3832       /* implicit end_primitives, needed in case there are any unflushed
3833          vertices in the cache. Note must not call end_primitive here
3834          since the exec_mask is not valid at this point. */
3835       end_primitive_masked(bld_base, lp_build_mask_value(bld->mask));
3836
3837       total_emitted_vertices_vec =
3838          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3839       emitted_prims_vec =
3840          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3841
3842       bld->gs_iface->gs_epilogue(bld->gs_iface,
3843                                  &bld->bld_base,
3844                                  total_emitted_vertices_vec,
3845                                  emitted_prims_vec);
3846    } else {
3847       gather_outputs(bld);
3848    }
3849 }
3850
3851 void
3852 lp_build_tgsi_soa(struct gallivm_state *gallivm,
3853                   const struct tgsi_token *tokens,
3854                   struct lp_type type,
3855                   struct lp_build_mask_context *mask,
3856                   LLVMValueRef consts_ptr,
3857                   LLVMValueRef const_sizes_ptr,
3858                   const struct lp_bld_tgsi_system_values *system_values,
3859                   const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
3860                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
3861                   LLVMValueRef context_ptr,
3862                   struct lp_build_sampler_soa *sampler,
3863                   const struct tgsi_shader_info *info,
3864                   const struct lp_build_tgsi_gs_iface *gs_iface)
3865 {
3866    struct lp_build_tgsi_soa_context bld;
3867
3868    struct lp_type res_type;
3869
3870    assert(type.length <= LP_MAX_VECTOR_LENGTH);
3871    memset(&res_type, 0, sizeof res_type);
3872    res_type.width = type.width;
3873    res_type.length = type.length;
3874    res_type.sign = 1;
3875
3876    /* Setup build context */
3877    memset(&bld, 0, sizeof bld);
3878    lp_build_context_init(&bld.bld_base.base, gallivm, type);
3879    lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
3880    lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
3881    lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
3882    {
3883       struct lp_type dbl_type;
3884       dbl_type = type;
3885       dbl_type.width *= 2;
3886       lp_build_context_init(&bld.bld_base.dbl_bld, gallivm, dbl_type);
3887    }
3888    bld.mask = mask;
3889    bld.inputs = inputs;
3890    bld.outputs = outputs;
3891    bld.consts_ptr = consts_ptr;
3892    bld.const_sizes_ptr = const_sizes_ptr;
3893    bld.sampler = sampler;
3894    bld.bld_base.info = info;
3895    bld.indirect_files = info->indirect_files;
3896    bld.context_ptr = context_ptr;
3897
3898    /*
3899     * If the number of temporaries is rather large then we just
3900     * allocate them as an array right from the start and treat
3901     * like indirect temporaries.
3902     */
3903    if (info->file_max[TGSI_FILE_TEMPORARY] >= LP_MAX_INLINED_TEMPS) {
3904       bld.indirect_files |= (1 << TGSI_FILE_TEMPORARY);
3905    }
3906    /*
3907     * For performance reason immediates are always backed in a static
3908     * array, but if their number is too great, we have to use just
3909     * a dynamically allocated array.
3910     */
3911    bld.use_immediates_array =
3912          (info->file_max[TGSI_FILE_IMMEDIATE] >= LP_MAX_INLINED_IMMEDIATES);
3913    if (bld.use_immediates_array) {
3914       bld.indirect_files |= (1 << TGSI_FILE_IMMEDIATE);
3915    }
3916
3917
3918    bld.bld_base.soa = TRUE;
3919    bld.bld_base.emit_debug = emit_debug;
3920    bld.bld_base.emit_fetch_funcs[TGSI_FILE_CONSTANT] = emit_fetch_constant;
3921    bld.bld_base.emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = emit_fetch_immediate;
3922    bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_input;
3923    bld.bld_base.emit_fetch_funcs[TGSI_FILE_TEMPORARY] = emit_fetch_temporary;
3924    bld.bld_base.emit_fetch_funcs[TGSI_FILE_SYSTEM_VALUE] = emit_fetch_system_value;
3925    bld.bld_base.emit_store = emit_store;
3926
3927    bld.bld_base.emit_declaration = lp_emit_declaration_soa;
3928    bld.bld_base.emit_immediate = lp_emit_immediate_soa;
3929
3930    bld.bld_base.emit_prologue = emit_prologue;
3931    bld.bld_base.emit_epilogue = emit_epilogue;
3932
3933    /* Set opcode actions */
3934    lp_set_default_actions_cpu(&bld.bld_base);
3935
3936    bld.bld_base.op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit;
3937    bld.bld_base.op_actions[TGSI_OPCODE_BGNSUB].emit = bgnsub_emit;
3938    bld.bld_base.op_actions[TGSI_OPCODE_BRK].emit = brk_emit;
3939    bld.bld_base.op_actions[TGSI_OPCODE_BREAKC].emit = breakc_emit;
3940    bld.bld_base.op_actions[TGSI_OPCODE_CAL].emit = cal_emit;
3941    bld.bld_base.op_actions[TGSI_OPCODE_CASE].emit = case_emit;
3942    bld.bld_base.op_actions[TGSI_OPCODE_CONT].emit = cont_emit;
3943    bld.bld_base.op_actions[TGSI_OPCODE_DDX].emit = ddx_emit;
3944    bld.bld_base.op_actions[TGSI_OPCODE_DDY].emit = ddy_emit;
3945    bld.bld_base.op_actions[TGSI_OPCODE_DEFAULT].emit = default_emit;
3946    bld.bld_base.op_actions[TGSI_OPCODE_ELSE].emit = else_emit;
3947    bld.bld_base.op_actions[TGSI_OPCODE_ENDIF].emit = endif_emit;
3948    bld.bld_base.op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit;
3949    bld.bld_base.op_actions[TGSI_OPCODE_ENDSUB].emit = endsub_emit;
3950    bld.bld_base.op_actions[TGSI_OPCODE_ENDSWITCH].emit = endswitch_emit;
3951    bld.bld_base.op_actions[TGSI_OPCODE_IF].emit = if_emit;
3952    bld.bld_base.op_actions[TGSI_OPCODE_UIF].emit = uif_emit;
3953    bld.bld_base.op_actions[TGSI_OPCODE_KILL_IF].emit = kill_if_emit;
3954    bld.bld_base.op_actions[TGSI_OPCODE_KILL].emit = kill_emit;
3955    bld.bld_base.op_actions[TGSI_OPCODE_RET].emit = ret_emit;
3956    bld.bld_base.op_actions[TGSI_OPCODE_SWITCH].emit = switch_emit;
3957    bld.bld_base.op_actions[TGSI_OPCODE_TEX].emit = tex_emit;
3958    bld.bld_base.op_actions[TGSI_OPCODE_TXB].emit = txb_emit;
3959    bld.bld_base.op_actions[TGSI_OPCODE_TXD].emit = txd_emit;
3960    bld.bld_base.op_actions[TGSI_OPCODE_TXL].emit = txl_emit;
3961    bld.bld_base.op_actions[TGSI_OPCODE_TXP].emit = txp_emit;
3962    bld.bld_base.op_actions[TGSI_OPCODE_TXQ].emit = txq_emit;
3963    bld.bld_base.op_actions[TGSI_OPCODE_TXF].emit = txf_emit;
3964    bld.bld_base.op_actions[TGSI_OPCODE_TEX2].emit = tex2_emit;
3965    bld.bld_base.op_actions[TGSI_OPCODE_TXB2].emit = txb2_emit;
3966    bld.bld_base.op_actions[TGSI_OPCODE_TXL2].emit = txl2_emit;
3967    bld.bld_base.op_actions[TGSI_OPCODE_TG4].emit = tg4_emit;
3968    /* DX10 sampling ops */
3969    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE].emit = sample_emit;
3970    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_B].emit = sample_b_emit;
3971    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C].emit = sample_c_emit;
3972    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C_LZ].emit = sample_c_lz_emit;
3973    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_D].emit = sample_d_emit;
3974    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_I].emit = sample_i_emit;
3975    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_L].emit = sample_l_emit;
3976    bld.bld_base.op_actions[TGSI_OPCODE_SVIEWINFO].emit = sviewinfo_emit;
3977
3978    if (gs_iface) {
3979       /* There's no specific value for this because it should always
3980        * be set, but apps using ext_geometry_shader4 quite often
3981        * were forgetting so we're using MAX_VERTEX_VARYING from
3982        * that spec even though we could debug_assert if it's not
3983        * set, but that's a lot uglier. */
3984       uint max_output_vertices;
3985
3986       /* inputs are always indirect with gs */
3987       bld.indirect_files |= (1 << TGSI_FILE_INPUT);
3988       bld.gs_iface = gs_iface;
3989       bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_gs_input;
3990       bld.bld_base.op_actions[TGSI_OPCODE_EMIT].emit = emit_vertex;
3991       bld.bld_base.op_actions[TGSI_OPCODE_ENDPRIM].emit = end_primitive;
3992
3993       max_output_vertices =
3994             info->properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
3995       if (!max_output_vertices)
3996          max_output_vertices = 32;
3997
3998       bld.max_output_vertices_vec =
3999          lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
4000                                 max_output_vertices);
4001    }
4002
4003    lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
4004
4005    bld.system_values = *system_values;
4006
4007    lp_build_tgsi_llvm(&bld.bld_base, tokens);
4008
4009    if (0) {
4010       LLVMBasicBlockRef block = LLVMGetInsertBlock(gallivm->builder);
4011       LLVMValueRef function = LLVMGetBasicBlockParent(block);
4012       debug_printf("11111111111111111111111111111 \n");
4013       tgsi_dump(tokens, 0);
4014       lp_debug_dump_value(function);
4015       debug_printf("2222222222222222222222222222 \n");
4016    }
4017
4018    if (0) {
4019       LLVMModuleRef module = LLVMGetGlobalParent(
4020          LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
4021       LLVMDumpModule(module);
4022
4023    }
4024    lp_exec_mask_fini(&bld.exec_mask);
4025 }