src/gallium/auxiliary/gallivm/lp_bld_tgsi_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * Copyright 2007-2008 VMware, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28
  29 /**
  30  * @file
  31  * TGSI to LLVM IR translation -- SoA.
  32  *
  33  * @author Jose Fonseca <jfonseca@vmware.com>
  34  *
  35  * Based on tgsi_sse2.c code written by Michal Krol, Keith Whitwell,
  36  * Brian Paul, and others.
  37  */
  38
  39 #include "pipe/p_config.h"
  40 #include "pipe/p_shader_tokens.h"
  41 #include "util/u_debug.h"
  42 #include "util/u_math.h"
  43 #include "util/u_memory.h"
  44 #include "tgsi/tgsi_dump.h"
  45 #include "tgsi/tgsi_exec.h"
  46 #include "tgsi/tgsi_info.h"
  47 #include "tgsi/tgsi_parse.h"
  48 #include "tgsi/tgsi_util.h"
  49 #include "tgsi/tgsi_scan.h"
  50 #include "tgsi/tgsi_strings.h"
  51 #include "lp_bld_tgsi_action.h"
  52 #include "lp_bld_type.h"
  53 #include "lp_bld_const.h"
  54 #include "lp_bld_arit.h"
  55 #include "lp_bld_bitarit.h"
  56 #include "lp_bld_gather.h"
  57 #include "lp_bld_init.h"
  58 #include "lp_bld_logic.h"
  59 #include "lp_bld_swizzle.h"
  60 #include "lp_bld_flow.h"
  61 #include "lp_bld_quad.h"
  62 #include "lp_bld_tgsi.h"
  63 #include "lp_bld_limits.h"
  64 #include "lp_bld_debug.h"
  65 #include "lp_bld_printf.h"
  66 #include "lp_bld_sample.h"
  67 #include "lp_bld_struct.h"
  68
  69 /* SM 4.0 says that subroutines can nest 32 deep and
  70  * we need one more for our main function */
  71 #define LP_MAX_NUM_FUNCS 33
  72
  73 #define DUMP_GS_EMITS 0
  74
  75 /*
  76  * If non-zero, the generated LLVM IR will print intermediate results on every TGSI
  77  * instruction.
  78  *
  79  * TODO:
  80  * - take execution masks in consideration
  81  * - debug control-flow instructions
  82  */
  83 #define DEBUG_EXECUTION 0
  84
  85
  86 /*
  87  * Emit code to print a register value.
  88  */
  89 static void
  90 emit_dump_reg(struct gallivm_state *gallivm,
  91               unsigned file,
  92               unsigned index,
  93               unsigned chan,
  94               LLVMValueRef value)
  95 {
  96    char buf[32];
  97
  98    util_snprintf(buf, sizeof buf, "    %s[%u].%c = ",
  99                  tgsi_file_name(file),
 100                  index, "xyzw"[chan]);
 101
 102    lp_build_print_value(gallivm, buf, value);
 103 }
 104
 105 /*
 106  * Return the context for the current function.
 107  * (always 'main', if shader doesn't do any function calls)
 108  */
 109 static inline struct function_ctx *
 110 func_ctx(struct lp_exec_mask *mask)
 111 {
 112    assert(mask->function_stack_size > 0);
 113    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 114    return &mask->function_stack[mask->function_stack_size - 1];
 115 }
 116
 117 /*
 118  * Returns true if we're in a loop.
 119  * It's global, meaning that it returns true even if there's
 120  * no loop inside the current function, but we were inside
 121  * a loop inside another function, from which this one was called.
 122  */
 123 static inline boolean
 124 mask_has_loop(struct lp_exec_mask *mask)
 125 {
 126    int i;
 127    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 128       const struct function_ctx *ctx = &mask->function_stack[i];
 129       if (ctx->loop_stack_size > 0)
 130          return TRUE;
 131    }
 132    return FALSE;
 133 }
 134
 135 /*
 136  * Returns true if we're inside a switch statement.
 137  * It's global, meaning that it returns true even if there's
 138  * no switch in the current function, but we were inside
 139  * a switch inside another function, from which this one was called.
 140  */
 141 static inline boolean
 142 mask_has_switch(struct lp_exec_mask *mask)
 143 {
 144    int i;
 145    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 146       const struct function_ctx *ctx = &mask->function_stack[i];
 147       if (ctx->switch_stack_size > 0)
 148          return TRUE;
 149    }
 150    return FALSE;
 151 }
 152
 153 /*
 154  * Returns true if we're inside a conditional.
 155  * It's global, meaning that it returns true even if there's
 156  * no conditional in the current function, but we were inside
 157  * a conditional inside another function, from which this one was called.
 158  */
 159 static inline boolean
 160 mask_has_cond(struct lp_exec_mask *mask)
 161 {
 162    int i;
 163    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 164       const struct function_ctx *ctx = &mask->function_stack[i];
 165       if (ctx->cond_stack_size > 0)
 166          return TRUE;
 167    }
 168    return FALSE;
 169 }
 170
 171
 172 /*
 173  * Initialize a function context at the specified index.
 174  */
 175 static void
 176 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
 177 {
 178    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 179    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 180    struct function_ctx *ctx =  &mask->function_stack[function_idx];
 181
 182    ctx->cond_stack_size = 0;
 183    ctx->loop_stack_size = 0;
 184    ctx->switch_stack_size = 0;
 185
 186    if (function_idx == 0) {
 187       ctx->ret_mask = mask->ret_mask;
 188    }
 189
 190    ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
 191                                        int_type, "looplimiter");
 192    LLVMBuildStore(
 193       builder,
 194       LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
 195       ctx->loop_limiter);
 196 }
 197
 198 static void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
 199 {
 200    mask->bld = bld;
 201    mask->has_mask = FALSE;
 202    mask->ret_in_main = FALSE;
 203    /* For the main function */
 204    mask->function_stack_size = 1;
 205
 206    mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
 207    mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
 208          mask->cond_mask = mask->switch_mask =
 209          LLVMConstAllOnes(mask->int_vec_type);
 210
 211    mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
 212                                  sizeof(mask->function_stack[0]));
 213    lp_exec_mask_function_init(mask, 0);
 214 }
 215
 216 static void
 217 lp_exec_mask_fini(struct lp_exec_mask *mask)
 218 {
 219    FREE(mask->function_stack);
 220 }
 221
 222 static void lp_exec_mask_update(struct lp_exec_mask *mask)
 223 {
 224    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 225    boolean has_loop_mask = mask_has_loop(mask);
 226    boolean has_cond_mask = mask_has_cond(mask);
 227    boolean has_switch_mask = mask_has_switch(mask);
 228    boolean has_ret_mask = mask->function_stack_size > 1 ||
 229          mask->ret_in_main;
 230
 231    if (has_loop_mask) {
 232       /*for loops we need to update the entire mask at runtime */
 233       LLVMValueRef tmp;
 234       assert(mask->break_mask);
 235       tmp = LLVMBuildAnd(builder,
 236                          mask->cont_mask,
 237                          mask->break_mask,
 238                          "maskcb");
 239       mask->exec_mask = LLVMBuildAnd(builder,
 240                                      mask->cond_mask,
 241                                      tmp,
 242                                      "maskfull");
 243    } else
 244       mask->exec_mask = mask->cond_mask;
 245
 246    if (has_switch_mask) {
 247       mask->exec_mask = LLVMBuildAnd(builder,
 248                                      mask->exec_mask,
 249                                      mask->switch_mask,
 250                                      "switchmask");
 251    }
 252
 253    if (has_ret_mask) {
 254       mask->exec_mask = LLVMBuildAnd(builder,
 255                                      mask->exec_mask,
 256                                      mask->ret_mask,
 257                                      "callmask");
 258    }
 259
 260    mask->has_mask = (has_cond_mask ||
 261                      has_loop_mask ||
 262                      has_switch_mask ||
 263                      has_ret_mask);
 264 }
 265
 266 static void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
 267                                    LLVMValueRef val)
 268 {
 269    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 270    struct function_ctx *ctx = func_ctx(mask);
 271
 272    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
 273       ctx->cond_stack_size++;
 274       return;
 275    }
 276    if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
 277       assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
 278    }
 279    ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
 280    assert(LLVMTypeOf(val) == mask->int_vec_type);
 281    mask->cond_mask = LLVMBuildAnd(builder,
 282                                   mask->cond_mask,
 283                                   val,
 284                                   "");
 285    lp_exec_mask_update(mask);
 286 }
 287
 288 static void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
 289 {
 290    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 291    struct function_ctx *ctx = func_ctx(mask);
 292    LLVMValueRef prev_mask;
 293    LLVMValueRef inv_mask;
 294
 295    assert(ctx->cond_stack_size);
 296    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 297       return;
 298    prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
 299    if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
 300       assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
 301    }
 302
 303    inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");
 304
 305    mask->cond_mask = LLVMBuildAnd(builder,
 306                                   inv_mask,
 307                                   prev_mask, "");
 308    lp_exec_mask_update(mask);
 309 }
 310
 311 static void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
 312 {
 313    struct function_ctx *ctx = func_ctx(mask);
 314    assert(ctx->cond_stack_size);
 315    --ctx->cond_stack_size;
 316    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 317       return;
 318    mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
 319    lp_exec_mask_update(mask);
 320 }
 321
 322 static void lp_exec_bgnloop(struct lp_exec_mask *mask)
 323 {
 324    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 325    struct function_ctx *ctx = func_ctx(mask);
 326
 327    if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
 328       ++ctx->loop_stack_size;
 329       return;
 330    }
 331
 332    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 333       ctx->break_type;
 334    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;
 335
 336    ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
 337    ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
 338    ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
 339    ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
 340    ++ctx->loop_stack_size;
 341
 342    ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
 343    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 344
 345    ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");
 346
 347    LLVMBuildBr(builder, ctx->loop_block);
 348    LLVMPositionBuilderAtEnd(builder, ctx->loop_block);
 349
 350    mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
 351
 352    lp_exec_mask_update(mask);
 353 }
 354
 355 static void lp_exec_break(struct lp_exec_mask *mask,
 356                           struct lp_build_tgsi_context * bld_base)
 357 {
 358    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 359    struct function_ctx *ctx = func_ctx(mask);
 360
 361    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 362       LLVMValueRef exec_mask = LLVMBuildNot(builder,
 363                                             mask->exec_mask,
 364                                             "break");
 365
 366       mask->break_mask = LLVMBuildAnd(builder,
 367                                       mask->break_mask,
 368                                       exec_mask, "break_full");
 369    }
 370    else {
 371       unsigned opcode = bld_base->instructions[bld_base->pc + 1].Instruction.Opcode;
 372       boolean break_always = (opcode == TGSI_OPCODE_ENDSWITCH ||
 373                               opcode == TGSI_OPCODE_CASE);
 374
 375
 376       if (ctx->switch_in_default) {
 377          /*
 378           * stop default execution but only if this is an unconditional switch.
 379           * (The condition here is not perfect since dead code after break is
 380           * allowed but should be sufficient since false negatives are just
 381           * unoptimized - so we don't have to pre-evaluate that).
 382           */
 383          if(break_always && ctx->switch_pc) {
 384             bld_base->pc = ctx->switch_pc;
 385             return;
 386          }
 387       }
 388
 389       if (break_always) {
 390          mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
 391       }
 392       else {
 393          LLVMValueRef exec_mask = LLVMBuildNot(builder,
 394                                                mask->exec_mask,
 395                                                "break");
 396          mask->switch_mask = LLVMBuildAnd(builder,
 397                                           mask->switch_mask,
 398                                           exec_mask, "break_switch");
 399       }
 400    }
 401
 402    lp_exec_mask_update(mask);
 403 }
 404
 405 static void lp_exec_break_condition(struct lp_exec_mask *mask,
 406                                     LLVMValueRef cond)
 407 {
 408    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 409    struct function_ctx *ctx = func_ctx(mask);
 410    LLVMValueRef cond_mask = LLVMBuildAnd(builder,
 411                                          mask->exec_mask,
 412                                          cond, "cond_mask");
 413    cond_mask = LLVMBuildNot(builder, cond_mask, "break_cond");
 414
 415    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 416       mask->break_mask = LLVMBuildAnd(builder,
 417                                       mask->break_mask,
 418                                       cond_mask, "breakc_full");
 419    }
 420    else {
 421       mask->switch_mask = LLVMBuildAnd(builder,
 422                                        mask->switch_mask,
 423                                        cond_mask, "breakc_switch");
 424    }
 425
 426    lp_exec_mask_update(mask);
 427 }
 428
 429 static void lp_exec_continue(struct lp_exec_mask *mask)
 430 {
 431    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 432    LLVMValueRef exec_mask = LLVMBuildNot(builder,
 433                                          mask->exec_mask,
 434                                          "");
 435
 436    mask->cont_mask = LLVMBuildAnd(builder,
 437                                   mask->cont_mask,
 438                                   exec_mask, "");
 439
 440    lp_exec_mask_update(mask);
 441 }
 442
 443
 444 static void lp_exec_endloop(struct gallivm_state *gallivm,
 445                             struct lp_exec_mask *mask)
 446 {
 447    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 448    struct function_ctx *ctx = func_ctx(mask);
 449    LLVMBasicBlockRef endloop;
 450    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 451    LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
 452                                                mask->bld->type.width *
 453                                                mask->bld->type.length);
 454    LLVMValueRef i1cond, i2cond, icond, limiter;
 455
 456    assert(mask->break_mask);
 457
 458
 459    assert(ctx->loop_stack_size);
 460    if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 461       --ctx->loop_stack_size;
 462       return;
 463    }
 464
 465    /*
 466     * Restore the cont_mask, but don't pop
 467     */
 468    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
 469    lp_exec_mask_update(mask);
 470
 471    /*
 472     * Unlike the continue mask, the break_mask must be preserved across loop
 473     * iterations
 474     */
 475    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 476
 477    /* Decrement the loop limiter */
 478    limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");
 479
 480    limiter = LLVMBuildSub(
 481       builder,
 482       limiter,
 483       LLVMConstInt(int_type, 1, false),
 484       "");
 485
 486    LLVMBuildStore(builder, limiter, ctx->loop_limiter);
 487
 488    /* i1cond = (mask != 0) */
 489    i1cond = LLVMBuildICmp(
 490       builder,
 491       LLVMIntNE,
 492       LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
 493       LLVMConstNull(reg_type), "i1cond");
 494
 495    /* i2cond = (looplimiter > 0) */
 496    i2cond = LLVMBuildICmp(
 497       builder,
 498       LLVMIntSGT,
 499       limiter,
 500       LLVMConstNull(int_type), "i2cond");
 501
 502    /* if( i1cond && i2cond ) */
 503    icond = LLVMBuildAnd(builder, i1cond, i2cond, "");
 504
 505    endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");
 506
 507    LLVMBuildCondBr(builder,
 508                    icond, ctx->loop_block, endloop);
 509
 510    LLVMPositionBuilderAtEnd(builder, endloop);
 511
 512    assert(ctx->loop_stack_size);
 513    --ctx->loop_stack_size;
 514    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
 515    mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
 516    ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
 517    ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
 518    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
 519          ctx->switch_stack_size];
 520
 521    lp_exec_mask_update(mask);
 522 }
 523
 524 static void lp_exec_switch(struct lp_exec_mask *mask,
 525                            LLVMValueRef switchval)
 526 {
 527    struct function_ctx *ctx = func_ctx(mask);
 528
 529    if (ctx->switch_stack_size >= LP_MAX_TGSI_NESTING ||
 530        ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 531       ctx->switch_stack_size++;
 532       return;
 533    }
 534
 535    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 536       ctx->break_type;
 537    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_SWITCH;
 538
 539    ctx->switch_stack[ctx->switch_stack_size].switch_mask = mask->switch_mask;
 540    ctx->switch_stack[ctx->switch_stack_size].switch_val = ctx->switch_val;
 541    ctx->switch_stack[ctx->switch_stack_size].switch_mask_default = ctx->switch_mask_default;
 542    ctx->switch_stack[ctx->switch_stack_size].switch_in_default = ctx->switch_in_default;
 543    ctx->switch_stack[ctx->switch_stack_size].switch_pc = ctx->switch_pc;
 544    ctx->switch_stack_size++;
 545
 546    mask->switch_mask = LLVMConstNull(mask->int_vec_type);
 547    ctx->switch_val = switchval;
 548    ctx->switch_mask_default = LLVMConstNull(mask->int_vec_type);
 549    ctx->switch_in_default = false;
 550    ctx->switch_pc = 0;
 551
 552    lp_exec_mask_update(mask);
 553 }
 554
 555 static void lp_exec_endswitch(struct lp_exec_mask *mask,
 556                               struct lp_build_tgsi_context * bld_base)
 557 {
 558    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 559    struct function_ctx *ctx = func_ctx(mask);
 560
 561    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 562       ctx->switch_stack_size--;
 563       return;
 564    }
 565
 566    /* check if there's deferred default if so do it now */
 567    if (ctx->switch_pc && !ctx->switch_in_default) {
 568       LLVMValueRef prevmask, defaultmask;
 569       unsigned tmp_pc;
 570       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 571       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 572       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 573       ctx->switch_in_default = true;
 574
 575       lp_exec_mask_update(mask);
 576
 577       assert(bld_base->instructions[ctx->switch_pc - 1].Instruction.Opcode ==
 578              TGSI_OPCODE_DEFAULT);
 579
 580       tmp_pc = bld_base->pc;
 581       bld_base->pc = ctx->switch_pc;
 582       /*
 583        * re-purpose switch_pc to point to here again, since we stop execution of
 584        * the deferred default after next break.
 585        */
 586       ctx->switch_pc = tmp_pc - 1;
 587
 588       return;
 589    }
 590
 591    else if (ctx->switch_pc && ctx->switch_in_default) {
 592       assert(bld_base->pc == ctx->switch_pc + 1);
 593    }
 594
 595    ctx->switch_stack_size--;
 596    mask->switch_mask = ctx->switch_stack[ctx->switch_stack_size].switch_mask;
 597    ctx->switch_val = ctx->switch_stack[ctx->switch_stack_size].switch_val;
 598    ctx->switch_mask_default = ctx->switch_stack[ctx->switch_stack_size].switch_mask_default;
 599    ctx->switch_in_default = ctx->switch_stack[ctx->switch_stack_size].switch_in_default;
 600    ctx->switch_pc = ctx->switch_stack[ctx->switch_stack_size].switch_pc;
 601
 602    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size];
 603
 604    lp_exec_mask_update(mask);
 605 }
 606
 607 static void lp_exec_case(struct lp_exec_mask *mask,
 608                          LLVMValueRef caseval)
 609 {
 610    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 611    struct function_ctx *ctx = func_ctx(mask);
 612
 613    LLVMValueRef casemask, prevmask;
 614
 615    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 616       return;
 617    }
 618
 619    /* skipping case mask evaluation here is NOT optional (not in all cases anyway). */
 620    if (!ctx->switch_in_default) {
 621       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 622       casemask = lp_build_cmp(mask->bld, PIPE_FUNC_EQUAL, caseval, ctx->switch_val);
 623       ctx->switch_mask_default = LLVMBuildOr(builder, casemask,
 624                                              ctx->switch_mask_default, "sw_default_mask");
 625       casemask = LLVMBuildOr(builder, casemask, mask->switch_mask, "");
 626       mask->switch_mask = LLVMBuildAnd(builder, casemask, prevmask, "sw_mask");
 627
 628       lp_exec_mask_update(mask);
 629    }
 630 }
 631
 632 /*
 633  * Analyse default statement in a switch.
 634  * \return true if default is last statement, false otherwise
 635  * \param default_pc_start contains pc of instruction to jump to
 636  *                         if default wasn't last but there's no
 637  *                         fallthrough into default.
 638  */
 639 static boolean default_analyse_is_last(struct lp_exec_mask *mask,
 640                                        struct lp_build_tgsi_context * bld_base,
 641                                        int *default_pc_start)
 642 {
 643    unsigned pc = bld_base->pc;
 644    struct function_ctx *ctx = func_ctx(mask);
 645    unsigned curr_switch_stack = ctx->switch_stack_size;
 646
 647    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 648       return false;
 649    }
 650
 651    /* skip over case statements which are together with default */
 652    while (bld_base->instructions[pc].Instruction.Opcode == TGSI_OPCODE_CASE) {
 653       pc++;
 654    }
 655
 656    while (pc != -1 && pc < bld_base->num_instructions) {
 657       unsigned opcode = bld_base->instructions[pc].Instruction.Opcode;
 658       switch (opcode) {
 659       case TGSI_OPCODE_CASE:
 660          if (curr_switch_stack == ctx->switch_stack_size) {
 661             *default_pc_start = pc - 1;
 662             return false;
 663          }
 664          break;
 665       case TGSI_OPCODE_SWITCH:
 666          curr_switch_stack++;
 667          break;
 668       case TGSI_OPCODE_ENDSWITCH:
 669          if (curr_switch_stack == ctx->switch_stack_size) {
 670             *default_pc_start = pc - 1;
 671             return true;
 672          }
 673          curr_switch_stack--;
 674          break;
 675       }
 676       pc++;
 677    }
 678    /* should never arrive here */
 679    assert(0);
 680    return true;
 681 }
 682
 683 static void lp_exec_default(struct lp_exec_mask *mask,
 684                             struct lp_build_tgsi_context * bld_base)
 685 {
 686    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 687    struct function_ctx *ctx = func_ctx(mask);
 688
 689    int default_exec_pc;
 690    boolean default_is_last;
 691
 692    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 693       return;
 694    }
 695
 696    /*
 697     * This is a messy opcode, because it may not be always at the end and
 698     * there can be fallthrough in and out of it.
 699     */
 700
 701    default_is_last = default_analyse_is_last(mask, bld_base, &default_exec_pc);
 702    /*
 703     * If it is last statement in switch (note that case statements appearing
 704     * "at the same time" as default don't change that) everything is just fine,
 705     * update switch mask and go on. This means we can handle default with
 706     * fallthrough INTO it without overhead, if it is last.
 707     */
 708    if (default_is_last) {
 709       LLVMValueRef prevmask, defaultmask;
 710       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 711       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 712       defaultmask = LLVMBuildOr(builder, defaultmask, mask->switch_mask, "");
 713       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 714       ctx->switch_in_default = true;
 715
 716       lp_exec_mask_update(mask);
 717    }
 718    else {
 719       /*
 720        * Technically, "case" immediately before default isn't really a
 721        * fallthrough, however we still have to count them as such as we
 722        * already have updated the masks.
 723        * If that happens in practice could add a switch optimizer pass
 724        * which just gets rid of all case statements appearing together with
 725        * default (or could do switch analysis at switch start time instead).
 726        */
 727       unsigned opcode = bld_base->instructions[bld_base->pc - 1].Instruction.Opcode;
 728       boolean ft_into = (opcode != TGSI_OPCODE_BRK &&
 729                          opcode != TGSI_OPCODE_SWITCH);
 730       /*
 731        * If it is not last statement and there was no fallthrough into it,
 732        * we record the PC and continue execution at next case (again, those
 733        * case encountered at the same time don't count). At endswitch
 734        * time, we update switchmask, and go back executing the code we skipped
 735        * until the next break (possibly re-executing some code with changed mask
 736        * if there was a fallthrough out of default).
 737        * Finally, if it is not last statement and there was a fallthrough into it,
 738        * do the same as with the former case, except instead of skipping the code
 739        * just execute it without updating the mask, then go back and re-execute.
 740        */
 741       ctx->switch_pc = bld_base->pc;
 742       if (!ft_into) {
 743          bld_base->pc = default_exec_pc;
 744       }
 745    }
 746 }
 747
 748
 749 /* stores val into an address pointed to by dst_ptr.
 750  * mask->exec_mask is used to figure out which bits of val
 751  * should be stored into the address
 752  * (0 means don't store this bit, 1 means do store).
 753  */
 754 static void lp_exec_mask_store(struct lp_exec_mask *mask,
 755                                struct lp_build_context *bld_store,
 756                                LLVMValueRef pred,
 757                                LLVMValueRef val,
 758                                LLVMValueRef dst_ptr)
 759 {
 760    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 761
 762    assert(lp_check_value(bld_store->type, val));
 763    assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
 764    assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val));
 765
 766    /* Mix the predicate and execution mask */
 767    if (mask->has_mask) {
 768       if (pred) {
 769          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
 770       } else {
 771          pred = mask->exec_mask;
 772       }
 773    }
 774
 775    if (pred) {
 776       LLVMValueRef res, dst;
 777
 778       dst = LLVMBuildLoad(builder, dst_ptr, "");
 779       res = lp_build_select(bld_store, pred, val, dst);
 780       LLVMBuildStore(builder, res, dst_ptr);
 781    } else
 782       LLVMBuildStore(builder, val, dst_ptr);
 783 }
 784
 785 static void lp_exec_mask_call(struct lp_exec_mask *mask,
 786                               int func,
 787                               int *pc)
 788 {
 789    if (mask->function_stack_size >= LP_MAX_NUM_FUNCS) {
 790       return;
 791    }
 792
 793    lp_exec_mask_function_init(mask, mask->function_stack_size);
 794    mask->function_stack[mask->function_stack_size].pc = *pc;
 795    mask->function_stack[mask->function_stack_size].ret_mask = mask->ret_mask;
 796    mask->function_stack_size++;
 797    *pc = func;
 798 }
 799
 800 static void lp_exec_mask_ret(struct lp_exec_mask *mask, int *pc)
 801 {
 802    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 803    struct function_ctx *ctx = func_ctx(mask);
 804    LLVMValueRef exec_mask;
 805
 806    if (ctx->cond_stack_size == 0 &&
 807        ctx->loop_stack_size == 0 &&
 808        ctx->switch_stack_size == 0 &&
 809        mask->function_stack_size == 1) {
 810       /* returning from main() */
 811       *pc = -1;
 812       return;
 813    }
 814
 815    if (mask->function_stack_size == 1) {
 816       /*
 817        * This requires special handling since we need to ensure
 818        * we don't drop the mask even if we have no call stack
 819        * (e.g. after a ret in a if clause after the endif)
 820        */
 821       mask->ret_in_main = TRUE;
 822    }
 823
 824    exec_mask = LLVMBuildNot(builder,
 825                             mask->exec_mask,
 826                             "ret");
 827
 828    mask->ret_mask = LLVMBuildAnd(builder,
 829                                  mask->ret_mask,
 830                                  exec_mask, "ret_full");
 831
 832    lp_exec_mask_update(mask);
 833 }
 834
 835 static void lp_exec_mask_bgnsub(struct lp_exec_mask *mask)
 836 {
 837 }
 838
 839 static void lp_exec_mask_endsub(struct lp_exec_mask *mask, int *pc)
 840 {
 841    struct function_ctx *ctx;
 842
 843    assert(mask->function_stack_size > 1);
 844    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 845
 846    ctx = func_ctx(mask);
 847    mask->function_stack_size--;
 848
 849    *pc = ctx->pc;
 850    mask->ret_mask = ctx->ret_mask;
 851
 852    lp_exec_mask_update(mask);
 853 }
 854
 855
 856 static LLVMValueRef
 857 get_file_ptr(struct lp_build_tgsi_soa_context *bld,
 858              unsigned file,
 859              unsigned index,
 860              unsigned chan)
 861 {
 862    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 863    LLVMValueRef (*array_of_vars)[TGSI_NUM_CHANNELS];
 864    LLVMValueRef var_of_array;
 865
 866    switch (file) {
 867    case TGSI_FILE_TEMPORARY:
 868       array_of_vars = bld->temps;
 869       var_of_array = bld->temps_array;
 870       break;
 871    case TGSI_FILE_OUTPUT:
 872       array_of_vars = bld->outputs;
 873       var_of_array = bld->outputs_array;
 874       break;
 875    default:
 876       assert(0);
 877       return NULL;
 878    }
 879
 880    assert(chan < 4);
 881
 882    if (bld->indirect_files & (1 << file)) {
 883       LLVMValueRef lindex = lp_build_const_int32(bld->bld_base.base.gallivm, index * 4 + chan);
 884       return LLVMBuildGEP(builder, var_of_array, &lindex, 1, "");
 885    }
 886    else {
 887       assert(index <= bld->bld_base.info->file_max[file]);
 888       return array_of_vars[index][chan];
 889    }
 890 }
 891
 892
 893 /**
 894  * Return pointer to a temporary register channel (src or dest).
 895  * Note that indirect addressing cannot be handled here.
 896  * \param index  which temporary register
 897  * \param chan  which channel of the temp register.
 898  */
 899 LLVMValueRef
 900 lp_get_temp_ptr_soa(struct lp_build_tgsi_soa_context *bld,
 901              unsigned index,
 902              unsigned chan)
 903 {
 904    return get_file_ptr(bld, TGSI_FILE_TEMPORARY, index, chan);
 905 }
 906
 907 /**
 908  * Return pointer to a output register channel (src or dest).
 909  * Note that indirect addressing cannot be handled here.
 910  * \param index  which output register
 911  * \param chan  which channel of the output register.
 912  */
 913 LLVMValueRef
 914 lp_get_output_ptr(struct lp_build_tgsi_soa_context *bld,
 915                unsigned index,
 916                unsigned chan)
 917 {
 918    return get_file_ptr(bld, TGSI_FILE_OUTPUT, index, chan);
 919 }
 920
 921 /*
 922  * If we have indirect addressing in outputs copy our alloca array
 923  * to the outputs slots specified by the caller to make sure
 924  * our outputs are delivered consistently via the same interface.
 925  */
 926 static void
 927 gather_outputs(struct lp_build_tgsi_soa_context * bld)
 928 {
 929    if ((bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
 930       unsigned index, chan;
 931       assert(bld->bld_base.info->num_outputs <=
 932              bld->bld_base.info->file_max[TGSI_FILE_OUTPUT] + 1);
 933       for (index = 0; index < bld->bld_base.info->num_outputs; ++index) {
 934          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 935             bld->outputs[index][chan] = lp_get_output_ptr(bld, index, chan);
 936          }
 937       }
 938    }
 939 }
 940
 941 /**
 942  * Gather vector.
 943  * XXX the lp_build_gather() function should be capable of doing this
 944  * with a little work.
 945  */
 946 static LLVMValueRef
 947 build_gather(struct lp_build_tgsi_context *bld_base,
 948              LLVMValueRef base_ptr,
 949              LLVMValueRef indexes,
 950              LLVMValueRef overflow_mask,
 951              LLVMValueRef indexes2)
 952 {
 953    struct gallivm_state *gallivm = bld_base->base.gallivm;
 954    LLVMBuilderRef builder = gallivm->builder;
 955    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 956    struct lp_build_context *bld = &bld_base->base;
 957    LLVMValueRef res;
 958    unsigned i;
 959
 960    if (indexes2)
 961       res = LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2));
 962    else
 963       res = bld->undef;
 964    /*
 965     * overflow_mask is a vector telling us which channels
 966     * in the vector overflowed. We use the overflow behavior for
 967     * constant buffers which is defined as:
 968     * Out of bounds access to constant buffer returns 0 in all
 969     * components. Out of bounds behavior is always with respect
 970     * to the size of the buffer bound at that slot.
 971     */
 972
 973    if (overflow_mask) {
 974       /*
 975        * We avoid per-element control flow here (also due to llvm going crazy,
 976        * though I suspect it's better anyway since overflow is likely rare).
 977        * Note that since we still fetch from buffers even if num_elements was
 978        * zero (in this case we'll fetch from index zero) the jit func callers
 979        * MUST provide valid fake constant buffers of size 4x32 (the values do
 980        * not matter), otherwise we'd still need (not per element though)
 981        * control flow.
 982        */
 983       indexes = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes);
 984       if (indexes2)
 985          indexes2 = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes2);
 986    }
 987
 988    /*
 989     * Loop over elements of index_vec, load scalar value, insert it into 'res'.
 990     */
 991    for (i = 0; i < bld->type.length * (indexes2 ? 2 : 1); i++) {
 992       LLVMValueRef si, di;
 993       LLVMValueRef index;
 994       LLVMValueRef scalar_ptr, scalar;
 995
 996       di = lp_build_const_int32(bld->gallivm, i);
 997       if (indexes2)
 998          si = lp_build_const_int32(bld->gallivm, i >> 1);
 999       else
1000          si = di;
1001
1002       if (indexes2 && (i & 1)) {
1003          index = LLVMBuildExtractElement(builder,
1004                                          indexes2, si, "");
1005       } else {
1006          index = LLVMBuildExtractElement(builder,
1007                                          indexes, si, "");
1008       }
1009       scalar_ptr = LLVMBuildGEP(builder, base_ptr,
1010                                 &index, 1, "gather_ptr");
1011       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1012
1013       res = LLVMBuildInsertElement(builder, res, scalar, di, "");
1014    }
1015
1016    if (overflow_mask) {
1017       if (indexes2) {
1018          res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1019          overflow_mask = LLVMBuildSExt(builder, overflow_mask,
1020                                        bld_base->dbl_bld.int_vec_type, "");
1021          res = lp_build_select(&bld_base->dbl_bld, overflow_mask,
1022                                bld_base->dbl_bld.zero, res);
1023       } else
1024          res = lp_build_select(bld, overflow_mask, bld->zero, res);
1025    }
1026
1027    return res;
1028 }
1029
1030
1031 /**
1032  * Scatter/store vector.
1033  */
1034 static void
1035 emit_mask_scatter(struct lp_build_tgsi_soa_context *bld,
1036                   LLVMValueRef base_ptr,
1037                   LLVMValueRef indexes,
1038                   LLVMValueRef values,
1039                   struct lp_exec_mask *mask,
1040                   LLVMValueRef pred)
1041 {
1042    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1043    LLVMBuilderRef builder = gallivm->builder;
1044    unsigned i;
1045
1046    /* Mix the predicate and execution mask */
1047    if (mask->has_mask) {
1048       if (pred) {
1049          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
1050       }
1051       else {
1052          pred = mask->exec_mask;
1053       }
1054    }
1055
1056    /*
1057     * Loop over elements of index_vec, store scalar value.
1058     */
1059    for (i = 0; i < bld->bld_base.base.type.length; i++) {
1060       LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1061       LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
1062       LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
1063       LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
1064       LLVMValueRef scalar_pred = pred ?
1065          LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
1066
1067       if (0)
1068          lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
1069                          ii, val, index, scalar_ptr);
1070
1071       if (scalar_pred) {
1072          LLVMValueRef real_val, dst_val;
1073          dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
1074          real_val = lp_build_select(&bld->elem_bld, scalar_pred, val, dst_val);
1075          LLVMBuildStore(builder, real_val, scalar_ptr);
1076       }
1077       else {
1078          LLVMBuildStore(builder, val, scalar_ptr);
1079       }
1080    }
1081 }
1082
1083
1084 /**
1085  * Read the current value of the ADDR register, convert the floats to
1086  * ints, add the base index and return the vector of offsets.
1087  * The offsets will be used to index into the constant buffer or
1088  * temporary register file.
1089  */
1090 static LLVMValueRef
1091 get_indirect_index(struct lp_build_tgsi_soa_context *bld,
1092                    unsigned reg_file, unsigned reg_index,
1093                    const struct tgsi_ind_register *indirect_reg)
1094 {
1095    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1096    struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
1097    /* always use X component of address register */
1098    unsigned swizzle = indirect_reg->Swizzle;
1099    LLVMValueRef base;
1100    LLVMValueRef rel;
1101    LLVMValueRef max_index;
1102    LLVMValueRef index;
1103
1104    assert(bld->indirect_files & (1 << reg_file));
1105
1106    base = lp_build_const_int_vec(bld->bld_base.base.gallivm, uint_bld->type, reg_index);
1107
1108    assert(swizzle < 4);
1109    switch (indirect_reg->File) {
1110    case TGSI_FILE_ADDRESS:
1111       rel = LLVMBuildLoad(builder,
1112                           bld->addr[indirect_reg->Index][swizzle],
1113                           "load addr reg");
1114       /* ADDR LLVM values already have LLVM integer type. */
1115       break;
1116    case TGSI_FILE_TEMPORARY:
1117       rel = lp_get_temp_ptr_soa(bld, indirect_reg->Index, swizzle);
1118       rel = LLVMBuildLoad(builder, rel, "load temp reg");
1119       /* TEMP LLVM values always have LLVM float type, but for indirection, the
1120        * value actually stored is expected to be an integer */
1121       rel = LLVMBuildBitCast(builder, rel, uint_bld->vec_type, "");
1122       break;
1123    default:
1124       assert(0);
1125       rel = uint_bld->zero;
1126    }
1127
1128    index = lp_build_add(uint_bld, base, rel);
1129
1130    /*
1131     * emit_fetch_constant handles constant buffer overflow so this code
1132     * is pointless for them.
1133     * Furthermore the D3D10 spec in section 6.5 says:
1134     * If the constant buffer bound to a slot is larger than the size
1135     * declared in the shader for that slot, implementations are allowed
1136     * to return incorrect data (not necessarily 0) for indices that are
1137     * larger than the declared size but smaller than the buffer size.
1138     */
1139    if (reg_file != TGSI_FILE_CONSTANT) {
1140       max_index = lp_build_const_int_vec(bld->bld_base.base.gallivm,
1141                                          uint_bld->type,
1142                                          bld->bld_base.info->file_max[reg_file]);
1143
1144       assert(!uint_bld->type.sign);
1145       index = lp_build_min(uint_bld, index, max_index);
1146    }
1147
1148    return index;
1149 }
1150
1151 static struct lp_build_context *
1152 stype_to_fetch(struct lp_build_tgsi_context * bld_base,
1153                enum tgsi_opcode_type stype)
1154 {
1155    struct lp_build_context *bld_fetch;
1156
1157    switch (stype) {
1158    case TGSI_TYPE_FLOAT:
1159    case TGSI_TYPE_UNTYPED:
1160       bld_fetch = &bld_base->base;
1161       break;
1162    case TGSI_TYPE_UNSIGNED:
1163       bld_fetch = &bld_base->uint_bld;
1164       break;
1165    case TGSI_TYPE_SIGNED:
1166       bld_fetch = &bld_base->int_bld;
1167       break;
1168    case TGSI_TYPE_DOUBLE:
1169       bld_fetch = &bld_base->dbl_bld;
1170       break;
1171    case TGSI_TYPE_VOID:
1172    default:
1173       assert(0);
1174       bld_fetch = NULL;
1175       break;
1176    }
1177    return bld_fetch;
1178 }
1179
1180 static LLVMValueRef
1181 get_soa_array_offsets(struct lp_build_context *uint_bld,
1182                       LLVMValueRef indirect_index,
1183                       unsigned chan_index,
1184                       boolean need_perelement_offset)
1185 {
1186    struct gallivm_state *gallivm = uint_bld->gallivm;
1187    LLVMValueRef chan_vec =
1188       lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
1189    LLVMValueRef length_vec =
1190       lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
1191    LLVMValueRef index_vec;
1192
1193    /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
1194    index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1195    index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
1196    index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
1197
1198    if (need_perelement_offset) {
1199       LLVMValueRef pixel_offsets;
1200       int i;
1201      /* build pixel offset vector: {0, 1, 2, 3, ...} */
1202       pixel_offsets = uint_bld->undef;
1203       for (i = 0; i < uint_bld->type.length; i++) {
1204          LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1205          pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
1206                                                 ii, ii, "");
1207       }
1208       index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
1209    }
1210    return index_vec;
1211 }
1212
1213 static LLVMValueRef
1214 emit_fetch_constant(
1215    struct lp_build_tgsi_context * bld_base,
1216    const struct tgsi_full_src_register * reg,
1217    enum tgsi_opcode_type stype,
1218    unsigned swizzle)
1219 {
1220    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1221    struct gallivm_state *gallivm = bld_base->base.gallivm;
1222    LLVMBuilderRef builder = gallivm->builder;
1223    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1224    unsigned dimension = 0;
1225    LLVMValueRef consts_ptr;
1226    LLVMValueRef num_consts;
1227    LLVMValueRef res;
1228
1229    /* XXX: Handle fetching xyzw components as a vector */
1230    assert(swizzle != ~0);
1231
1232    if (reg->Register.Dimension) {
1233       assert(!reg->Dimension.Indirect);
1234       dimension = reg->Dimension.Index;
1235       assert(dimension < LP_MAX_TGSI_CONST_BUFFERS);
1236    }
1237
1238    consts_ptr = bld->consts[dimension];
1239    num_consts = bld->consts_sizes[dimension];
1240
1241    if (reg->Register.Indirect) {
1242       LLVMValueRef indirect_index;
1243       LLVMValueRef swizzle_vec =
1244          lp_build_const_int_vec(gallivm, uint_bld->type, swizzle);
1245       LLVMValueRef index_vec;  /* index into the const buffer */
1246       LLVMValueRef overflow_mask;
1247       LLVMValueRef index_vec2 = NULL;
1248
1249       indirect_index = get_indirect_index(bld,
1250                                           reg->Register.File,
1251                                           reg->Register.Index,
1252                                           &reg->Indirect);
1253
1254       /* All fetches are from the same constant buffer, so
1255        * we need to propagate the size to a vector to do a
1256        * vector comparison */
1257       num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
1258       /* Construct a boolean vector telling us which channels
1259        * overflow the bound constant buffer */
1260       overflow_mask = lp_build_compare(gallivm, uint_bld->type, PIPE_FUNC_GEQUAL,
1261                                        indirect_index, num_consts);
1262
1263       /* index_vec = indirect_index * 4 + swizzle */
1264       index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1265       index_vec = lp_build_add(uint_bld, index_vec, swizzle_vec);
1266
1267       if (stype == TGSI_TYPE_DOUBLE) {
1268          LLVMValueRef swizzle_vec2;
1269          swizzle_vec2 = lp_build_const_int_vec(gallivm, uint_bld->type, swizzle + 1);
1270          index_vec2 = lp_build_shl_imm(uint_bld, indirect_index, 2);
1271          index_vec2 = lp_build_add(uint_bld, index_vec2, swizzle_vec2);
1272       }
1273       /* Gather values from the constant buffer */
1274       res = build_gather(bld_base, consts_ptr, index_vec, overflow_mask, index_vec2);
1275    }
1276    else {
1277       LLVMValueRef index;  /* index into the const buffer */
1278       LLVMValueRef scalar, scalar_ptr;
1279       struct lp_build_context *bld_broad = &bld_base->base;
1280       index = lp_build_const_int32(gallivm, reg->Register.Index * 4 + swizzle);
1281
1282       scalar_ptr = LLVMBuildGEP(builder, consts_ptr,
1283                                 &index, 1, "");
1284       if (stype == TGSI_TYPE_DOUBLE) {
1285          LLVMTypeRef dptr_type = LLVMPointerType(LLVMDoubleTypeInContext(gallivm->context), 0);
1286          scalar_ptr = LLVMBuildBitCast(builder, scalar_ptr, dptr_type, "");
1287          bld_broad = &bld_base->dbl_bld;
1288       }
1289       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1290       res = lp_build_broadcast_scalar(bld_broad, scalar);
1291    }
1292
1293    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED || stype == TGSI_TYPE_DOUBLE) {
1294       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1295       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1296    }
1297
1298    return res;
1299 }
1300
1301 /**
1302  * Fetch double values from two separate channels.
1303  * Doubles are stored split across two channels, like xy and zw.
1304  * This function creates a set of 16 floats,
1305  * extracts the values from the two channels,
1306  * puts them in the correct place, then casts to 8 doubles.
1307  */
1308 static LLVMValueRef
1309 emit_fetch_double(
1310    struct lp_build_tgsi_context * bld_base,
1311    enum tgsi_opcode_type stype,
1312    LLVMValueRef input,
1313    LLVMValueRef input2)
1314 {
1315    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1316    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1317    LLVMBuilderRef builder = gallivm->builder;
1318    LLVMValueRef res;
1319    struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1320    int i;
1321    LLVMValueRef shuffles[16];
1322    int len = bld_base->base.type.length * 2;
1323    assert(len <= 16);
1324
1325    for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
1326       shuffles[i] = lp_build_const_int32(gallivm, i / 2);
1327       shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
1328    }
1329    res = LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
1330
1331    return LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1332 }
1333
1334 static LLVMValueRef
1335 emit_fetch_immediate(
1336    struct lp_build_tgsi_context * bld_base,
1337    const struct tgsi_full_src_register * reg,
1338    enum tgsi_opcode_type stype,
1339    unsigned swizzle)
1340 {
1341    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1342    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1343    LLVMBuilderRef builder = gallivm->builder;
1344    LLVMValueRef res = NULL;
1345
1346    if (bld->use_immediates_array || reg->Register.Indirect) {
1347       LLVMValueRef imms_array;
1348       LLVMTypeRef fptr_type;
1349
1350       /* cast imms_array pointer to float* */
1351       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1352       imms_array = LLVMBuildBitCast(builder, bld->imms_array, fptr_type, "");
1353
1354       if (reg->Register.Indirect) {
1355          LLVMValueRef indirect_index;
1356          LLVMValueRef index_vec;  /* index into the immediate register array */
1357          LLVMValueRef index_vec2 = NULL;
1358          indirect_index = get_indirect_index(bld,
1359                                              reg->Register.File,
1360                                              reg->Register.Index,
1361                                              &reg->Indirect);
1362          /*
1363           * Unlike for other reg classes, adding pixel offsets is unnecessary -
1364           * immediates are stored as full vectors (FIXME??? - might be better
1365           * to store them the same as constants) but all elements are the same
1366           * in any case.
1367           */
1368          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1369                                            indirect_index,
1370                                            swizzle,
1371                                            FALSE);
1372          if (stype == TGSI_TYPE_DOUBLE)
1373             index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
1374                                               indirect_index,
1375                                               swizzle + 1,
1376                                               FALSE);
1377          /* Gather values from the immediate register array */
1378          res = build_gather(bld_base, imms_array, index_vec, NULL, index_vec2);
1379       } else {
1380          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1381                                         reg->Register.Index * 4 + swizzle);
1382          LLVMValueRef imms_ptr =  LLVMBuildGEP(builder,
1383                                                 bld->imms_array, &lindex, 1, "");
1384          res = LLVMBuildLoad(builder, imms_ptr, "");
1385
1386          if (stype == TGSI_TYPE_DOUBLE) {
1387             LLVMValueRef lindex1;
1388             LLVMValueRef imms_ptr2;
1389             LLVMValueRef res2;
1390
1391             lindex1 = lp_build_const_int32(gallivm,
1392                                            reg->Register.Index * 4 + swizzle + 1);
1393             imms_ptr2 = LLVMBuildGEP(builder,
1394                                       bld->imms_array, &lindex1, 1, "");
1395             res2 = LLVMBuildLoad(builder, imms_ptr2, "");
1396             res = emit_fetch_double(bld_base, stype, res, res2);
1397          }
1398       }
1399    }
1400    else {
1401       res = bld->immediates[reg->Register.Index][swizzle];
1402       if (stype == TGSI_TYPE_DOUBLE)
1403          res = emit_fetch_double(bld_base, stype, res, bld->immediates[reg->Register.Index][swizzle + 1]);
1404    }
1405
1406    if (stype == TGSI_TYPE_UNSIGNED) {
1407       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1408    } else if (stype == TGSI_TYPE_SIGNED) {
1409       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1410    } else if (stype == TGSI_TYPE_DOUBLE) {
1411       res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1412    }
1413    return res;
1414 }
1415
1416 static LLVMValueRef
1417 emit_fetch_input(
1418    struct lp_build_tgsi_context * bld_base,
1419    const struct tgsi_full_src_register * reg,
1420    enum tgsi_opcode_type stype,
1421    unsigned swizzle)
1422 {
1423    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1424    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1425    LLVMBuilderRef builder = gallivm->builder;
1426    LLVMValueRef res;
1427
1428    if (reg->Register.Indirect) {
1429       LLVMValueRef indirect_index;
1430       LLVMValueRef index_vec;  /* index into the input reg array */
1431       LLVMValueRef index_vec2 = NULL;
1432       LLVMValueRef inputs_array;
1433       LLVMTypeRef fptr_type;
1434
1435       indirect_index = get_indirect_index(bld,
1436                                           reg->Register.File,
1437                                           reg->Register.Index,
1438                                           &reg->Indirect);
1439
1440       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1441                                         indirect_index,
1442                                         swizzle,
1443                                         TRUE);
1444       if (stype == TGSI_TYPE_DOUBLE) {
1445          index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
1446                                            indirect_index,
1447                                            swizzle + 1,
1448                                            TRUE);
1449       }
1450       /* cast inputs_array pointer to float* */
1451       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1452       inputs_array = LLVMBuildBitCast(builder, bld->inputs_array, fptr_type, "");
1453
1454       /* Gather values from the input register array */
1455       res = build_gather(bld_base, inputs_array, index_vec, NULL, index_vec2);
1456    } else {
1457       if (bld->indirect_files & (1 << TGSI_FILE_INPUT)) {
1458          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1459                                         reg->Register.Index * 4 + swizzle);
1460          LLVMValueRef input_ptr = LLVMBuildGEP(builder,
1461                                                bld->inputs_array, &lindex, 1, "");
1462
1463          res = LLVMBuildLoad(builder, input_ptr, "");
1464          if (stype == TGSI_TYPE_DOUBLE) {
1465             LLVMValueRef lindex1;
1466             LLVMValueRef input_ptr2;
1467             LLVMValueRef res2;
1468
1469             lindex1 = lp_build_const_int32(gallivm,
1470                                            reg->Register.Index * 4 + swizzle + 1);
1471             input_ptr2 = LLVMBuildGEP(builder,
1472                                       bld->inputs_array, &lindex1, 1, "");
1473             res2 = LLVMBuildLoad(builder, input_ptr2, "");
1474             res = emit_fetch_double(bld_base, stype, res, res2);
1475          }
1476       }
1477       else {
1478          res = bld->inputs[reg->Register.Index][swizzle];
1479          if (stype == TGSI_TYPE_DOUBLE)
1480             res = emit_fetch_double(bld_base, stype, res, bld->inputs[reg->Register.Index][swizzle + 1]);
1481       }
1482    }
1483
1484    assert(res);
1485
1486    if (stype == TGSI_TYPE_UNSIGNED) {
1487       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1488    } else if (stype == TGSI_TYPE_SIGNED) {
1489       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1490    } else if (stype == TGSI_TYPE_DOUBLE) {
1491       res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1492    }
1493
1494    return res;
1495 }
1496
1497
1498 static LLVMValueRef
1499 emit_fetch_gs_input(
1500    struct lp_build_tgsi_context * bld_base,
1501    const struct tgsi_full_src_register * reg,
1502    enum tgsi_opcode_type stype,
1503    unsigned swizzle)
1504 {
1505    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1506    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1507    const struct tgsi_shader_info *info = bld->bld_base.info;
1508    LLVMBuilderRef builder = gallivm->builder;
1509    LLVMValueRef attrib_index = NULL;
1510    LLVMValueRef vertex_index = NULL;
1511    LLVMValueRef swizzle_index = lp_build_const_int32(gallivm, swizzle);
1512    LLVMValueRef res;
1513
1514    if (info->input_semantic_name[reg->Register.Index] == TGSI_SEMANTIC_PRIMID) {
1515       /* This is really a system value not a regular input */
1516       assert(!reg->Register.Indirect);
1517       assert(!reg->Dimension.Indirect);
1518       res = bld->system_values.prim_id;
1519       if (stype != TGSI_TYPE_UNSIGNED && stype != TGSI_TYPE_SIGNED) {
1520          res = LLVMBuildBitCast(builder, res, bld_base->base.vec_type, "");
1521       }
1522       return res;
1523    }
1524
1525    if (reg->Register.Indirect) {
1526       attrib_index = get_indirect_index(bld,
1527                                         reg->Register.File,
1528                                         reg->Register.Index,
1529                                         &reg->Indirect);
1530    } else {
1531       attrib_index = lp_build_const_int32(gallivm, reg->Register.Index);
1532    }
1533
1534    if (reg->Dimension.Indirect) {
1535       vertex_index = get_indirect_index(bld,
1536                                         reg->Register.File,
1537                                         reg->Dimension.Index,
1538                                         &reg->DimIndirect);
1539    } else {
1540       vertex_index = lp_build_const_int32(gallivm, reg->Dimension.Index);
1541    }
1542
1543    res = bld->gs_iface->fetch_input(bld->gs_iface, bld_base,
1544                                     reg->Dimension.Indirect,
1545                                     vertex_index,
1546                                     reg->Register.Indirect,
1547                                     attrib_index,
1548                                     swizzle_index);
1549
1550    assert(res);
1551
1552    if (stype == TGSI_TYPE_UNSIGNED) {
1553       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1554    } else if (stype == TGSI_TYPE_SIGNED) {
1555       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1556    } else if (stype == TGSI_TYPE_DOUBLE) {
1557       res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
1558    }
1559
1560    return res;
1561 }
1562
1563 static LLVMValueRef
1564 emit_fetch_temporary(
1565    struct lp_build_tgsi_context * bld_base,
1566    const struct tgsi_full_src_register * reg,
1567    enum tgsi_opcode_type stype,
1568    unsigned swizzle)
1569 {
1570    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1571    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1572    LLVMBuilderRef builder = gallivm->builder;
1573    LLVMValueRef res;
1574
1575    if (reg->Register.Indirect) {
1576       LLVMValueRef indirect_index;
1577       LLVMValueRef index_vec, index_vec2 = NULL;  /* index into the temp reg array */
1578       LLVMValueRef temps_array;
1579       LLVMTypeRef fptr_type;
1580
1581       indirect_index = get_indirect_index(bld,
1582                                           reg->Register.File,
1583                                           reg->Register.Index,
1584                                           &reg->Indirect);
1585
1586       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1587                                         indirect_index,
1588                                         swizzle,
1589                                         TRUE);
1590       if (stype == TGSI_TYPE_DOUBLE) {
1591                index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
1592                                                   indirect_index,
1593                                                   swizzle + 1,
1594                                                   TRUE);
1595       }
1596
1597       /* cast temps_array pointer to float* */
1598       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1599       temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1600
1601       /* Gather values from the temporary register array */
1602       res = build_gather(bld_base, temps_array, index_vec, NULL, index_vec2);
1603    }
1604    else {
1605       LLVMValueRef temp_ptr;
1606       temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle);
1607       res = LLVMBuildLoad(builder, temp_ptr, "");
1608
1609       if (stype == TGSI_TYPE_DOUBLE) {
1610          LLVMValueRef temp_ptr2, res2;
1611
1612          temp_ptr2 = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle + 1);
1613          res2 = LLVMBuildLoad(builder, temp_ptr2, "");
1614          res = emit_fetch_double(bld_base, stype, res, res2);
1615       }
1616    }
1617
1618    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED || stype == TGSI_TYPE_DOUBLE) {
1619       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1620       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1621    }
1622
1623    return res;
1624 }
1625
1626 static LLVMValueRef
1627 emit_fetch_system_value(
1628    struct lp_build_tgsi_context * bld_base,
1629    const struct tgsi_full_src_register * reg,
1630    enum tgsi_opcode_type stype,
1631    unsigned swizzle)
1632 {
1633    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1634    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1635    const struct tgsi_shader_info *info = bld->bld_base.info;
1636    LLVMBuilderRef builder = gallivm->builder;
1637    LLVMValueRef res;
1638    enum tgsi_opcode_type atype; // Actual type of the value
1639
1640    assert(!reg->Register.Indirect);
1641
1642    switch (info->system_value_semantic_name[reg->Register.Index]) {
1643    case TGSI_SEMANTIC_INSTANCEID:
1644       res = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
1645       atype = TGSI_TYPE_UNSIGNED;
1646       break;
1647
1648    case TGSI_SEMANTIC_VERTEXID:
1649       res = bld->system_values.vertex_id;
1650       atype = TGSI_TYPE_UNSIGNED;
1651       break;
1652
1653    case TGSI_SEMANTIC_VERTEXID_NOBASE:
1654       res = bld->system_values.vertex_id_nobase;
1655       atype = TGSI_TYPE_UNSIGNED;
1656       break;
1657
1658    case TGSI_SEMANTIC_BASEVERTEX:
1659       res = bld->system_values.basevertex;
1660       atype = TGSI_TYPE_UNSIGNED;
1661       break;
1662
1663    case TGSI_SEMANTIC_PRIMID:
1664       res = bld->system_values.prim_id;
1665       atype = TGSI_TYPE_UNSIGNED;
1666       break;
1667
1668    case TGSI_SEMANTIC_INVOCATIONID:
1669       res = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.invocation_id);
1670       atype = TGSI_TYPE_UNSIGNED;
1671       break;
1672
1673    default:
1674       assert(!"unexpected semantic in emit_fetch_system_value");
1675       res = bld_base->base.zero;
1676       atype = TGSI_TYPE_FLOAT;
1677       break;
1678    }
1679
1680    if (atype != stype) {
1681       if (stype == TGSI_TYPE_FLOAT) {
1682          res = LLVMBuildBitCast(builder, res, bld_base->base.vec_type, "");
1683       } else if (stype == TGSI_TYPE_UNSIGNED) {
1684          res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1685       } else if (stype == TGSI_TYPE_SIGNED) {
1686          res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1687       }
1688    }
1689
1690    return res;
1691 }
1692
1693 /**
1694  * Register fetch with derivatives.
1695  */
1696 static void
1697 emit_fetch_deriv(
1698    struct lp_build_tgsi_soa_context *bld,
1699    LLVMValueRef src,
1700    LLVMValueRef *res,
1701    LLVMValueRef *ddx,
1702    LLVMValueRef *ddy)
1703 {
1704    if(res)
1705       *res = src;
1706
1707    /* TODO: use interpolation coeffs for inputs */
1708
1709    if(ddx)
1710       *ddx = lp_build_ddx(&bld->bld_base.base, src);
1711
1712    if(ddy)
1713       *ddy = lp_build_ddy(&bld->bld_base.base, src);
1714 }
1715
1716
1717 /**
1718  * Predicate.
1719  */
1720 static void
1721 emit_fetch_predicate(
1722    struct lp_build_tgsi_soa_context *bld,
1723    const struct tgsi_full_instruction *inst,
1724    LLVMValueRef *pred)
1725 {
1726    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1727    unsigned index;
1728    unsigned char swizzles[4];
1729    LLVMValueRef unswizzled[4] = {NULL, NULL, NULL, NULL};
1730    LLVMValueRef value;
1731    unsigned chan;
1732
1733    if (!inst->Instruction.Predicate) {
1734       TGSI_FOR_EACH_CHANNEL( chan ) {
1735          pred[chan] = NULL;
1736       }
1737       return;
1738    }
1739
1740    swizzles[0] = inst->Predicate.SwizzleX;
1741    swizzles[1] = inst->Predicate.SwizzleY;
1742    swizzles[2] = inst->Predicate.SwizzleZ;
1743    swizzles[3] = inst->Predicate.SwizzleW;
1744
1745    index = inst->Predicate.Index;
1746    assert(index < LP_MAX_TGSI_PREDS);
1747
1748    TGSI_FOR_EACH_CHANNEL( chan ) {
1749       unsigned swizzle = swizzles[chan];
1750
1751       /*
1752        * Only fetch the predicate register channels that are actually listed
1753        * in the swizzles
1754        */
1755       if (!unswizzled[swizzle]) {
1756          value = LLVMBuildLoad(builder,
1757                                bld->preds[index][swizzle], "");
1758
1759          /*
1760           * Convert the value to an integer mask.
1761           *
1762           * TODO: Short-circuit this comparison -- a D3D setp_xx instructions
1763           * is needlessly causing two comparisons due to storing the intermediate
1764           * result as float vector instead of an integer mask vector.
1765           */
1766          value = lp_build_compare(bld->bld_base.base.gallivm,
1767                                   bld->bld_base.base.type,
1768                                   PIPE_FUNC_NOTEQUAL,
1769                                   value,
1770                                   bld->bld_base.base.zero);
1771          if (inst->Predicate.Negate) {
1772             value = LLVMBuildNot(builder, value, "");
1773          }
1774
1775          unswizzled[swizzle] = value;
1776       } else {
1777          value = unswizzled[swizzle];
1778       }
1779
1780       pred[chan] = value;
1781    }
1782 }
1783
1784 /**
1785  * store an array of 8 doubles into two arrays of 8 floats
1786  * i.e.
1787  * value is d0, d1, d2, d3 etc.
1788  * each double has high and low pieces x, y
1789  * so gets stored into the separate channels as:
1790  * chan_ptr = d0.x, d1.x, d2.x, d3.x
1791  * chan_ptr2 = d0.y, d1.y, d2.y, d3.y
1792  */
1793 static void
1794 emit_store_double_chan(struct lp_build_tgsi_context *bld_base,
1795                        int dtype,
1796                        LLVMValueRef chan_ptr, LLVMValueRef chan_ptr2,
1797                        LLVMValueRef pred,
1798                        LLVMValueRef value)
1799 {
1800    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1801    struct gallivm_state *gallivm = bld_base->base.gallivm;
1802    LLVMBuilderRef builder = gallivm->builder;
1803    struct lp_build_context *float_bld = &bld_base->base;
1804    int i;
1805    LLVMValueRef temp, temp2;
1806    LLVMValueRef shuffles[8];
1807    LLVMValueRef shuffles2[8];
1808
1809    for (i = 0; i < bld_base->base.type.length; i++) {
1810       shuffles[i] = lp_build_const_int32(gallivm, i * 2);
1811       shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
1812    }
1813
1814    temp = LLVMBuildShuffleVector(builder, value,
1815                                  LLVMGetUndef(LLVMTypeOf(value)),
1816                                  LLVMConstVector(shuffles,
1817                                                  bld_base->base.type.length),
1818                                  "");
1819    temp2 = LLVMBuildShuffleVector(builder, value,
1820                                   LLVMGetUndef(LLVMTypeOf(value)),
1821                                   LLVMConstVector(shuffles2,
1822                                                   bld_base->base.type.length),
1823                                   "");
1824
1825    lp_exec_mask_store(&bld->exec_mask, float_bld, pred, temp, chan_ptr);
1826    lp_exec_mask_store(&bld->exec_mask, float_bld, pred, temp2, chan_ptr2);
1827 }
1828
1829 /**
1830  * Register store.
1831  */
1832 static void
1833 emit_store_chan(
1834    struct lp_build_tgsi_context *bld_base,
1835    const struct tgsi_full_instruction *inst,
1836    unsigned index,
1837    unsigned chan_index,
1838    LLVMValueRef pred,
1839    LLVMValueRef value)
1840 {
1841    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1842    struct gallivm_state *gallivm = bld_base->base.gallivm;
1843    LLVMBuilderRef builder = gallivm->builder;
1844    const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1845    struct lp_build_context *float_bld = &bld_base->base;
1846    struct lp_build_context *int_bld = &bld_base->int_bld;
1847    LLVMValueRef indirect_index = NULL;
1848    enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode);
1849
1850    /*
1851     * Apply saturation.
1852     *
1853     * It is always assumed to be float.
1854     */
1855    if (inst->Instruction.Saturate) {
1856       assert(dtype == TGSI_TYPE_FLOAT ||
1857              dtype == TGSI_TYPE_UNTYPED);
1858       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1859       value = lp_build_clamp_zero_one_nanzero(float_bld, value);
1860    }
1861
1862    if (reg->Register.Indirect) {
1863       /*
1864        * Currently the mesa/st doesn't generate indirect stores
1865        * to doubles, it normally uses MOV to do indirect stores.
1866        */
1867       assert(dtype != TGSI_TYPE_DOUBLE);
1868       indirect_index = get_indirect_index(bld,
1869                                           reg->Register.File,
1870                                           reg->Register.Index,
1871                                           &reg->Indirect);
1872    } else {
1873       assert(reg->Register.Index <=
1874                              bld_base->info->file_max[reg->Register.File]);
1875    }
1876
1877    if (DEBUG_EXECUTION) {
1878       emit_dump_reg(gallivm, reg->Register.File, reg->Register.Index, chan_index, value);
1879    }
1880
1881    switch( reg->Register.File ) {
1882    case TGSI_FILE_OUTPUT:
1883       /* Outputs are always stored as floats */
1884       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1885
1886       if (reg->Register.Indirect) {
1887          LLVMValueRef index_vec;  /* indexes into the output registers */
1888          LLVMValueRef outputs_array;
1889          LLVMTypeRef fptr_type;
1890
1891          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1892                                            indirect_index,
1893                                            chan_index,
1894                                            TRUE);
1895
1896          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1897          outputs_array = LLVMBuildBitCast(builder, bld->outputs_array, fptr_type, "");
1898
1899          /* Scatter store values into output registers */
1900          emit_mask_scatter(bld, outputs_array, index_vec, value,
1901                            &bld->exec_mask, pred);
1902       }
1903       else {
1904          LLVMValueRef out_ptr = lp_get_output_ptr(bld, reg->Register.Index,
1905                                                   chan_index);
1906
1907          if (dtype == TGSI_TYPE_DOUBLE) {
1908             LLVMValueRef out_ptr2 = lp_get_output_ptr(bld, reg->Register.Index,
1909                                                       chan_index + 1);
1910             emit_store_double_chan(bld_base, dtype, out_ptr, out_ptr2,
1911                                    pred, value);
1912          } else
1913             lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, out_ptr);
1914       }
1915       break;
1916
1917    case TGSI_FILE_TEMPORARY:
1918       /* Temporaries are always stored as floats */
1919       if (dtype != TGSI_TYPE_DOUBLE)
1920          value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1921       else
1922          value = LLVMBuildBitCast(builder, value,  LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2), "");
1923
1924       if (reg->Register.Indirect) {
1925          LLVMValueRef index_vec;  /* indexes into the temp registers */
1926          LLVMValueRef temps_array;
1927          LLVMTypeRef fptr_type;
1928
1929          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1930                                            indirect_index,
1931                                            chan_index,
1932                                            TRUE);
1933
1934          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1935          temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1936
1937          /* Scatter store values into temp registers */
1938          emit_mask_scatter(bld, temps_array, index_vec, value,
1939                            &bld->exec_mask, pred);
1940       }
1941       else {
1942          LLVMValueRef temp_ptr;
1943          temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, chan_index);
1944
1945          if (dtype == TGSI_TYPE_DOUBLE) {
1946             LLVMValueRef temp_ptr2 = lp_get_temp_ptr_soa(bld,
1947                                                          reg->Register.Index,
1948                                                          chan_index + 1);
1949             emit_store_double_chan(bld_base, dtype, temp_ptr, temp_ptr2,
1950                                    pred, value);
1951          }
1952          else
1953             lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, temp_ptr);
1954       }
1955       break;
1956
1957    case TGSI_FILE_ADDRESS:
1958       assert(dtype == TGSI_TYPE_SIGNED);
1959       assert(LLVMTypeOf(value) == int_bld->vec_type);
1960       value = LLVMBuildBitCast(builder, value, int_bld->vec_type, "");
1961       lp_exec_mask_store(&bld->exec_mask, int_bld, pred, value,
1962                          bld->addr[reg->Register.Index][chan_index]);
1963       break;
1964
1965    case TGSI_FILE_PREDICATE:
1966       assert(LLVMTypeOf(value) == float_bld->vec_type);
1967       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1968       lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value,
1969                          bld->preds[reg->Register.Index][chan_index]);
1970       break;
1971
1972    default:
1973       assert( 0 );
1974    }
1975
1976    (void)dtype;
1977 }
1978
1979 /*
1980  * Called at the beginning of the translation of each TGSI instruction, to
1981  * emit some debug code.
1982  */
1983 static void
1984 emit_debug(
1985    struct lp_build_tgsi_context * bld_base,
1986    const struct tgsi_full_instruction * inst,
1987    const struct tgsi_opcode_info * info)
1988
1989 {
1990    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1991
1992    if (DEBUG_EXECUTION) {
1993       /*
1994        * Dump the TGSI instruction.
1995        */
1996
1997       struct gallivm_state *gallivm = bld_base->base.gallivm;
1998       char buf[512];
1999       buf[0] = '$';
2000       buf[1] = ' ';
2001       tgsi_dump_instruction_str(inst, bld_base->pc, &buf[2], sizeof buf - 2);
2002       lp_build_printf(gallivm, buf);
2003
2004       /* Dump the execution mask.
2005        */
2006       if (bld->exec_mask.has_mask) {
2007          lp_build_print_value(gallivm, "    mask = ", bld->exec_mask.exec_mask);
2008       }
2009    }
2010 }
2011
2012 static void
2013 emit_store(
2014    struct lp_build_tgsi_context * bld_base,
2015    const struct tgsi_full_instruction * inst,
2016    const struct tgsi_opcode_info * info,
2017    LLVMValueRef dst[4])
2018
2019 {
2020    unsigned chan_index;
2021    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2022    enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode);
2023    if(info->num_dst) {
2024       LLVMValueRef pred[TGSI_NUM_CHANNELS];
2025
2026       emit_fetch_predicate( bld, inst, pred );
2027
2028       TGSI_FOR_EACH_DST0_ENABLED_CHANNEL( inst, chan_index ) {
2029
2030          if (dtype == TGSI_TYPE_DOUBLE && (chan_index == 1 || chan_index == 3))
2031              continue;
2032          emit_store_chan(bld_base, inst, 0, chan_index, pred[chan_index], dst[chan_index]);
2033       }
2034    }
2035 }
2036
2037 static unsigned
2038 tgsi_to_pipe_tex_target(unsigned tgsi_target)
2039 {
2040    switch (tgsi_target) {
2041    case TGSI_TEXTURE_BUFFER:
2042       return PIPE_BUFFER;
2043    case TGSI_TEXTURE_1D:
2044    case TGSI_TEXTURE_SHADOW1D:
2045       return PIPE_TEXTURE_1D;
2046    case TGSI_TEXTURE_2D:
2047    case TGSI_TEXTURE_SHADOW2D:
2048    case TGSI_TEXTURE_2D_MSAA:
2049       return PIPE_TEXTURE_2D;
2050    case TGSI_TEXTURE_3D:
2051       return PIPE_TEXTURE_3D;
2052    case TGSI_TEXTURE_CUBE:
2053    case TGSI_TEXTURE_SHADOWCUBE:
2054       return PIPE_TEXTURE_CUBE;
2055    case TGSI_TEXTURE_RECT:
2056    case TGSI_TEXTURE_SHADOWRECT:
2057       return PIPE_TEXTURE_RECT;
2058    case TGSI_TEXTURE_1D_ARRAY:
2059    case TGSI_TEXTURE_SHADOW1D_ARRAY:
2060       return PIPE_TEXTURE_1D_ARRAY;
2061    case TGSI_TEXTURE_2D_ARRAY:
2062    case TGSI_TEXTURE_SHADOW2D_ARRAY:
2063    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2064       return PIPE_TEXTURE_2D_ARRAY;
2065    case TGSI_TEXTURE_CUBE_ARRAY:
2066    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2067       return PIPE_TEXTURE_CUBE_ARRAY;
2068    default:
2069       assert(0);
2070       return PIPE_BUFFER;
2071    }
2072 }
2073
2074
2075 static enum lp_sampler_lod_property
2076 lp_build_lod_property(
2077    struct lp_build_tgsi_context *bld_base,
2078    const struct tgsi_full_instruction *inst,
2079    unsigned src_op)
2080 {
2081    const struct tgsi_full_src_register *reg = &inst->Src[src_op];
2082    enum lp_sampler_lod_property lod_property;
2083
2084    /*
2085     * Not much we can do here. We could try catching inputs declared
2086     * with constant interpolation but not sure it's worth it - since for
2087     * TEX opcodes as well as FETCH/LD the lod comes from same reg as
2088     * the coords, so it could only work for SAMPLE/TXQ/SVIEWINFO), just
2089     * like the constant/immediate recognition below.
2090     * What seems to be of more value would be to recognize temps holding
2091     * broadcasted scalars but no way we can do it.
2092     * Tried asking llvm but without any success (using LLVMIsConstant
2093     * even though this isn't exactly what we'd need), even as simple as
2094     * IMM[0] UINT32 (0,-1,0,0)
2095     * MOV TEMP[0] IMM[0].yyyy
2096     * SVIEWINFO TEMP[1], TEMP[0].xxxx, SVIEWINFO[0]
2097     * doesn't work.
2098     * This means there's ZERO chance this will ever catch a scalar lod
2099     * with traditional tex opcodes as well as texel fetches, since the lod
2100     * comes from the same reg as coords (except some test shaders using
2101     * constant coords maybe).
2102     * There's at least hope for sample opcodes as well as size queries.
2103     */
2104    if (reg->Register.File == TGSI_FILE_CONSTANT ||
2105        reg->Register.File == TGSI_FILE_IMMEDIATE) {
2106       lod_property = LP_SAMPLER_LOD_SCALAR;
2107    }
2108    else if (bld_base->info->processor == TGSI_PROCESSOR_FRAGMENT) {
2109       if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2110          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2111       }
2112       else {
2113          lod_property = LP_SAMPLER_LOD_PER_QUAD;
2114       }
2115    }
2116    else {
2117       /* never use scalar (per-quad) lod the results are just too wrong. */
2118       lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2119    }
2120    return lod_property;
2121 }
2122
2123
2124 /**
2125  * High-level instruction translators.
2126  */
2127
2128 static void
2129 emit_tex( struct lp_build_tgsi_soa_context *bld,
2130           const struct tgsi_full_instruction *inst,
2131           enum lp_build_tex_modifier modifier,
2132           LLVMValueRef *texel,
2133           unsigned sampler_reg,
2134           enum lp_sampler_op_type sampler_op)
2135 {
2136    unsigned unit = inst->Src[sampler_reg].Register.Index;
2137    LLVMValueRef oow = NULL;
2138    LLVMValueRef lod = NULL;
2139    LLVMValueRef coords[5];
2140    LLVMValueRef offsets[3] = { NULL };
2141    struct lp_derivatives derivs;
2142    struct lp_sampler_params params;
2143    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2144    unsigned num_derivs, num_offsets, i;
2145    unsigned shadow_coord = 0;
2146    unsigned layer_coord = 0;
2147    unsigned sample_key = sampler_op << LP_SAMPLER_OP_TYPE_SHIFT;
2148
2149    memset(&params, 0, sizeof(params));
2150
2151    if (!bld->sampler) {
2152       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2153       for (i = 0; i < 4; i++) {
2154          texel[i] = bld->bld_base.base.undef;
2155       }
2156       return;
2157    }
2158
2159    switch (inst->Texture.Texture) {
2160    case TGSI_TEXTURE_1D_ARRAY:
2161       layer_coord = 1;
2162       /* fallthrough */
2163    case TGSI_TEXTURE_1D:
2164       num_offsets = 1;
2165       num_derivs = 1;
2166       break;
2167    case TGSI_TEXTURE_2D_ARRAY:
2168       layer_coord = 2;
2169       /* fallthrough */
2170    case TGSI_TEXTURE_2D:
2171    case TGSI_TEXTURE_RECT:
2172       num_offsets = 2;
2173       num_derivs = 2;
2174       break;
2175    case TGSI_TEXTURE_SHADOW1D_ARRAY:
2176       layer_coord = 1;
2177       /* fallthrough */
2178    case TGSI_TEXTURE_SHADOW1D:
2179       shadow_coord = 2;
2180       num_offsets = 1;
2181       num_derivs = 1;
2182       break;
2183    case TGSI_TEXTURE_SHADOW2D_ARRAY:
2184       layer_coord = 2;
2185       shadow_coord = 3;
2186       num_offsets = 2;
2187       num_derivs = 2;
2188       break;
2189    case TGSI_TEXTURE_SHADOW2D:
2190    case TGSI_TEXTURE_SHADOWRECT:
2191       shadow_coord = 2;
2192       num_offsets = 2;
2193       num_derivs = 2;
2194       break;
2195    case TGSI_TEXTURE_CUBE:
2196       num_offsets = 2;
2197       num_derivs = 3;
2198       break;
2199    case TGSI_TEXTURE_3D:
2200       num_offsets = 3;
2201       num_derivs = 3;
2202       break;
2203    case TGSI_TEXTURE_SHADOWCUBE:
2204       shadow_coord = 3;
2205       num_offsets = 2;
2206       num_derivs = 3;
2207       break;
2208    case TGSI_TEXTURE_CUBE_ARRAY:
2209       num_offsets = 2;
2210       num_derivs = 3;
2211       layer_coord = 3;
2212       break;
2213    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2214       num_offsets = 2;
2215       num_derivs = 3;
2216       layer_coord = 3;
2217       shadow_coord = 4; /* shadow coord special different reg */
2218       break;
2219    case TGSI_TEXTURE_2D_MSAA:
2220    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2221    default:
2222       assert(0);
2223       return;
2224    }
2225
2226    /* Note lod and especially projected are illegal in a LOT of cases */
2227    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2228        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2229       if (inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
2230           inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY) {
2231          /* note that shadow cube array with bias/explicit lod does not exist */
2232          lod = lp_build_emit_fetch(&bld->bld_base, inst, 1, 0);
2233       }
2234       else {
2235          lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2236       }
2237       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2238          sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
2239       }
2240       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2241          sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2242       }
2243       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2244    }
2245
2246    if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED) {
2247       oow = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2248       oow = lp_build_rcp(&bld->bld_base.base, oow);
2249    }
2250
2251    for (i = 0; i < num_derivs; i++) {
2252       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2253       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2254          coords[i] = lp_build_mul(&bld->bld_base.base, coords[i], oow);
2255    }
2256    for (i = num_derivs; i < 5; i++) {
2257       coords[i] = bld->bld_base.base.undef;
2258    }
2259
2260    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2261    if (layer_coord) {
2262       if (layer_coord == 3) {
2263          coords[3] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2264       }
2265       else {
2266          coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2267       }
2268       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2269          coords[2] = lp_build_mul(&bld->bld_base.base, coords[2], oow);
2270    }
2271    /* Shadow coord occupies always 5th slot. */
2272    if (shadow_coord) {
2273       sample_key |= LP_SAMPLER_SHADOW;
2274       if (shadow_coord == 4) {
2275          coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 1, 0);
2276       }
2277       else {
2278          coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 0, shadow_coord);
2279       }
2280       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2281          coords[4] = lp_build_mul(&bld->bld_base.base, coords[4], oow);
2282    }
2283
2284    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2285       unsigned dim;
2286       sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
2287       for (dim = 0; dim < num_derivs; ++dim) {
2288          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 1, dim);
2289          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 2, dim);
2290       }
2291       params.derivs = &derivs;
2292       /*
2293        * could also check all src regs if constant but I doubt such
2294        * cases exist in practice.
2295        */
2296       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2297          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2298             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2299          }
2300          else {
2301             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2302          }
2303       }
2304       else {
2305          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2306       }
2307    }
2308    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
2309
2310    /* we don't handle the 4 offset version of tg4 */
2311    if (inst->Texture.NumOffsets == 1) {
2312       unsigned dim;
2313       sample_key |= LP_SAMPLER_OFFSETS;
2314       for (dim = 0; dim < num_offsets; dim++) {
2315          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2316       }
2317    }
2318
2319    params.type = bld->bld_base.base.type;
2320    params.sample_key = sample_key;
2321    params.texture_index = unit;
2322    params.sampler_index = unit;
2323    params.context_ptr = bld->context_ptr;
2324    params.thread_data_ptr = bld->thread_data_ptr;
2325    params.coords = coords;
2326    params.offsets = offsets;
2327    params.lod = lod;
2328    params.texel = texel;
2329
2330    bld->sampler->emit_tex_sample(bld->sampler,
2331                                  bld->bld_base.base.gallivm,
2332                                  &params);
2333 }
2334
2335 static void
2336 emit_sample(struct lp_build_tgsi_soa_context *bld,
2337             const struct tgsi_full_instruction *inst,
2338             enum lp_build_tex_modifier modifier,
2339             boolean compare,
2340             LLVMValueRef *texel)
2341 {
2342    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2343    unsigned texture_unit, sampler_unit;
2344    LLVMValueRef lod = NULL;
2345    LLVMValueRef coords[5];
2346    LLVMValueRef offsets[3] = { NULL };
2347    struct lp_derivatives derivs;
2348    struct lp_sampler_params params;
2349    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2350
2351    unsigned num_offsets, num_derivs, i;
2352    unsigned layer_coord = 0;
2353    unsigned sample_key = LP_SAMPLER_OP_TEXTURE << LP_SAMPLER_OP_TYPE_SHIFT;
2354
2355    memset(&params, 0, sizeof(params));
2356
2357    if (!bld->sampler) {
2358       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2359       for (i = 0; i < 4; i++) {
2360          texel[i] = bld->bld_base.base.undef;
2361       }
2362       return;
2363    }
2364
2365    /*
2366     * unlike old-style tex opcodes the texture/sampler indices
2367     * always come from src1 and src2 respectively.
2368     */
2369    texture_unit = inst->Src[1].Register.Index;
2370    sampler_unit = inst->Src[2].Register.Index;
2371
2372    /*
2373     * Note inst->Texture.Texture will contain the number of offsets,
2374     * however the target information is NOT there and comes from the
2375     * declared sampler views instead.
2376     */
2377    switch (bld->sv[texture_unit].Resource) {
2378    case TGSI_TEXTURE_1D:
2379       num_offsets = 1;
2380       num_derivs = 1;
2381       break;
2382    case TGSI_TEXTURE_1D_ARRAY:
2383       layer_coord = 1;
2384       num_offsets = 1;
2385       num_derivs = 1;
2386       break;
2387    case TGSI_TEXTURE_2D:
2388    case TGSI_TEXTURE_RECT:
2389       num_offsets = 2;
2390       num_derivs = 2;
2391       break;
2392    case TGSI_TEXTURE_2D_ARRAY:
2393       layer_coord = 2;
2394       num_offsets = 2;
2395       num_derivs = 2;
2396       break;
2397    case TGSI_TEXTURE_CUBE:
2398       num_offsets = 2;
2399       num_derivs = 3;
2400       break;
2401    case TGSI_TEXTURE_3D:
2402       num_offsets = 3;
2403       num_derivs = 3;
2404       break;
2405    case TGSI_TEXTURE_CUBE_ARRAY:
2406       layer_coord = 3;
2407       num_offsets = 2;
2408       num_derivs = 3;
2409       break;
2410    default:
2411       assert(0);
2412       return;
2413    }
2414
2415    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2416        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2417       lod = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2418       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2419          sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
2420       }
2421       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2422          sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2423       }
2424       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2425    }
2426    else if (modifier == LP_BLD_TEX_MODIFIER_LOD_ZERO) {
2427       /* XXX might be better to explicitly pass the level zero information */
2428       sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2429       lod = lp_build_const_vec(gallivm, bld->bld_base.base.type, 0.0F);
2430    }
2431
2432    for (i = 0; i < num_derivs; i++) {
2433       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2434    }
2435    for (i = num_derivs; i < 5; i++) {
2436       coords[i] = bld->bld_base.base.undef;
2437    }
2438
2439    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2440    if (layer_coord) {
2441       if (layer_coord == 3)
2442          coords[3] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2443       else
2444          coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2445    }
2446    /* Shadow coord occupies always 5th slot. */
2447    if (compare) {
2448       sample_key |= LP_SAMPLER_SHADOW;
2449       coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2450    }
2451
2452    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2453       unsigned dim;
2454       sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
2455       for (dim = 0; dim < num_derivs; ++dim) {
2456          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 3, dim);
2457          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 4, dim);
2458       }
2459       params.derivs = &derivs;
2460       /*
2461        * could also check all src regs if constant but I doubt such
2462        * cases exist in practice.
2463        */
2464       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2465          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2466             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2467          }
2468          else {
2469             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2470          }
2471       }
2472       else {
2473          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2474       }
2475    }
2476
2477    /* some advanced gather instructions (txgo) would require 4 offsets */
2478    if (inst->Texture.NumOffsets == 1) {
2479       unsigned dim;
2480       sample_key |= LP_SAMPLER_OFFSETS;
2481       for (dim = 0; dim < num_offsets; dim++) {
2482          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2483       }
2484    }
2485    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
2486
2487    params.type = bld->bld_base.base.type;
2488    params.sample_key = sample_key;
2489    params.texture_index = texture_unit;
2490    params.sampler_index = sampler_unit;
2491    params.context_ptr = bld->context_ptr;
2492    params.thread_data_ptr = bld->thread_data_ptr;
2493    params.coords = coords;
2494    params.offsets = offsets;
2495    params.lod = lod;
2496    params.texel = texel;
2497
2498    bld->sampler->emit_tex_sample(bld->sampler,
2499                                  bld->bld_base.base.gallivm,
2500                                  &params);
2501
2502    if (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2503        inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2504        inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2505        inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA) {
2506       unsigned char swizzles[4];
2507       swizzles[0] = inst->Src[1].Register.SwizzleX;
2508       swizzles[1] = inst->Src[1].Register.SwizzleY;
2509       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2510       swizzles[3] = inst->Src[1].Register.SwizzleW;
2511
2512       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2513    }
2514 }
2515
2516 static void
2517 emit_fetch_texels( struct lp_build_tgsi_soa_context *bld,
2518                    const struct tgsi_full_instruction *inst,
2519                    LLVMValueRef *texel,
2520                    boolean is_samplei)
2521 {
2522    unsigned unit, target;
2523    LLVMValueRef coord_undef = LLVMGetUndef(bld->bld_base.base.int_vec_type);
2524    LLVMValueRef explicit_lod = NULL;
2525    LLVMValueRef coords[5];
2526    LLVMValueRef offsets[3] = { NULL };
2527    struct lp_sampler_params params;
2528    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2529    unsigned dims, i;
2530    unsigned layer_coord = 0;
2531    unsigned sample_key = LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
2532
2533    memset(&params, 0, sizeof(params));
2534
2535    if (!bld->sampler) {
2536       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2537       for (i = 0; i < 4; i++) {
2538          texel[i] = coord_undef;
2539       }
2540       return;
2541    }
2542
2543    unit = inst->Src[1].Register.Index;
2544
2545    if (is_samplei) {
2546       target = bld->sv[unit].Resource;
2547    }
2548    else {
2549       target = inst->Texture.Texture;
2550    }
2551
2552    switch (target) {
2553    case TGSI_TEXTURE_1D:
2554    case TGSI_TEXTURE_BUFFER:
2555       dims = 1;
2556       break;
2557    case TGSI_TEXTURE_1D_ARRAY:
2558       layer_coord = 1;
2559       dims = 1;
2560       break;
2561    case TGSI_TEXTURE_2D:
2562    case TGSI_TEXTURE_RECT:
2563    case TGSI_TEXTURE_2D_MSAA:
2564       dims = 2;
2565       break;
2566    case TGSI_TEXTURE_2D_ARRAY:
2567    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2568       layer_coord = 2;
2569       dims = 2;
2570       break;
2571    case TGSI_TEXTURE_3D:
2572       dims = 3;
2573       break;
2574    default:
2575       assert(0);
2576       return;
2577    }
2578
2579    /* always have lod except for buffers and msaa targets ? */
2580    if (target != TGSI_TEXTURE_BUFFER &&
2581        target != TGSI_TEXTURE_2D_MSAA &&
2582        target != TGSI_TEXTURE_2D_ARRAY_MSAA) {
2583       sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
2584       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2585       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2586    }
2587    /* XXX: for real msaa support, the w component would be the sample index. */
2588
2589    for (i = 0; i < dims; i++) {
2590       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2591    }
2592    /* never use more than 3 coords here but emit_fetch_texel copies all 5 anyway */
2593    for (i = dims; i < 5; i++) {
2594       coords[i] = coord_undef;
2595    }
2596    if (layer_coord)
2597       coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2598
2599    if (inst->Texture.NumOffsets == 1) {
2600       unsigned dim;
2601       sample_key |= LP_SAMPLER_OFFSETS;
2602       for (dim = 0; dim < dims; dim++) {
2603          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2604       }
2605    }
2606    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
2607
2608    params.type = bld->bld_base.base.type;
2609    params.sample_key = sample_key;
2610    params.texture_index = unit;
2611    params.sampler_index = unit;
2612    params.context_ptr = bld->context_ptr;
2613    params.thread_data_ptr = bld->thread_data_ptr;
2614    params.coords = coords;
2615    params.offsets = offsets;
2616    params.derivs = NULL;
2617    params.lod = explicit_lod;
2618    params.texel = texel;
2619
2620    bld->sampler->emit_tex_sample(bld->sampler,
2621                                  bld->bld_base.base.gallivm,
2622                                  &params);
2623
2624    if (is_samplei &&
2625        (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2626         inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2627         inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2628         inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA)) {
2629       unsigned char swizzles[4];
2630       swizzles[0] = inst->Src[1].Register.SwizzleX;
2631       swizzles[1] = inst->Src[1].Register.SwizzleY;
2632       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2633       swizzles[3] = inst->Src[1].Register.SwizzleW;
2634
2635       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2636    }
2637 }
2638
2639 static void
2640 emit_size_query( struct lp_build_tgsi_soa_context *bld,
2641                  const struct tgsi_full_instruction *inst,
2642                  LLVMValueRef *sizes_out,
2643                  boolean is_sviewinfo)
2644 {
2645    LLVMValueRef explicit_lod;
2646    enum lp_sampler_lod_property lod_property;
2647    unsigned has_lod;
2648    unsigned i;
2649    unsigned unit = inst->Src[1].Register.Index;
2650    unsigned target, pipe_target;
2651
2652    if (is_sviewinfo) {
2653       target = bld->sv[unit].Resource;
2654    }
2655    else {
2656       target = inst->Texture.Texture;
2657    }
2658    switch (target) {
2659    case TGSI_TEXTURE_BUFFER:
2660    case TGSI_TEXTURE_RECT:
2661    case TGSI_TEXTURE_SHADOWRECT:
2662       has_lod = 0;
2663       break;
2664    default:
2665       has_lod = 1;
2666       break;
2667    }
2668
2669    if (!bld->sampler) {
2670       _debug_printf("warning: found texture query instruction but no sampler generator supplied\n");
2671       for (i = 0; i < 4; i++)
2672          sizes_out[i] = bld->bld_base.int_bld.undef;
2673       return;
2674    }
2675
2676    if (has_lod) {
2677       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 0);
2678       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2679    }
2680    else {
2681       explicit_lod = NULL;
2682       lod_property = LP_SAMPLER_LOD_SCALAR;
2683    }
2684
2685
2686    pipe_target = tgsi_to_pipe_tex_target(target);
2687
2688    bld->sampler->emit_size_query(bld->sampler,
2689                                  bld->bld_base.base.gallivm,
2690                                  bld->bld_base.int_bld.type,
2691                                  unit, pipe_target,
2692                                  bld->context_ptr,
2693                                  TRUE,
2694                                  lod_property,
2695                                  explicit_lod,
2696                                  sizes_out);
2697 }
2698
2699 static boolean
2700 near_end_of_shader(struct lp_build_tgsi_soa_context *bld,
2701                    int pc)
2702 {
2703    int i;
2704
2705    for (i = 0; i < 5; i++) {
2706       unsigned opcode;
2707
2708       if (pc + i >= bld->bld_base.info->num_instructions)
2709          return TRUE;
2710
2711       opcode = bld->bld_base.instructions[pc + i].Instruction.Opcode;
2712
2713       if (opcode == TGSI_OPCODE_END)
2714          return TRUE;
2715
2716       if (opcode == TGSI_OPCODE_TEX ||
2717          opcode == TGSI_OPCODE_TXP ||
2718          opcode == TGSI_OPCODE_TXD ||
2719          opcode == TGSI_OPCODE_TXB ||
2720          opcode == TGSI_OPCODE_TXL ||
2721          opcode == TGSI_OPCODE_TXF ||
2722          opcode == TGSI_OPCODE_TXQ ||
2723          opcode == TGSI_OPCODE_TEX2 ||
2724          opcode == TGSI_OPCODE_TXB2 ||
2725          opcode == TGSI_OPCODE_TXL2 ||
2726          opcode == TGSI_OPCODE_SAMPLE ||
2727          opcode == TGSI_OPCODE_SAMPLE_B ||
2728          opcode == TGSI_OPCODE_SAMPLE_C ||
2729          opcode == TGSI_OPCODE_SAMPLE_C_LZ ||
2730          opcode == TGSI_OPCODE_SAMPLE_D ||
2731          opcode == TGSI_OPCODE_SAMPLE_I ||
2732          opcode == TGSI_OPCODE_SAMPLE_L ||
2733          opcode == TGSI_OPCODE_SVIEWINFO ||
2734          opcode == TGSI_OPCODE_CAL ||
2735          opcode == TGSI_OPCODE_CALLNZ ||
2736          opcode == TGSI_OPCODE_IF ||
2737          opcode == TGSI_OPCODE_UIF ||
2738          opcode == TGSI_OPCODE_BGNLOOP ||
2739          opcode == TGSI_OPCODE_SWITCH)
2740          return FALSE;
2741    }
2742
2743    return TRUE;
2744 }
2745
2746
2747
2748 /**
2749  * Kill fragment if any of the src register values are negative.
2750  */
2751 static void
2752 emit_kill_if(
2753    struct lp_build_tgsi_soa_context *bld,
2754    const struct tgsi_full_instruction *inst,
2755    int pc)
2756 {
2757    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2758    const struct tgsi_full_src_register *reg = &inst->Src[0];
2759    LLVMValueRef terms[TGSI_NUM_CHANNELS];
2760    LLVMValueRef mask;
2761    unsigned chan_index;
2762
2763    memset(&terms, 0, sizeof terms);
2764
2765    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2766       unsigned swizzle;
2767
2768       /* Unswizzle channel */
2769       swizzle = tgsi_util_get_full_src_register_swizzle( reg, chan_index );
2770
2771       /* Check if the component has not been already tested. */
2772       assert(swizzle < TGSI_NUM_CHANNELS);
2773       if( !terms[swizzle] )
2774          /* TODO: change the comparison operator instead of setting the sign */
2775          terms[swizzle] =  lp_build_emit_fetch(&bld->bld_base, inst, 0, chan_index );
2776    }
2777
2778    mask = NULL;
2779    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2780       if(terms[chan_index]) {
2781          LLVMValueRef chan_mask;
2782
2783          /*
2784           * If term < 0 then mask = 0 else mask = ~0.
2785           */
2786          chan_mask = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GEQUAL, terms[chan_index], bld->bld_base.base.zero);
2787
2788          if(mask)
2789             mask = LLVMBuildAnd(builder, mask, chan_mask, "");
2790          else
2791             mask = chan_mask;
2792       }
2793    }
2794
2795    if (bld->exec_mask.has_mask) {
2796       LLVMValueRef invmask;
2797       invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2798       mask = LLVMBuildOr(builder, mask, invmask, "");
2799    }
2800
2801    lp_build_mask_update(bld->mask, mask);
2802    if (!near_end_of_shader(bld, pc))
2803       lp_build_mask_check(bld->mask);
2804 }
2805
2806
2807 /**
2808  * Unconditional fragment kill.
2809  * The only predication is the execution mask which will apply if
2810  * we're inside a loop or conditional.
2811  */
2812 static void
2813 emit_kill(struct lp_build_tgsi_soa_context *bld,
2814           int pc)
2815 {
2816    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2817    LLVMValueRef mask;
2818
2819    /* For those channels which are "alive", disable fragment shader
2820     * execution.
2821     */
2822    if (bld->exec_mask.has_mask) {
2823       mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2824    }
2825    else {
2826       LLVMValueRef zero = LLVMConstNull(bld->bld_base.base.int_vec_type);
2827       mask = zero;
2828    }
2829
2830    lp_build_mask_update(bld->mask, mask);
2831
2832    if (!near_end_of_shader(bld, pc))
2833       lp_build_mask_check(bld->mask);
2834 }
2835
2836
2837 /**
2838  * Emit code which will dump the value of all the temporary registers
2839  * to stdout.
2840  */
2841 static void
2842 emit_dump_file(struct lp_build_tgsi_soa_context *bld,
2843                unsigned file)
2844 {
2845    const struct tgsi_shader_info *info = bld->bld_base.info;
2846    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2847    LLVMBuilderRef builder = gallivm->builder;
2848    LLVMValueRef reg_ptr;
2849    int index;
2850    int max_index = info->file_max[file];
2851
2852    /*
2853     * Some register files, particularly constants, can be very large,
2854     * and dumping everything could make this unusably slow.
2855     */
2856    max_index = MIN2(max_index, 32);
2857
2858    for (index = 0; index <= max_index; index++) {
2859       LLVMValueRef res;
2860       unsigned mask;
2861       int chan;
2862
2863       if (index < 8 * sizeof(unsigned) &&
2864           (info->file_mask[file] & (1 << index)) == 0)  {
2865          /* This was not declared.*/
2866          continue;
2867       }
2868
2869       if (file == TGSI_FILE_INPUT) {
2870          mask = info->input_usage_mask[index];
2871       } else {
2872          mask = TGSI_WRITEMASK_XYZW;
2873       }
2874
2875       for (chan = 0; chan < 4; chan++) {
2876          if ((mask & (1 << chan)) == 0) {
2877             /* This channel is not used.*/
2878             continue;
2879          }
2880
2881          if (file == TGSI_FILE_CONSTANT) {
2882             struct tgsi_full_src_register reg;
2883             memset(&reg, 0, sizeof reg);
2884             reg.Register.File = file;
2885             reg.Register.Index = index;
2886             reg.Register.SwizzleX = 0;
2887             reg.Register.SwizzleY = 1;
2888             reg.Register.SwizzleZ = 2;
2889             reg.Register.SwizzleW = 3;
2890
2891             res = bld->bld_base.emit_fetch_funcs[file](&bld->bld_base, &reg, TGSI_TYPE_FLOAT, chan);
2892             if (!res) {
2893                continue;
2894             }
2895          } else if (file == TGSI_FILE_INPUT) {
2896             res = bld->inputs[index][chan];
2897             if (!res) {
2898                continue;
2899             }
2900          } else if (file == TGSI_FILE_TEMPORARY) {
2901             reg_ptr = lp_get_temp_ptr_soa(bld, index, chan);
2902             assert(reg_ptr);
2903             res = LLVMBuildLoad(builder, reg_ptr, "");
2904          } else if (file == TGSI_FILE_OUTPUT) {
2905             reg_ptr = lp_get_output_ptr(bld, index, chan);
2906             assert(reg_ptr);
2907             res = LLVMBuildLoad(builder, reg_ptr, "");
2908          } else {
2909             assert(0);
2910             continue;
2911          }
2912
2913          emit_dump_reg(gallivm, file, index, chan, res);
2914       }
2915    }
2916 }
2917
2918
2919
2920 void
2921 lp_emit_declaration_soa(
2922    struct lp_build_tgsi_context *bld_base,
2923    const struct tgsi_full_declaration *decl)
2924 {
2925    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2926    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2927    LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
2928    const unsigned first = decl->Range.First;
2929    const unsigned last = decl->Range.Last;
2930    unsigned idx, i;
2931
2932    assert(last <= bld->bld_base.info->file_max[decl->Declaration.File]);
2933
2934    switch (decl->Declaration.File) {
2935    case TGSI_FILE_TEMPORARY:
2936       if (!(bld->indirect_files & (1 << TGSI_FILE_TEMPORARY))) {
2937          assert(last < LP_MAX_INLINED_TEMPS);
2938          for (idx = first; idx <= last; ++idx) {
2939             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2940                bld->temps[idx][i] = lp_build_alloca(gallivm, vec_type, "temp");
2941          }
2942       }
2943       break;
2944
2945    case TGSI_FILE_OUTPUT:
2946       if (!(bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
2947          for (idx = first; idx <= last; ++idx) {
2948             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2949                bld->outputs[idx][i] = lp_build_alloca(gallivm,
2950                                                       vec_type, "output");
2951          }
2952       }
2953       break;
2954
2955    case TGSI_FILE_ADDRESS:
2956       /* ADDR registers are only allocated with an integer LLVM IR type,
2957        * as they are guaranteed to always have integers.
2958        * XXX: Not sure if this exception is worthwhile (or the whole idea of
2959        * an ADDR register for that matter).
2960        */
2961       assert(last < LP_MAX_TGSI_ADDRS);
2962       for (idx = first; idx <= last; ++idx) {
2963          assert(idx < LP_MAX_TGSI_ADDRS);
2964          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2965             bld->addr[idx][i] = lp_build_alloca(gallivm, bld_base->base.int_vec_type, "addr");
2966       }
2967       break;
2968
2969    case TGSI_FILE_PREDICATE:
2970       assert(last < LP_MAX_TGSI_PREDS);
2971       for (idx = first; idx <= last; ++idx) {
2972          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2973             bld->preds[idx][i] = lp_build_alloca(gallivm, vec_type,
2974                                                  "predicate");
2975       }
2976       break;
2977
2978    case TGSI_FILE_SAMPLER_VIEW:
2979       /*
2980        * The target stored here MUST match whatever there actually
2981        * is in the set sampler views (what about return type?).
2982        */
2983       assert(last < PIPE_MAX_SHADER_SAMPLER_VIEWS);
2984       for (idx = first; idx <= last; ++idx) {
2985          bld->sv[idx] = decl->SamplerView;
2986       }
2987       break;
2988
2989    case TGSI_FILE_CONSTANT:
2990    {
2991       /*
2992        * We could trivially fetch the per-buffer pointer when fetching the
2993        * constant, relying on llvm to figure out it's always the same pointer
2994        * anyway. However, doing so results in a huge (more than factor of 10)
2995        * slowdown in llvm compilation times for some (but not all) shaders
2996        * (more specifically, the IR optimization spends way more time in
2997        * DominatorTree::dominates). At least with llvm versions 3.1, 3.3.
2998        */
2999       unsigned idx2D = decl->Dim.Index2D;
3000       LLVMValueRef index2D = lp_build_const_int32(gallivm, idx2D);
3001       assert(idx2D < LP_MAX_TGSI_CONST_BUFFERS);
3002       bld->consts[idx2D] =
3003          lp_build_array_get(gallivm, bld->consts_ptr, index2D);
3004       bld->consts_sizes[idx2D] =
3005          lp_build_array_get(gallivm, bld->const_sizes_ptr, index2D);
3006    }
3007       break;
3008
3009    default:
3010       /* don't need to declare other vars */
3011       break;
3012    }
3013 }
3014
3015
3016 void lp_emit_immediate_soa(
3017    struct lp_build_tgsi_context *bld_base,
3018    const struct tgsi_full_immediate *imm)
3019 {
3020    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
3021    struct gallivm_state * gallivm = bld_base->base.gallivm;
3022    LLVMValueRef imms[4];
3023    unsigned i;
3024    const uint size = imm->Immediate.NrTokens - 1;
3025    assert(size <= 4);
3026    switch (imm->Immediate.DataType) {
3027    case TGSI_IMM_FLOAT32:
3028       for( i = 0; i < size; ++i )
3029          imms[i] =
3030                lp_build_const_vec(gallivm, bld_base->base.type, imm->u[i].Float);
3031
3032       break;
3033    case TGSI_IMM_FLOAT64:
3034    case TGSI_IMM_UINT32:
3035       for( i = 0; i < size; ++i ) {
3036          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->uint_bld.type, imm->u[i].Uint);
3037          imms[i] = LLVMConstBitCast(tmp, bld_base->base.vec_type);
3038       }
3039
3040       break;
3041    case TGSI_IMM_INT32:
3042       for( i = 0; i < size; ++i ) {
3043          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->int_bld.type, imm->u[i].Int);
3044          imms[i] = LLVMConstBitCast(tmp, bld_base->base.vec_type);
3045       }
3046
3047       break;
3048    }
3049    for( i = size; i < 4; ++i )
3050       imms[i] = bld_base->base.undef;
3051
3052    if (bld->use_immediates_array) {
3053       unsigned index = bld->num_immediates;
3054       struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
3055       LLVMBuilderRef builder = gallivm->builder;
3056
3057       assert(bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE));
3058       for (i = 0; i < 4; ++i ) {
3059          LLVMValueRef lindex = lp_build_const_int32(
3060                   bld->bld_base.base.gallivm, index * 4 + i);
3061          LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
3062                                              bld->imms_array, &lindex, 1, "");
3063          LLVMBuildStore(builder, imms[i], imm_ptr);
3064       }
3065    } else {
3066       /* simply copy the immediate values into the next immediates[] slot */
3067       unsigned i;
3068       assert(imm->Immediate.NrTokens - 1 <= 4);
3069       assert(bld->num_immediates < LP_MAX_INLINED_IMMEDIATES);
3070
3071       for(i = 0; i < 4; ++i )
3072          bld->immediates[bld->num_immediates][i] = imms[i];
3073
3074       if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
3075          unsigned index = bld->num_immediates;
3076          struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
3077          LLVMBuilderRef builder = gallivm->builder;
3078          for (i = 0; i < 4; ++i ) {
3079             LLVMValueRef lindex = lp_build_const_int32(
3080                      bld->bld_base.base.gallivm, index * 4 + i);
3081             LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
3082                                                 bld->imms_array, &lindex, 1, "");
3083             LLVMBuildStore(builder,
3084                            bld->immediates[index][i],
3085                            imm_ptr);
3086          }
3087       }
3088    }
3089
3090    bld->num_immediates++;
3091 }
3092
3093 static void
3094 ddx_emit(
3095    const struct lp_build_tgsi_action * action,
3096    struct lp_build_tgsi_context * bld_base,
3097    struct lp_build_emit_data * emit_data)
3098 {
3099    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3100
3101    emit_fetch_deriv(bld, emit_data->args[0], NULL,
3102                     &emit_data->output[emit_data->chan], NULL);
3103 }
3104
3105 static void
3106 ddy_emit(
3107    const struct lp_build_tgsi_action * action,
3108    struct lp_build_tgsi_context * bld_base,
3109    struct lp_build_emit_data * emit_data)
3110 {
3111    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3112
3113    emit_fetch_deriv(bld, emit_data->args[0], NULL, NULL,
3114                     &emit_data->output[emit_data->chan]);
3115 }
3116
3117 static void
3118 kill_emit(
3119    const struct lp_build_tgsi_action * action,
3120    struct lp_build_tgsi_context * bld_base,
3121    struct lp_build_emit_data * emit_data)
3122 {
3123    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3124
3125    emit_kill(bld, bld_base->pc - 1);
3126 }
3127
3128 static void
3129 kill_if_emit(
3130    const struct lp_build_tgsi_action * action,
3131    struct lp_build_tgsi_context * bld_base,
3132    struct lp_build_emit_data * emit_data)
3133 {
3134    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3135
3136    emit_kill_if(bld, emit_data->inst, bld_base->pc - 1);
3137 }
3138
3139 static void
3140 tex_emit(
3141    const struct lp_build_tgsi_action * action,
3142    struct lp_build_tgsi_context * bld_base,
3143    struct lp_build_emit_data * emit_data)
3144 {
3145    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3146
3147    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3148             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3149 }
3150
3151 static void
3152 tex2_emit(
3153    const struct lp_build_tgsi_action * action,
3154    struct lp_build_tgsi_context * bld_base,
3155    struct lp_build_emit_data * emit_data)
3156 {
3157    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3158
3159    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3160             emit_data->output, 2, LP_SAMPLER_OP_TEXTURE);
3161 }
3162
3163 static void
3164 txb_emit(
3165    const struct lp_build_tgsi_action * action,
3166    struct lp_build_tgsi_context * bld_base,
3167    struct lp_build_emit_data * emit_data)
3168 {
3169    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3170
3171    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
3172             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3173 }
3174
3175 static void
3176 txb2_emit(
3177    const struct lp_build_tgsi_action * action,
3178    struct lp_build_tgsi_context * bld_base,
3179    struct lp_build_emit_data * emit_data)
3180 {
3181    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3182
3183    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
3184             emit_data->output, 2, LP_SAMPLER_OP_TEXTURE);
3185 }
3186
3187 static void
3188 txd_emit(
3189    const struct lp_build_tgsi_action * action,
3190    struct lp_build_tgsi_context * bld_base,
3191    struct lp_build_emit_data * emit_data)
3192 {
3193    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3194
3195    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
3196             emit_data->output, 3, LP_SAMPLER_OP_TEXTURE);
3197 }
3198
3199 static void
3200 txl_emit(
3201    const struct lp_build_tgsi_action * action,
3202    struct lp_build_tgsi_context * bld_base,
3203    struct lp_build_emit_data * emit_data)
3204 {
3205    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3206
3207    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3208             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3209 }
3210
3211 static void
3212 txl2_emit(
3213    const struct lp_build_tgsi_action * action,
3214    struct lp_build_tgsi_context * bld_base,
3215    struct lp_build_emit_data * emit_data)
3216 {
3217    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3218
3219    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3220             emit_data->output, 2, LP_SAMPLER_OP_TEXTURE);
3221 }
3222
3223 static void
3224 txp_emit(
3225    const struct lp_build_tgsi_action * action,
3226    struct lp_build_tgsi_context * bld_base,
3227    struct lp_build_emit_data * emit_data)
3228 {
3229    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3230
3231    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_PROJECTED,
3232             emit_data->output, 1, LP_SAMPLER_OP_TEXTURE);
3233 }
3234
3235 static void
3236 tg4_emit(
3237    const struct lp_build_tgsi_action * action,
3238    struct lp_build_tgsi_context * bld_base,
3239    struct lp_build_emit_data * emit_data)
3240 {
3241    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3242
3243    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3244             emit_data->output, 2, LP_SAMPLER_OP_GATHER);
3245 }
3246
3247 static void
3248 txq_emit(
3249    const struct lp_build_tgsi_action * action,
3250    struct lp_build_tgsi_context * bld_base,
3251    struct lp_build_emit_data * emit_data)
3252 {
3253    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3254
3255    emit_size_query(bld, emit_data->inst, emit_data->output, FALSE);
3256 }
3257
3258 static void
3259 txf_emit(
3260    const struct lp_build_tgsi_action * action,
3261    struct lp_build_tgsi_context * bld_base,
3262    struct lp_build_emit_data * emit_data)
3263 {
3264    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3265
3266    emit_fetch_texels(bld, emit_data->inst, emit_data->output, FALSE);
3267 }
3268
3269 static void
3270 sample_i_emit(
3271    const struct lp_build_tgsi_action * action,
3272    struct lp_build_tgsi_context * bld_base,
3273    struct lp_build_emit_data * emit_data)
3274 {
3275    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3276
3277    emit_fetch_texels(bld, emit_data->inst, emit_data->output, TRUE);
3278 }
3279
3280 static void
3281 sample_emit(
3282    const struct lp_build_tgsi_action * action,
3283    struct lp_build_tgsi_context * bld_base,
3284    struct lp_build_emit_data * emit_data)
3285 {
3286    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3287
3288    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3289                FALSE, emit_data->output);
3290 }
3291
3292 static void
3293 sample_b_emit(
3294    const struct lp_build_tgsi_action * action,
3295    struct lp_build_tgsi_context * bld_base,
3296    struct lp_build_emit_data * emit_data)
3297 {
3298    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3299
3300    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
3301                FALSE, emit_data->output);
3302 }
3303
3304 static void
3305 sample_c_emit(
3306    const struct lp_build_tgsi_action * action,
3307    struct lp_build_tgsi_context * bld_base,
3308    struct lp_build_emit_data * emit_data)
3309 {
3310    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3311
3312    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3313                TRUE, emit_data->output);
3314 }
3315
3316 static void
3317 sample_c_lz_emit(
3318    const struct lp_build_tgsi_action * action,
3319    struct lp_build_tgsi_context * bld_base,
3320    struct lp_build_emit_data * emit_data)
3321 {
3322    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3323
3324    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_ZERO,
3325                TRUE, emit_data->output);
3326 }
3327
3328 static void
3329 sample_d_emit(
3330    const struct lp_build_tgsi_action * action,
3331    struct lp_build_tgsi_context * bld_base,
3332    struct lp_build_emit_data * emit_data)
3333 {
3334    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3335
3336    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
3337                FALSE, emit_data->output);
3338 }
3339
3340 static void
3341 sample_l_emit(
3342    const struct lp_build_tgsi_action * action,
3343    struct lp_build_tgsi_context * bld_base,
3344    struct lp_build_emit_data * emit_data)
3345 {
3346    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3347
3348    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3349                FALSE, emit_data->output);
3350 }
3351
3352 static void
3353 sviewinfo_emit(
3354    const struct lp_build_tgsi_action * action,
3355    struct lp_build_tgsi_context * bld_base,
3356    struct lp_build_emit_data * emit_data)
3357 {
3358    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3359
3360    emit_size_query(bld, emit_data->inst, emit_data->output, TRUE);
3361 }
3362
3363 static LLVMValueRef
3364 mask_vec(struct lp_build_tgsi_context *bld_base)
3365 {
3366    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3367    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3368    struct lp_exec_mask *exec_mask = &bld->exec_mask;
3369
3370    if (!exec_mask->has_mask) {
3371       return lp_build_mask_value(bld->mask);
3372    }
3373    return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
3374                        exec_mask->exec_mask, "");
3375 }
3376
3377 static void
3378 increment_vec_ptr_by_mask(struct lp_build_tgsi_context * bld_base,
3379                           LLVMValueRef ptr,
3380                           LLVMValueRef mask)
3381 {
3382    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3383    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3384
3385    current_vec = LLVMBuildSub(builder, current_vec, mask, "");
3386
3387    LLVMBuildStore(builder, current_vec, ptr);
3388 }
3389
3390 static void
3391 clear_uint_vec_ptr_from_mask(struct lp_build_tgsi_context * bld_base,
3392                              LLVMValueRef ptr,
3393                              LLVMValueRef mask)
3394 {
3395    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3396    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3397
3398    current_vec = lp_build_select(&bld_base->uint_bld,
3399                                  mask,
3400                                  bld_base->uint_bld.zero,
3401                                  current_vec);
3402
3403    LLVMBuildStore(builder, current_vec, ptr);
3404 }
3405
3406 static LLVMValueRef
3407 clamp_mask_to_max_output_vertices(struct lp_build_tgsi_soa_context * bld,
3408                                   LLVMValueRef current_mask_vec,
3409                                   LLVMValueRef total_emitted_vertices_vec)
3410 {
3411    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3412    struct lp_build_context *int_bld = &bld->bld_base.int_bld;
3413    LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
3414                                         total_emitted_vertices_vec,
3415                                         bld->max_output_vertices_vec);
3416
3417    return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
3418 }
3419
3420 static void
3421 emit_vertex(
3422    const struct lp_build_tgsi_action * action,
3423    struct lp_build_tgsi_context * bld_base,
3424    struct lp_build_emit_data * emit_data)
3425 {
3426    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3427    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3428
3429    if (bld->gs_iface->emit_vertex) {
3430       LLVMValueRef mask = mask_vec(bld_base);
3431       LLVMValueRef total_emitted_vertices_vec =
3432          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3433       mask = clamp_mask_to_max_output_vertices(bld, mask,
3434                                                total_emitted_vertices_vec);
3435       gather_outputs(bld);
3436       bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base,
3437                                  bld->outputs,
3438                                  total_emitted_vertices_vec);
3439       increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr,
3440                                 mask);
3441       increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr,
3442                                 mask);
3443 #if DUMP_GS_EMITS
3444       lp_build_print_value(bld->bld_base.base.gallivm,
3445                            " +++ emit vertex masked ones = ",
3446                            mask);
3447       lp_build_print_value(bld->bld_base.base.gallivm,
3448                            " +++ emit vertex emitted = ",
3449                            total_emitted_vertices_vec);
3450 #endif
3451    }
3452 }
3453
3454
3455 static void
3456 end_primitive_masked(struct lp_build_tgsi_context * bld_base,
3457                      LLVMValueRef mask)
3458 {
3459    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3460    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3461
3462    if (bld->gs_iface->end_primitive) {
3463       struct lp_build_context *uint_bld = &bld_base->uint_bld;
3464       LLVMValueRef emitted_vertices_vec =
3465          LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr, "");
3466       LLVMValueRef emitted_prims_vec =
3467          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3468
3469       LLVMValueRef emitted_mask = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3470                                                emitted_vertices_vec,
3471                                                uint_bld->zero);
3472       /* We need to combine the current execution mask with the mask
3473          telling us which, if any, execution slots actually have
3474          unemitted primitives, this way we make sure that end_primitives
3475          executes only on the paths that have unflushed vertices */
3476       mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
3477
3478       bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base,
3479                                    emitted_vertices_vec,
3480                                    emitted_prims_vec);
3481
3482 #if DUMP_GS_EMITS
3483       lp_build_print_value(bld->bld_base.base.gallivm,
3484                            " +++ end prim masked ones = ",
3485                            mask);
3486       lp_build_print_value(bld->bld_base.base.gallivm,
3487                            " +++ end prim emitted verts1 = ",
3488                            emitted_vertices_vec);
3489       lp_build_print_value(bld->bld_base.base.gallivm,
3490                            " +++ end prim emitted prims1 = ",
3491                            LLVMBuildLoad(builder,
3492                                          bld->emitted_prims_vec_ptr, ""));
3493 #endif
3494       increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr,
3495                                 mask);
3496       clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr,
3497                                    mask);
3498 #if DUMP_GS_EMITS
3499       lp_build_print_value(bld->bld_base.base.gallivm,
3500                            " +++ end prim emitted verts2 = ",
3501                            LLVMBuildLoad(builder,
3502                                          bld->emitted_vertices_vec_ptr, ""));
3503 #endif
3504    }
3505
3506 }
3507
3508 static void
3509 end_primitive(
3510    const struct lp_build_tgsi_action * action,
3511    struct lp_build_tgsi_context * bld_base,
3512    struct lp_build_emit_data * emit_data)
3513 {
3514    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3515
3516    if (bld->gs_iface->end_primitive) {
3517       LLVMValueRef mask = mask_vec(bld_base);
3518       end_primitive_masked(bld_base, mask);
3519    }
3520 }
3521
3522 static void
3523 cal_emit(
3524    const struct lp_build_tgsi_action * action,
3525    struct lp_build_tgsi_context * bld_base,
3526    struct lp_build_emit_data * emit_data)
3527 {
3528    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3529
3530    lp_exec_mask_call(&bld->exec_mask, emit_data->inst->Label.Label,
3531                      &bld_base->pc);
3532 }
3533
3534 static void
3535 ret_emit(
3536    const struct lp_build_tgsi_action * action,
3537    struct lp_build_tgsi_context * bld_base,
3538    struct lp_build_emit_data * emit_data)
3539 {
3540    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3541
3542    lp_exec_mask_ret(&bld->exec_mask, &bld_base->pc);
3543 }
3544
3545 static void
3546 brk_emit(
3547    const struct lp_build_tgsi_action * action,
3548    struct lp_build_tgsi_context * bld_base,
3549    struct lp_build_emit_data * emit_data)
3550 {
3551    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3552
3553    lp_exec_break(&bld->exec_mask, bld_base);
3554 }
3555
3556 static void
3557 breakc_emit(
3558    const struct lp_build_tgsi_action * action,
3559    struct lp_build_tgsi_context * bld_base,
3560    struct lp_build_emit_data * emit_data)
3561 {
3562    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3563    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3564    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3565    LLVMValueRef unsigned_cond =
3566       LLVMBuildBitCast(builder, emit_data->args[0], uint_bld->vec_type, "");
3567    LLVMValueRef cond = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3568                                     unsigned_cond,
3569                                     uint_bld->zero);
3570
3571    lp_exec_break_condition(&bld->exec_mask, cond);
3572 }
3573
3574 static void
3575 if_emit(
3576    const struct lp_build_tgsi_action * action,
3577    struct lp_build_tgsi_context * bld_base,
3578    struct lp_build_emit_data * emit_data)
3579 {
3580    LLVMValueRef tmp;
3581    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3582
3583    tmp = lp_build_cmp(&bld_base->base, PIPE_FUNC_NOTEQUAL,
3584                       emit_data->args[0], bld->bld_base.base.zero);
3585    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3586 }
3587
3588 static void
3589 uif_emit(
3590    const struct lp_build_tgsi_action * action,
3591    struct lp_build_tgsi_context * bld_base,
3592    struct lp_build_emit_data * emit_data)
3593 {
3594    LLVMValueRef tmp;
3595    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3596    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3597
3598    tmp = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3599                       emit_data->args[0], uint_bld->zero);
3600    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3601 }
3602
3603 static void
3604 case_emit(
3605    const struct lp_build_tgsi_action * action,
3606    struct lp_build_tgsi_context * bld_base,
3607    struct lp_build_emit_data * emit_data)
3608 {
3609    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3610
3611    lp_exec_case(&bld->exec_mask, emit_data->args[0]);
3612 }
3613
3614 static void
3615 default_emit(
3616    const struct lp_build_tgsi_action * action,
3617    struct lp_build_tgsi_context * bld_base,
3618    struct lp_build_emit_data * emit_data)
3619 {
3620    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3621
3622    lp_exec_default(&bld->exec_mask, bld_base);
3623 }
3624
3625 static void
3626 switch_emit(
3627    const struct lp_build_tgsi_action * action,
3628    struct lp_build_tgsi_context * bld_base,
3629    struct lp_build_emit_data * emit_data)
3630 {
3631    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3632
3633    lp_exec_switch(&bld->exec_mask, emit_data->args[0]);
3634 }
3635
3636 static void
3637 endswitch_emit(
3638    const struct lp_build_tgsi_action * action,
3639    struct lp_build_tgsi_context * bld_base,
3640    struct lp_build_emit_data * emit_data)
3641 {
3642    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3643
3644    lp_exec_endswitch(&bld->exec_mask, bld_base);
3645 }
3646
3647 static void
3648 bgnloop_emit(
3649    const struct lp_build_tgsi_action * action,
3650    struct lp_build_tgsi_context * bld_base,
3651    struct lp_build_emit_data * emit_data)
3652 {
3653    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3654
3655    lp_exec_bgnloop(&bld->exec_mask);
3656 }
3657
3658 static void
3659 bgnsub_emit(
3660    const struct lp_build_tgsi_action * action,
3661    struct lp_build_tgsi_context * bld_base,
3662    struct lp_build_emit_data * emit_data)
3663 {
3664    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3665
3666    lp_exec_mask_bgnsub(&bld->exec_mask);
3667 }
3668
3669 static void
3670 else_emit(
3671    const struct lp_build_tgsi_action * action,
3672    struct lp_build_tgsi_context * bld_base,
3673    struct lp_build_emit_data * emit_data)
3674 {
3675    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3676
3677    lp_exec_mask_cond_invert(&bld->exec_mask);
3678 }
3679
3680 static void
3681 endif_emit(
3682    const struct lp_build_tgsi_action * action,
3683    struct lp_build_tgsi_context * bld_base,
3684    struct lp_build_emit_data * emit_data)
3685 {
3686    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3687
3688    lp_exec_mask_cond_pop(&bld->exec_mask);
3689 }
3690
3691 static void
3692 endloop_emit(
3693    const struct lp_build_tgsi_action * action,
3694    struct lp_build_tgsi_context * bld_base,
3695    struct lp_build_emit_data * emit_data)
3696 {
3697    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3698
3699    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
3700 }
3701
3702 static void
3703 endsub_emit(
3704    const struct lp_build_tgsi_action * action,
3705    struct lp_build_tgsi_context * bld_base,
3706    struct lp_build_emit_data * emit_data)
3707 {
3708    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3709
3710    lp_exec_mask_endsub(&bld->exec_mask, &bld_base->pc);
3711 }
3712
3713 static void
3714 cont_emit(
3715    const struct lp_build_tgsi_action * action,
3716    struct lp_build_tgsi_context * bld_base,
3717    struct lp_build_emit_data * emit_data)
3718 {
3719    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3720
3721    lp_exec_continue(&bld->exec_mask);
3722 }
3723
3724 static void emit_prologue(struct lp_build_tgsi_context * bld_base)
3725 {
3726    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3727    struct gallivm_state * gallivm = bld_base->base.gallivm;
3728
3729    if (bld->indirect_files & (1 << TGSI_FILE_TEMPORARY)) {
3730       LLVMValueRef array_size =
3731          lp_build_const_int32(gallivm,
3732                          bld_base->info->file_max[TGSI_FILE_TEMPORARY] * 4 + 4);
3733       bld->temps_array = lp_build_array_alloca(gallivm,
3734                                               bld_base->base.vec_type, array_size,
3735                                               "temp_array");
3736    }
3737
3738    if (bld->indirect_files & (1 << TGSI_FILE_OUTPUT)) {
3739       LLVMValueRef array_size =
3740          lp_build_const_int32(gallivm,
3741                             bld_base->info->file_max[TGSI_FILE_OUTPUT] * 4 + 4);
3742       bld->outputs_array = lp_build_array_alloca(gallivm,
3743                                                 bld_base->base.vec_type, array_size,
3744                                                 "output_array");
3745    }
3746
3747    if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
3748       LLVMValueRef array_size =
3749          lp_build_const_int32(gallivm,
3750                          bld_base->info->file_max[TGSI_FILE_IMMEDIATE] * 4 + 4);
3751       bld->imms_array = lp_build_array_alloca(gallivm,
3752                                               bld_base->base.vec_type, array_size,
3753                                               "imms_array");
3754    }
3755
3756    /* If we have indirect addressing in inputs we need to copy them into
3757     * our alloca array to be able to iterate over them */
3758    if (bld->indirect_files & (1 << TGSI_FILE_INPUT) && !bld->gs_iface) {
3759       unsigned index, chan;
3760       LLVMTypeRef vec_type = bld_base->base.vec_type;
3761       LLVMValueRef array_size = lp_build_const_int32(gallivm,
3762             bld_base->info->file_max[TGSI_FILE_INPUT]*4 + 4);
3763       bld->inputs_array = lp_build_array_alloca(gallivm,
3764                                                vec_type, array_size,
3765                                                "input_array");
3766
3767       assert(bld_base->info->num_inputs
3768                         <= bld_base->info->file_max[TGSI_FILE_INPUT] + 1);
3769
3770       for (index = 0; index < bld_base->info->num_inputs; ++index) {
3771          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3772             LLVMValueRef lindex =
3773                lp_build_const_int32(gallivm, index * 4 + chan);
3774             LLVMValueRef input_ptr =
3775                LLVMBuildGEP(gallivm->builder, bld->inputs_array,
3776                             &lindex, 1, "");
3777             LLVMValueRef value = bld->inputs[index][chan];
3778             if (value)
3779                LLVMBuildStore(gallivm->builder, value, input_ptr);
3780          }
3781       }
3782    }
3783
3784    if (bld->gs_iface) {
3785       struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
3786       bld->emitted_prims_vec_ptr =
3787          lp_build_alloca(gallivm,
3788                          uint_bld->vec_type,
3789                          "emitted_prims_ptr");
3790       bld->emitted_vertices_vec_ptr =
3791          lp_build_alloca(gallivm,
3792                          uint_bld->vec_type,
3793                          "emitted_vertices_ptr");
3794       bld->total_emitted_vertices_vec_ptr =
3795          lp_build_alloca(gallivm,
3796                          uint_bld->vec_type,
3797                          "total_emitted_vertices_ptr");
3798
3799       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3800                      bld->emitted_prims_vec_ptr);
3801       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3802                      bld->emitted_vertices_vec_ptr);
3803       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3804                      bld->total_emitted_vertices_vec_ptr);
3805    }
3806
3807    if (DEBUG_EXECUTION) {
3808       lp_build_printf(gallivm, "\n");
3809       emit_dump_file(bld, TGSI_FILE_CONSTANT);
3810       if (!bld->gs_iface)
3811          emit_dump_file(bld, TGSI_FILE_INPUT);
3812    }
3813 }
3814
3815 static void emit_epilogue(struct lp_build_tgsi_context * bld_base)
3816 {
3817    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3818    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3819
3820    if (DEBUG_EXECUTION) {
3821       /* for debugging */
3822       if (0) {
3823          emit_dump_file(bld, TGSI_FILE_TEMPORARY);
3824       }
3825       emit_dump_file(bld, TGSI_FILE_OUTPUT);
3826       lp_build_printf(bld_base->base.gallivm, "\n");
3827    }
3828
3829    /* If we have indirect addressing in outputs we need to copy our alloca array
3830     * to the outputs slots specified by the caller */
3831    if (bld->gs_iface) {
3832       LLVMValueRef total_emitted_vertices_vec;
3833       LLVMValueRef emitted_prims_vec;
3834       /* implicit end_primitives, needed in case there are any unflushed
3835          vertices in the cache. Note must not call end_primitive here
3836          since the exec_mask is not valid at this point. */
3837       end_primitive_masked(bld_base, lp_build_mask_value(bld->mask));
3838
3839       total_emitted_vertices_vec =
3840          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3841       emitted_prims_vec =
3842          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3843
3844       bld->gs_iface->gs_epilogue(bld->gs_iface,
3845                                  &bld->bld_base,
3846                                  total_emitted_vertices_vec,
3847                                  emitted_prims_vec);
3848    } else {
3849       gather_outputs(bld);
3850    }
3851 }
3852
3853 void
3854 lp_build_tgsi_soa(struct gallivm_state *gallivm,
3855                   const struct tgsi_token *tokens,
3856                   struct lp_type type,
3857                   struct lp_build_mask_context *mask,
3858                   LLVMValueRef consts_ptr,
3859                   LLVMValueRef const_sizes_ptr,
3860                   const struct lp_bld_tgsi_system_values *system_values,
3861                   const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
3862                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
3863                   LLVMValueRef context_ptr,
3864                   LLVMValueRef thread_data_ptr,
3865                   struct lp_build_sampler_soa *sampler,
3866                   const struct tgsi_shader_info *info,
3867                   const struct lp_build_tgsi_gs_iface *gs_iface)
3868 {
3869    struct lp_build_tgsi_soa_context bld;
3870
3871    struct lp_type res_type;
3872
3873    assert(type.length <= LP_MAX_VECTOR_LENGTH);
3874    memset(&res_type, 0, sizeof res_type);
3875    res_type.width = type.width;
3876    res_type.length = type.length;
3877    res_type.sign = 1;
3878
3879    /* Setup build context */
3880    memset(&bld, 0, sizeof bld);
3881    lp_build_context_init(&bld.bld_base.base, gallivm, type);
3882    lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
3883    lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
3884    lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
3885    {
3886       struct lp_type dbl_type;
3887       dbl_type = type;
3888       dbl_type.width *= 2;
3889       lp_build_context_init(&bld.bld_base.dbl_bld, gallivm, dbl_type);
3890    }
3891    bld.mask = mask;
3892    bld.inputs = inputs;
3893    bld.outputs = outputs;
3894    bld.consts_ptr = consts_ptr;
3895    bld.const_sizes_ptr = const_sizes_ptr;
3896    bld.sampler = sampler;
3897    bld.bld_base.info = info;
3898    bld.indirect_files = info->indirect_files;
3899    bld.context_ptr = context_ptr;
3900    bld.thread_data_ptr = thread_data_ptr;
3901
3902    /*
3903     * If the number of temporaries is rather large then we just
3904     * allocate them as an array right from the start and treat
3905     * like indirect temporaries.
3906     */
3907    if (info->file_max[TGSI_FILE_TEMPORARY] >= LP_MAX_INLINED_TEMPS) {
3908       bld.indirect_files |= (1 << TGSI_FILE_TEMPORARY);
3909    }
3910    /*
3911     * For performance reason immediates are always backed in a static
3912     * array, but if their number is too great, we have to use just
3913     * a dynamically allocated array.
3914     */
3915    bld.use_immediates_array =
3916          (info->file_max[TGSI_FILE_IMMEDIATE] >= LP_MAX_INLINED_IMMEDIATES);
3917    if (bld.use_immediates_array) {
3918       bld.indirect_files |= (1 << TGSI_FILE_IMMEDIATE);
3919    }
3920
3921
3922    bld.bld_base.soa = TRUE;
3923    bld.bld_base.emit_debug = emit_debug;
3924    bld.bld_base.emit_fetch_funcs[TGSI_FILE_CONSTANT] = emit_fetch_constant;
3925    bld.bld_base.emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = emit_fetch_immediate;
3926    bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_input;
3927    bld.bld_base.emit_fetch_funcs[TGSI_FILE_TEMPORARY] = emit_fetch_temporary;
3928    bld.bld_base.emit_fetch_funcs[TGSI_FILE_SYSTEM_VALUE] = emit_fetch_system_value;
3929    bld.bld_base.emit_store = emit_store;
3930
3931    bld.bld_base.emit_declaration = lp_emit_declaration_soa;
3932    bld.bld_base.emit_immediate = lp_emit_immediate_soa;
3933
3934    bld.bld_base.emit_prologue = emit_prologue;
3935    bld.bld_base.emit_epilogue = emit_epilogue;
3936
3937    /* Set opcode actions */
3938    lp_set_default_actions_cpu(&bld.bld_base);
3939
3940    bld.bld_base.op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit;
3941    bld.bld_base.op_actions[TGSI_OPCODE_BGNSUB].emit = bgnsub_emit;
3942    bld.bld_base.op_actions[TGSI_OPCODE_BRK].emit = brk_emit;
3943    bld.bld_base.op_actions[TGSI_OPCODE_BREAKC].emit = breakc_emit;
3944    bld.bld_base.op_actions[TGSI_OPCODE_CAL].emit = cal_emit;
3945    bld.bld_base.op_actions[TGSI_OPCODE_CASE].emit = case_emit;
3946    bld.bld_base.op_actions[TGSI_OPCODE_CONT].emit = cont_emit;
3947    bld.bld_base.op_actions[TGSI_OPCODE_DDX].emit = ddx_emit;
3948    bld.bld_base.op_actions[TGSI_OPCODE_DDY].emit = ddy_emit;
3949    bld.bld_base.op_actions[TGSI_OPCODE_DEFAULT].emit = default_emit;
3950    bld.bld_base.op_actions[TGSI_OPCODE_ELSE].emit = else_emit;
3951    bld.bld_base.op_actions[TGSI_OPCODE_ENDIF].emit = endif_emit;
3952    bld.bld_base.op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit;
3953    bld.bld_base.op_actions[TGSI_OPCODE_ENDSUB].emit = endsub_emit;
3954    bld.bld_base.op_actions[TGSI_OPCODE_ENDSWITCH].emit = endswitch_emit;
3955    bld.bld_base.op_actions[TGSI_OPCODE_IF].emit = if_emit;
3956    bld.bld_base.op_actions[TGSI_OPCODE_UIF].emit = uif_emit;
3957    bld.bld_base.op_actions[TGSI_OPCODE_KILL_IF].emit = kill_if_emit;
3958    bld.bld_base.op_actions[TGSI_OPCODE_KILL].emit = kill_emit;
3959    bld.bld_base.op_actions[TGSI_OPCODE_RET].emit = ret_emit;
3960    bld.bld_base.op_actions[TGSI_OPCODE_SWITCH].emit = switch_emit;
3961    bld.bld_base.op_actions[TGSI_OPCODE_TEX].emit = tex_emit;
3962    bld.bld_base.op_actions[TGSI_OPCODE_TXB].emit = txb_emit;
3963    bld.bld_base.op_actions[TGSI_OPCODE_TXD].emit = txd_emit;
3964    bld.bld_base.op_actions[TGSI_OPCODE_TXL].emit = txl_emit;
3965    bld.bld_base.op_actions[TGSI_OPCODE_TXP].emit = txp_emit;
3966    bld.bld_base.op_actions[TGSI_OPCODE_TXQ].emit = txq_emit;
3967    bld.bld_base.op_actions[TGSI_OPCODE_TXF].emit = txf_emit;
3968    bld.bld_base.op_actions[TGSI_OPCODE_TEX2].emit = tex2_emit;
3969    bld.bld_base.op_actions[TGSI_OPCODE_TXB2].emit = txb2_emit;
3970    bld.bld_base.op_actions[TGSI_OPCODE_TXL2].emit = txl2_emit;
3971    bld.bld_base.op_actions[TGSI_OPCODE_TG4].emit = tg4_emit;
3972    /* DX10 sampling ops */
3973    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE].emit = sample_emit;
3974    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_B].emit = sample_b_emit;
3975    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C].emit = sample_c_emit;
3976    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C_LZ].emit = sample_c_lz_emit;
3977    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_D].emit = sample_d_emit;
3978    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_I].emit = sample_i_emit;
3979    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_L].emit = sample_l_emit;
3980    bld.bld_base.op_actions[TGSI_OPCODE_SVIEWINFO].emit = sviewinfo_emit;
3981
3982    if (gs_iface) {
3983       /* There's no specific value for this because it should always
3984        * be set, but apps using ext_geometry_shader4 quite often
3985        * were forgetting so we're using MAX_VERTEX_VARYING from
3986        * that spec even though we could debug_assert if it's not
3987        * set, but that's a lot uglier. */
3988       uint max_output_vertices;
3989
3990       /* inputs are always indirect with gs */
3991       bld.indirect_files |= (1 << TGSI_FILE_INPUT);
3992       bld.gs_iface = gs_iface;
3993       bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_gs_input;
3994       bld.bld_base.op_actions[TGSI_OPCODE_EMIT].emit = emit_vertex;
3995       bld.bld_base.op_actions[TGSI_OPCODE_ENDPRIM].emit = end_primitive;
3996
3997       max_output_vertices =
3998             info->properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
3999       if (!max_output_vertices)
4000          max_output_vertices = 32;
4001
4002       bld.max_output_vertices_vec =
4003          lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
4004                                 max_output_vertices);
4005    }
4006
4007    lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
4008
4009    bld.system_values = *system_values;
4010
4011    lp_build_tgsi_llvm(&bld.bld_base, tokens);
4012
4013    if (0) {
4014       LLVMBasicBlockRef block = LLVMGetInsertBlock(gallivm->builder);
4015       LLVMValueRef function = LLVMGetBasicBlockParent(block);
4016       debug_printf("11111111111111111111111111111 \n");
4017       tgsi_dump(tokens, 0);
4018       lp_debug_dump_value(function);
4019       debug_printf("2222222222222222222222222222 \n");
4020    }
4021
4022    if (0) {
4023       LLVMModuleRef module = LLVMGetGlobalParent(
4024          LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
4025       LLVMDumpModule(module);
4026
4027    }
4028    lp_exec_mask_fini(&bld.exec_mask);
4029 }