src/gallium/auxiliary/gallivm/lp_bld_tgsi_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * Copyright 2007-2008 VMware, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28
  29 /**
  30  * @file
  31  * TGSI to LLVM IR translation -- SoA.
  32  *
  33  * @author Jose Fonseca <jfonseca@vmware.com>
  34  *
  35  * Based on tgsi_sse2.c code written by Michal Krol, Keith Whitwell,
  36  * Brian Paul, and others.
  37  */
  38
  39 #include "pipe/p_config.h"
  40 #include "pipe/p_shader_tokens.h"
  41 #include "util/u_debug.h"
  42 #include "util/u_math.h"
  43 #include "util/u_memory.h"
  44 #include "tgsi/tgsi_dump.h"
  45 #include "tgsi/tgsi_exec.h"
  46 #include "tgsi/tgsi_info.h"
  47 #include "tgsi/tgsi_parse.h"
  48 #include "tgsi/tgsi_util.h"
  49 #include "tgsi/tgsi_scan.h"
  50 #include "tgsi/tgsi_strings.h"
  51 #include "lp_bld_tgsi_action.h"
  52 #include "lp_bld_type.h"
  53 #include "lp_bld_const.h"
  54 #include "lp_bld_arit.h"
  55 #include "lp_bld_bitarit.h"
  56 #include "lp_bld_gather.h"
  57 #include "lp_bld_init.h"
  58 #include "lp_bld_logic.h"
  59 #include "lp_bld_swizzle.h"
  60 #include "lp_bld_flow.h"
  61 #include "lp_bld_quad.h"
  62 #include "lp_bld_tgsi.h"
  63 #include "lp_bld_limits.h"
  64 #include "lp_bld_debug.h"
  65 #include "lp_bld_printf.h"
  66 #include "lp_bld_sample.h"
  67 #include "lp_bld_struct.h"
  68
  69 /* SM 4.0 says that subroutines can nest 32 deep and
  70  * we need one more for our main function */
  71 #define LP_MAX_NUM_FUNCS 33
  72
  73 #define DUMP_GS_EMITS 0
  74
  75 /*
  76  * If non-zero, the generated LLVM IR will print intermediate results on every TGSI
  77  * instruction.
  78  *
  79  * TODO:
  80  * - take execution masks in consideration
  81  * - debug control-flow instructions
  82  */
  83 #define DEBUG_EXECUTION 0
  84
  85
  86 /*
  87  * Emit code to print a register value.
  88  */
  89 static void
  90 emit_dump_reg(struct gallivm_state *gallivm,
  91               unsigned file,
  92               unsigned index,
  93               unsigned chan,
  94               LLVMValueRef value)
  95 {
  96    char buf[32];
  97
  98    util_snprintf(buf, sizeof buf, "    %s[%u].%c = ",
  99                  tgsi_file_name(file),
 100                  index, "xyzw"[chan]);
 101
 102    lp_build_print_value(gallivm, buf, value);
 103 }
 104
 105 /*
 106  * Return the context for the current function.
 107  * (always 'main', if shader doesn't do any function calls)
 108  */
 109 static INLINE struct function_ctx *
 110 func_ctx(struct lp_exec_mask *mask)
 111 {
 112    assert(mask->function_stack_size > 0);
 113    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 114    return &mask->function_stack[mask->function_stack_size - 1];
 115 }
 116
 117 /*
 118  * Returns true if we're in a loop.
 119  * It's global, meaning that it returns true even if there's
 120  * no loop inside the current function, but we were inside
 121  * a loop inside another function, from which this one was called.
 122  */
 123 static INLINE boolean
 124 mask_has_loop(struct lp_exec_mask *mask)
 125 {
 126    int i;
 127    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 128       const struct function_ctx *ctx = &mask->function_stack[i];
 129       if (ctx->loop_stack_size > 0)
 130          return TRUE;
 131    }
 132    return FALSE;
 133 }
 134
 135 /*
 136  * Returns true if we're inside a switch statement.
 137  * It's global, meaning that it returns true even if there's
 138  * no switch in the current function, but we were inside
 139  * a switch inside another function, from which this one was called.
 140  */
 141 static INLINE boolean
 142 mask_has_switch(struct lp_exec_mask *mask)
 143 {
 144    int i;
 145    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 146       const struct function_ctx *ctx = &mask->function_stack[i];
 147       if (ctx->switch_stack_size > 0)
 148          return TRUE;
 149    }
 150    return FALSE;
 151 }
 152
 153 /*
 154  * Returns true if we're inside a conditional.
 155  * It's global, meaning that it returns true even if there's
 156  * no conditional in the current function, but we were inside
 157  * a conditional inside another function, from which this one was called.
 158  */
 159 static INLINE boolean
 160 mask_has_cond(struct lp_exec_mask *mask)
 161 {
 162    int i;
 163    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 164       const struct function_ctx *ctx = &mask->function_stack[i];
 165       if (ctx->cond_stack_size > 0)
 166          return TRUE;
 167    }
 168    return FALSE;
 169 }
 170
 171
 172 /*
 173  * Initialize a function context at the specified index.
 174  */
 175 static void
 176 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
 177 {
 178    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 179    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 180    struct function_ctx *ctx =  &mask->function_stack[function_idx];
 181
 182    ctx->cond_stack_size = 0;
 183    ctx->loop_stack_size = 0;
 184    ctx->switch_stack_size = 0;
 185
 186    if (function_idx == 0) {
 187       ctx->ret_mask = mask->ret_mask;
 188    }
 189
 190    ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
 191                                        int_type, "looplimiter");
 192    LLVMBuildStore(
 193       builder,
 194       LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
 195       ctx->loop_limiter);
 196 }
 197
 198 static void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
 199 {
 200    mask->bld = bld;
 201    mask->has_mask = FALSE;
 202    mask->ret_in_main = FALSE;
 203    /* For the main function */
 204    mask->function_stack_size = 1;
 205
 206    mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
 207    mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
 208          mask->cond_mask = mask->switch_mask =
 209          LLVMConstAllOnes(mask->int_vec_type);
 210
 211    mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
 212                                  sizeof(mask->function_stack[0]));
 213    lp_exec_mask_function_init(mask, 0);
 214 }
 215
 216 static void
 217 lp_exec_mask_fini(struct lp_exec_mask *mask)
 218 {
 219    FREE(mask->function_stack);
 220 }
 221
 222 static void lp_exec_mask_update(struct lp_exec_mask *mask)
 223 {
 224    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 225    boolean has_loop_mask = mask_has_loop(mask);
 226    boolean has_cond_mask = mask_has_cond(mask);
 227    boolean has_switch_mask = mask_has_switch(mask);
 228    boolean has_ret_mask = mask->function_stack_size > 1 ||
 229          mask->ret_in_main;
 230
 231    if (has_loop_mask) {
 232       /*for loops we need to update the entire mask at runtime */
 233       LLVMValueRef tmp;
 234       assert(mask->break_mask);
 235       tmp = LLVMBuildAnd(builder,
 236                          mask->cont_mask,
 237                          mask->break_mask,
 238                          "maskcb");
 239       mask->exec_mask = LLVMBuildAnd(builder,
 240                                      mask->cond_mask,
 241                                      tmp,
 242                                      "maskfull");
 243    } else
 244       mask->exec_mask = mask->cond_mask;
 245
 246    if (has_switch_mask) {
 247       mask->exec_mask = LLVMBuildAnd(builder,
 248                                      mask->exec_mask,
 249                                      mask->switch_mask,
 250                                      "switchmask");
 251    }
 252
 253    if (has_ret_mask) {
 254       mask->exec_mask = LLVMBuildAnd(builder,
 255                                      mask->exec_mask,
 256                                      mask->ret_mask,
 257                                      "callmask");
 258    }
 259
 260    mask->has_mask = (has_cond_mask ||
 261                      has_loop_mask ||
 262                      has_switch_mask ||
 263                      has_ret_mask);
 264 }
 265
 266 static void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
 267                                    LLVMValueRef val)
 268 {
 269    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 270    struct function_ctx *ctx = func_ctx(mask);
 271
 272    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
 273       ctx->cond_stack_size++;
 274       return;
 275    }
 276    if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
 277       assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
 278    }
 279    ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
 280    assert(LLVMTypeOf(val) == mask->int_vec_type);
 281    mask->cond_mask = LLVMBuildAnd(builder,
 282                                   mask->cond_mask,
 283                                   val,
 284                                   "");
 285    lp_exec_mask_update(mask);
 286 }
 287
 288 static void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
 289 {
 290    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 291    struct function_ctx *ctx = func_ctx(mask);
 292    LLVMValueRef prev_mask;
 293    LLVMValueRef inv_mask;
 294
 295    assert(ctx->cond_stack_size);
 296    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 297       return;
 298    prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
 299    if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
 300       assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
 301    }
 302
 303    inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");
 304
 305    mask->cond_mask = LLVMBuildAnd(builder,
 306                                   inv_mask,
 307                                   prev_mask, "");
 308    lp_exec_mask_update(mask);
 309 }
 310
 311 static void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
 312 {
 313    struct function_ctx *ctx = func_ctx(mask);
 314    assert(ctx->cond_stack_size);
 315    --ctx->cond_stack_size;
 316    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 317       return;
 318    mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
 319    lp_exec_mask_update(mask);
 320 }
 321
 322 static void lp_exec_bgnloop(struct lp_exec_mask *mask)
 323 {
 324    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 325    struct function_ctx *ctx = func_ctx(mask);
 326
 327    if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
 328       ++ctx->loop_stack_size;
 329       return;
 330    }
 331
 332    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 333       ctx->break_type;
 334    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;
 335
 336    ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
 337    ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
 338    ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
 339    ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
 340    ++ctx->loop_stack_size;
 341
 342    ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
 343    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 344
 345    ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");
 346
 347    LLVMBuildBr(builder, ctx->loop_block);
 348    LLVMPositionBuilderAtEnd(builder, ctx->loop_block);
 349
 350    mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
 351
 352    lp_exec_mask_update(mask);
 353 }
 354
 355 static void lp_exec_break(struct lp_exec_mask *mask,
 356                           struct lp_build_tgsi_context * bld_base)
 357 {
 358    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 359    struct function_ctx *ctx = func_ctx(mask);
 360
 361    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 362       LLVMValueRef exec_mask = LLVMBuildNot(builder,
 363                                             mask->exec_mask,
 364                                             "break");
 365
 366       mask->break_mask = LLVMBuildAnd(builder,
 367                                       mask->break_mask,
 368                                       exec_mask, "break_full");
 369    }
 370    else {
 371       unsigned opcode = bld_base->instructions[bld_base->pc + 1].Instruction.Opcode;
 372       boolean break_always = (opcode == TGSI_OPCODE_ENDSWITCH ||
 373                               opcode == TGSI_OPCODE_CASE);
 374
 375
 376       if (ctx->switch_in_default) {
 377          /*
 378           * stop default execution but only if this is an unconditional switch.
 379           * (The condition here is not perfect since dead code after break is
 380           * allowed but should be sufficient since false negatives are just
 381           * unoptimized - so we don't have to pre-evaluate that).
 382           */
 383          if(break_always && ctx->switch_pc) {
 384             bld_base->pc = ctx->switch_pc;
 385             return;
 386          }
 387       }
 388
 389       if (break_always) {
 390          mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
 391       }
 392       else {
 393          LLVMValueRef exec_mask = LLVMBuildNot(builder,
 394                                                mask->exec_mask,
 395                                                "break");
 396          mask->switch_mask = LLVMBuildAnd(builder,
 397                                           mask->switch_mask,
 398                                           exec_mask, "break_switch");
 399       }
 400    }
 401
 402    lp_exec_mask_update(mask);
 403 }
 404
 405 static void lp_exec_break_condition(struct lp_exec_mask *mask,
 406                                     LLVMValueRef cond)
 407 {
 408    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 409    struct function_ctx *ctx = func_ctx(mask);
 410    LLVMValueRef cond_mask = LLVMBuildAnd(builder,
 411                                          mask->exec_mask,
 412                                          cond, "cond_mask");
 413    cond_mask = LLVMBuildNot(builder, cond_mask, "break_cond");
 414
 415    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 416       mask->break_mask = LLVMBuildAnd(builder,
 417                                       mask->break_mask,
 418                                       cond_mask, "breakc_full");
 419    }
 420    else {
 421       mask->switch_mask = LLVMBuildAnd(builder,
 422                                        mask->switch_mask,
 423                                        cond_mask, "breakc_switch");
 424    }
 425
 426    lp_exec_mask_update(mask);
 427 }
 428
 429 static void lp_exec_continue(struct lp_exec_mask *mask)
 430 {
 431    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 432    LLVMValueRef exec_mask = LLVMBuildNot(builder,
 433                                          mask->exec_mask,
 434                                          "");
 435
 436    mask->cont_mask = LLVMBuildAnd(builder,
 437                                   mask->cont_mask,
 438                                   exec_mask, "");
 439
 440    lp_exec_mask_update(mask);
 441 }
 442
 443
 444 static void lp_exec_endloop(struct gallivm_state *gallivm,
 445                             struct lp_exec_mask *mask)
 446 {
 447    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 448    struct function_ctx *ctx = func_ctx(mask);
 449    LLVMBasicBlockRef endloop;
 450    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 451    LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
 452                                                mask->bld->type.width *
 453                                                mask->bld->type.length);
 454    LLVMValueRef i1cond, i2cond, icond, limiter;
 455
 456    assert(mask->break_mask);
 457
 458
 459    assert(ctx->loop_stack_size);
 460    if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 461       --ctx->loop_stack_size;
 462       return;
 463    }
 464
 465    /*
 466     * Restore the cont_mask, but don't pop
 467     */
 468    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
 469    lp_exec_mask_update(mask);
 470
 471    /*
 472     * Unlike the continue mask, the break_mask must be preserved across loop
 473     * iterations
 474     */
 475    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 476
 477    /* Decrement the loop limiter */
 478    limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");
 479
 480    limiter = LLVMBuildSub(
 481       builder,
 482       limiter,
 483       LLVMConstInt(int_type, 1, false),
 484       "");
 485
 486    LLVMBuildStore(builder, limiter, ctx->loop_limiter);
 487
 488    /* i1cond = (mask != 0) */
 489    i1cond = LLVMBuildICmp(
 490       builder,
 491       LLVMIntNE,
 492       LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
 493       LLVMConstNull(reg_type), "i1cond");
 494
 495    /* i2cond = (looplimiter > 0) */
 496    i2cond = LLVMBuildICmp(
 497       builder,
 498       LLVMIntSGT,
 499       limiter,
 500       LLVMConstNull(int_type), "i2cond");
 501
 502    /* if( i1cond && i2cond ) */
 503    icond = LLVMBuildAnd(builder, i1cond, i2cond, "");
 504
 505    endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");
 506
 507    LLVMBuildCondBr(builder,
 508                    icond, ctx->loop_block, endloop);
 509
 510    LLVMPositionBuilderAtEnd(builder, endloop);
 511
 512    assert(ctx->loop_stack_size);
 513    --ctx->loop_stack_size;
 514    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
 515    mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
 516    ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
 517    ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
 518    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
 519          ctx->switch_stack_size];
 520
 521    lp_exec_mask_update(mask);
 522 }
 523
 524 static void lp_exec_switch(struct lp_exec_mask *mask,
 525                            LLVMValueRef switchval)
 526 {
 527    struct function_ctx *ctx = func_ctx(mask);
 528
 529    if (ctx->switch_stack_size >= LP_MAX_TGSI_NESTING ||
 530        ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 531       ctx->switch_stack_size++;
 532       return;
 533    }
 534
 535    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 536       ctx->break_type;
 537    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_SWITCH;
 538
 539    ctx->switch_stack[ctx->switch_stack_size].switch_mask = mask->switch_mask;
 540    ctx->switch_stack[ctx->switch_stack_size].switch_val = ctx->switch_val;
 541    ctx->switch_stack[ctx->switch_stack_size].switch_mask_default = ctx->switch_mask_default;
 542    ctx->switch_stack[ctx->switch_stack_size].switch_in_default = ctx->switch_in_default;
 543    ctx->switch_stack[ctx->switch_stack_size].switch_pc = ctx->switch_pc;
 544    ctx->switch_stack_size++;
 545
 546    mask->switch_mask = LLVMConstNull(mask->int_vec_type);
 547    ctx->switch_val = switchval;
 548    ctx->switch_mask_default = LLVMConstNull(mask->int_vec_type);
 549    ctx->switch_in_default = false;
 550    ctx->switch_pc = 0;
 551
 552    lp_exec_mask_update(mask);
 553 }
 554
 555 static void lp_exec_endswitch(struct lp_exec_mask *mask,
 556                               struct lp_build_tgsi_context * bld_base)
 557 {
 558    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 559    struct function_ctx *ctx = func_ctx(mask);
 560
 561    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 562       ctx->switch_stack_size--;
 563       return;
 564    }
 565
 566    /* check if there's deferred default if so do it now */
 567    if (ctx->switch_pc && !ctx->switch_in_default) {
 568       LLVMValueRef prevmask, defaultmask;
 569       unsigned tmp_pc;
 570       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 571       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 572       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 573       ctx->switch_in_default = true;
 574
 575       lp_exec_mask_update(mask);
 576
 577       assert(bld_base->instructions[ctx->switch_pc - 1].Instruction.Opcode ==
 578              TGSI_OPCODE_DEFAULT);
 579
 580       tmp_pc = bld_base->pc;
 581       bld_base->pc = ctx->switch_pc;
 582       /*
 583        * re-purpose switch_pc to point to here again, since we stop execution of
 584        * the deferred default after next break.
 585        */
 586       ctx->switch_pc = tmp_pc - 1;
 587
 588       return;
 589    }
 590
 591    else if (ctx->switch_pc && ctx->switch_in_default) {
 592       assert(bld_base->pc == ctx->switch_pc + 1);
 593    }
 594
 595    ctx->switch_stack_size--;
 596    mask->switch_mask = ctx->switch_stack[ctx->switch_stack_size].switch_mask;
 597    ctx->switch_val = ctx->switch_stack[ctx->switch_stack_size].switch_val;
 598    ctx->switch_mask_default = ctx->switch_stack[ctx->switch_stack_size].switch_mask_default;
 599    ctx->switch_in_default = ctx->switch_stack[ctx->switch_stack_size].switch_in_default;
 600    ctx->switch_pc = ctx->switch_stack[ctx->switch_stack_size].switch_pc;
 601
 602    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size];
 603
 604    lp_exec_mask_update(mask);
 605 }
 606
 607 static void lp_exec_case(struct lp_exec_mask *mask,
 608                          LLVMValueRef caseval)
 609 {
 610    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 611    struct function_ctx *ctx = func_ctx(mask);
 612
 613    LLVMValueRef casemask, prevmask;
 614
 615    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 616       return;
 617    }
 618
 619    /* skipping case mask evaluation here is NOT optional (not in all cases anyway). */
 620    if (!ctx->switch_in_default) {
 621       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 622       casemask = lp_build_cmp(mask->bld, PIPE_FUNC_EQUAL, caseval, ctx->switch_val);
 623       ctx->switch_mask_default = LLVMBuildOr(builder, casemask,
 624                                              ctx->switch_mask_default, "sw_default_mask");
 625       casemask = LLVMBuildOr(builder, casemask, mask->switch_mask, "");
 626       mask->switch_mask = LLVMBuildAnd(builder, casemask, prevmask, "sw_mask");
 627
 628       lp_exec_mask_update(mask);
 629    }
 630 }
 631
 632 /*
 633  * Analyse default statement in a switch.
 634  * \return true if default is last statement, false otherwise
 635  * \param default_pc_start contains pc of instruction to jump to
 636  *                         if default wasn't last but there's no
 637  *                         fallthrough into default.
 638  */
 639 static boolean default_analyse_is_last(struct lp_exec_mask *mask,
 640                                        struct lp_build_tgsi_context * bld_base,
 641                                        int *default_pc_start)
 642 {
 643    unsigned pc = bld_base->pc;
 644    struct function_ctx *ctx = func_ctx(mask);
 645    unsigned curr_switch_stack = ctx->switch_stack_size;
 646
 647    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 648       return false;
 649    }
 650
 651    /* skip over case statements which are together with default */
 652    while (bld_base->instructions[pc].Instruction.Opcode == TGSI_OPCODE_CASE) {
 653       pc++;
 654    }
 655
 656    while (pc != -1 && pc < bld_base->num_instructions) {
 657       unsigned opcode = bld_base->instructions[pc].Instruction.Opcode;
 658       switch (opcode) {
 659       case TGSI_OPCODE_CASE:
 660          if (curr_switch_stack == ctx->switch_stack_size) {
 661             *default_pc_start = pc - 1;
 662             return false;
 663          }
 664          break;
 665       case TGSI_OPCODE_SWITCH:
 666          curr_switch_stack++;
 667          break;
 668       case TGSI_OPCODE_ENDSWITCH:
 669          if (curr_switch_stack == ctx->switch_stack_size) {
 670             *default_pc_start = pc - 1;
 671             return true;
 672          }
 673          curr_switch_stack--;
 674          break;
 675       }
 676       pc++;
 677    }
 678    /* should never arrive here */
 679    assert(0);
 680    return true;
 681 }
 682
 683 static void lp_exec_default(struct lp_exec_mask *mask,
 684                             struct lp_build_tgsi_context * bld_base)
 685 {
 686    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 687    struct function_ctx *ctx = func_ctx(mask);
 688
 689    int default_exec_pc;
 690    boolean default_is_last;
 691
 692    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 693       return;
 694    }
 695
 696    /*
 697     * This is a messy opcode, because it may not be always at the end and
 698     * there can be fallthrough in and out of it.
 699     */
 700
 701    default_is_last = default_analyse_is_last(mask, bld_base, &default_exec_pc);
 702    /*
 703     * If it is last statement in switch (note that case statements appearing
 704     * "at the same time" as default don't change that) everything is just fine,
 705     * update switch mask and go on. This means we can handle default with
 706     * fallthrough INTO it without overhead, if it is last.
 707     */
 708    if (default_is_last) {
 709       LLVMValueRef prevmask, defaultmask;
 710       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 711       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 712       defaultmask = LLVMBuildOr(builder, defaultmask, mask->switch_mask, "");
 713       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 714       ctx->switch_in_default = true;
 715
 716       lp_exec_mask_update(mask);
 717    }
 718    else {
 719       /*
 720        * Technically, "case" immediately before default isn't really a
 721        * fallthrough, however we still have to count them as such as we
 722        * already have updated the masks.
 723        * If that happens in practice could add a switch optimizer pass
 724        * which just gets rid of all case statements appearing together with
 725        * default (or could do switch analysis at switch start time instead).
 726        */
 727       unsigned opcode = bld_base->instructions[bld_base->pc - 1].Instruction.Opcode;
 728       boolean ft_into = (opcode != TGSI_OPCODE_BRK &&
 729                          opcode != TGSI_OPCODE_SWITCH);
 730       /*
 731        * If it is not last statement and there was no fallthrough into it,
 732        * we record the PC and continue execution at next case (again, those
 733        * case encountered at the same time don't count). At endswitch
 734        * time, we update switchmask, and go back executing the code we skipped
 735        * until the next break (possibly re-executing some code with changed mask
 736        * if there was a fallthrough out of default).
 737        * Finally, if it is not last statement and there was a fallthrough into it,
 738        * do the same as with the former case, except instead of skipping the code
 739        * just execute it without updating the mask, then go back and re-execute.
 740        */
 741       ctx->switch_pc = bld_base->pc;
 742       if (!ft_into) {
 743          bld_base->pc = default_exec_pc;
 744       }
 745    }
 746 }
 747
 748
 749 /* stores val into an address pointed to by dst_ptr.
 750  * mask->exec_mask is used to figure out which bits of val
 751  * should be stored into the address
 752  * (0 means don't store this bit, 1 means do store).
 753  */
 754 static void lp_exec_mask_store(struct lp_exec_mask *mask,
 755                                struct lp_build_context *bld_store,
 756                                LLVMValueRef pred,
 757                                LLVMValueRef val,
 758                                LLVMValueRef dst_ptr)
 759 {
 760    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 761
 762    assert(lp_check_value(bld_store->type, val));
 763    assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
 764    assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val));
 765
 766    /* Mix the predicate and execution mask */
 767    if (mask->has_mask) {
 768       if (pred) {
 769          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
 770       } else {
 771          pred = mask->exec_mask;
 772       }
 773    }
 774
 775    if (pred) {
 776       LLVMValueRef res, dst;
 777
 778       dst = LLVMBuildLoad(builder, dst_ptr, "");
 779       res = lp_build_select(bld_store, pred, val, dst);
 780       LLVMBuildStore(builder, res, dst_ptr);
 781    } else
 782       LLVMBuildStore(builder, val, dst_ptr);
 783 }
 784
 785 static void lp_exec_mask_call(struct lp_exec_mask *mask,
 786                               int func,
 787                               int *pc)
 788 {
 789    if (mask->function_stack_size >= LP_MAX_NUM_FUNCS) {
 790       return;
 791    }
 792
 793    lp_exec_mask_function_init(mask, mask->function_stack_size);
 794    mask->function_stack[mask->function_stack_size].pc = *pc;
 795    mask->function_stack[mask->function_stack_size].ret_mask = mask->ret_mask;
 796    mask->function_stack_size++;
 797    *pc = func;
 798 }
 799
 800 static void lp_exec_mask_ret(struct lp_exec_mask *mask, int *pc)
 801 {
 802    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 803    struct function_ctx *ctx = func_ctx(mask);
 804    LLVMValueRef exec_mask;
 805
 806    if (ctx->cond_stack_size == 0 &&
 807        ctx->loop_stack_size == 0 &&
 808        ctx->switch_stack_size == 0 &&
 809        mask->function_stack_size == 1) {
 810       /* returning from main() */
 811       *pc = -1;
 812       return;
 813    }
 814
 815    if (mask->function_stack_size == 1) {
 816       /*
 817        * This requires special handling since we need to ensure
 818        * we don't drop the mask even if we have no call stack
 819        * (e.g. after a ret in a if clause after the endif)
 820        */
 821       mask->ret_in_main = TRUE;
 822    }
 823
 824    exec_mask = LLVMBuildNot(builder,
 825                             mask->exec_mask,
 826                             "ret");
 827
 828    mask->ret_mask = LLVMBuildAnd(builder,
 829                                  mask->ret_mask,
 830                                  exec_mask, "ret_full");
 831
 832    lp_exec_mask_update(mask);
 833 }
 834
 835 static void lp_exec_mask_bgnsub(struct lp_exec_mask *mask)
 836 {
 837 }
 838
 839 static void lp_exec_mask_endsub(struct lp_exec_mask *mask, int *pc)
 840 {
 841    struct function_ctx *ctx;
 842
 843    assert(mask->function_stack_size > 1);
 844    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 845
 846    ctx = func_ctx(mask);
 847    mask->function_stack_size--;
 848
 849    *pc = ctx->pc;
 850    mask->ret_mask = ctx->ret_mask;
 851
 852    lp_exec_mask_update(mask);
 853 }
 854
 855
 856 static LLVMValueRef
 857 get_file_ptr(struct lp_build_tgsi_soa_context *bld,
 858              unsigned file,
 859              unsigned index,
 860              unsigned chan)
 861 {
 862    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 863    LLVMValueRef (*array_of_vars)[TGSI_NUM_CHANNELS];
 864    LLVMValueRef var_of_array;
 865
 866    switch (file) {
 867    case TGSI_FILE_TEMPORARY:
 868       array_of_vars = bld->temps;
 869       var_of_array = bld->temps_array;
 870       break;
 871    case TGSI_FILE_OUTPUT:
 872       array_of_vars = bld->outputs;
 873       var_of_array = bld->outputs_array;
 874       break;
 875    default:
 876       assert(0);
 877       return NULL;
 878    }
 879
 880    assert(chan < 4);
 881
 882    if (bld->indirect_files & (1 << file)) {
 883       LLVMValueRef lindex = lp_build_const_int32(bld->bld_base.base.gallivm, index * 4 + chan);
 884       return LLVMBuildGEP(builder, var_of_array, &lindex, 1, "");
 885    }
 886    else {
 887       assert(index <= bld->bld_base.info->file_max[file]);
 888       return array_of_vars[index][chan];
 889    }
 890 }
 891
 892
 893 /**
 894  * Return pointer to a temporary register channel (src or dest).
 895  * Note that indirect addressing cannot be handled here.
 896  * \param index  which temporary register
 897  * \param chan  which channel of the temp register.
 898  */
 899 LLVMValueRef
 900 lp_get_temp_ptr_soa(struct lp_build_tgsi_soa_context *bld,
 901              unsigned index,
 902              unsigned chan)
 903 {
 904    return get_file_ptr(bld, TGSI_FILE_TEMPORARY, index, chan);
 905 }
 906
 907 /**
 908  * Return pointer to a output register channel (src or dest).
 909  * Note that indirect addressing cannot be handled here.
 910  * \param index  which output register
 911  * \param chan  which channel of the output register.
 912  */
 913 LLVMValueRef
 914 lp_get_output_ptr(struct lp_build_tgsi_soa_context *bld,
 915                unsigned index,
 916                unsigned chan)
 917 {
 918    return get_file_ptr(bld, TGSI_FILE_OUTPUT, index, chan);
 919 }
 920
 921 /*
 922  * If we have indirect addressing in outputs copy our alloca array
 923  * to the outputs slots specified by the caller to make sure
 924  * our outputs are delivered consistently via the same interface.
 925  */
 926 static void
 927 gather_outputs(struct lp_build_tgsi_soa_context * bld)
 928 {
 929    if ((bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
 930       unsigned index, chan;
 931       assert(bld->bld_base.info->num_outputs <=
 932              bld->bld_base.info->file_max[TGSI_FILE_OUTPUT] + 1);
 933       for (index = 0; index < bld->bld_base.info->num_outputs; ++index) {
 934          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 935             bld->outputs[index][chan] = lp_get_output_ptr(bld, index, chan);
 936          }
 937       }
 938    }
 939 }
 940
 941 /**
 942  * Gather vector.
 943  * XXX the lp_build_gather() function should be capable of doing this
 944  * with a little work.
 945  */
 946 static LLVMValueRef
 947 build_gather(struct lp_build_context *bld,
 948              LLVMValueRef base_ptr,
 949              LLVMValueRef indexes,
 950              LLVMValueRef *overflow_mask)
 951 {
 952    LLVMBuilderRef builder = bld->gallivm->builder;
 953    LLVMValueRef res = bld->undef;
 954    unsigned i;
 955    LLVMValueRef temp_ptr;
 956
 957    if (overflow_mask) {
 958       temp_ptr = lp_build_alloca(
 959          bld->gallivm,
 960          lp_build_vec_type(bld->gallivm, bld->type), "");
 961    }
 962
 963    /*
 964     * Loop over elements of index_vec, load scalar value, insert it into 'res'.
 965     */
 966    for (i = 0; i < bld->type.length; i++) {
 967       LLVMValueRef ii = lp_build_const_int32(bld->gallivm, i);
 968       LLVMValueRef index = LLVMBuildExtractElement(builder,
 969                                                    indexes, ii, "");
 970       LLVMValueRef scalar_ptr, scalar;
 971       LLVMValueRef overflow;
 972       struct lp_build_if_state if_ctx;
 973
 974       /*
 975        * overflow_mask is a boolean vector telling us which channels
 976        * in the vector overflowed. We use the overflow behavior for
 977        * constant buffers which is defined as:
 978        * Out of bounds access to constant buffer returns 0 in all
 979        * componenets. Out of bounds behavior is always with respect
 980        * to the size of the buffer bound at that slot.
 981        */
 982       if (overflow_mask) {
 983          overflow = LLVMBuildExtractElement(builder, *overflow_mask,
 984                                             ii, "");
 985          lp_build_if(&if_ctx, bld->gallivm, overflow);
 986          {
 987             LLVMValueRef val = LLVMBuildLoad(builder, temp_ptr, "");
 988             val = LLVMBuildInsertElement(
 989                builder, val,
 990                LLVMConstNull(LLVMFloatTypeInContext(bld->gallivm->context)),
 991                ii, "");
 992             LLVMBuildStore(builder, val, temp_ptr);
 993          }
 994          lp_build_else(&if_ctx);
 995          {
 996             LLVMValueRef val = LLVMBuildLoad(builder, temp_ptr, "");
 997
 998             scalar_ptr = LLVMBuildGEP(builder, base_ptr,
 999                                       &index, 1, "gather_ptr");
1000             scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1001
1002             val = LLVMBuildInsertElement(builder, val, scalar, ii, "");
1003
1004             LLVMBuildStore(builder, val, temp_ptr);
1005          }
1006          lp_build_endif(&if_ctx);
1007       } else {
1008          scalar_ptr = LLVMBuildGEP(builder, base_ptr,
1009                                    &index, 1, "gather_ptr");
1010          scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1011
1012          res = LLVMBuildInsertElement(builder, res, scalar, ii, "");
1013       }
1014    }
1015
1016    if (overflow_mask) {
1017       res = LLVMBuildLoad(builder, temp_ptr, "gather_val");
1018    }
1019
1020    return res;
1021 }
1022
1023
1024 /**
1025  * Scatter/store vector.
1026  */
1027 static void
1028 emit_mask_scatter(struct lp_build_tgsi_soa_context *bld,
1029                   LLVMValueRef base_ptr,
1030                   LLVMValueRef indexes,
1031                   LLVMValueRef values,
1032                   struct lp_exec_mask *mask,
1033                   LLVMValueRef pred)
1034 {
1035    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1036    LLVMBuilderRef builder = gallivm->builder;
1037    unsigned i;
1038
1039    /* Mix the predicate and execution mask */
1040    if (mask->has_mask) {
1041       if (pred) {
1042          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
1043       }
1044       else {
1045          pred = mask->exec_mask;
1046       }
1047    }
1048
1049    /*
1050     * Loop over elements of index_vec, store scalar value.
1051     */
1052    for (i = 0; i < bld->bld_base.base.type.length; i++) {
1053       LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1054       LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
1055       LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
1056       LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
1057       LLVMValueRef scalar_pred = pred ?
1058          LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
1059
1060       if (0)
1061          lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
1062                          ii, val, index, scalar_ptr);
1063
1064       if (scalar_pred) {
1065          LLVMValueRef real_val, dst_val;
1066          dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
1067          real_val = lp_build_select(&bld->elem_bld, scalar_pred, val, dst_val);
1068          LLVMBuildStore(builder, real_val, scalar_ptr);
1069       }
1070       else {
1071          LLVMBuildStore(builder, val, scalar_ptr);
1072       }
1073    }
1074 }
1075
1076
1077 /**
1078  * Read the current value of the ADDR register, convert the floats to
1079  * ints, add the base index and return the vector of offsets.
1080  * The offsets will be used to index into the constant buffer or
1081  * temporary register file.
1082  */
1083 static LLVMValueRef
1084 get_indirect_index(struct lp_build_tgsi_soa_context *bld,
1085                    unsigned reg_file, unsigned reg_index,
1086                    const struct tgsi_ind_register *indirect_reg)
1087 {
1088    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1089    struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
1090    /* always use X component of address register */
1091    unsigned swizzle = indirect_reg->Swizzle;
1092    LLVMValueRef base;
1093    LLVMValueRef rel;
1094    LLVMValueRef max_index;
1095    LLVMValueRef index;
1096
1097    assert(bld->indirect_files & (1 << reg_file));
1098
1099    base = lp_build_const_int_vec(bld->bld_base.base.gallivm, uint_bld->type, reg_index);
1100
1101    assert(swizzle < 4);
1102    switch (indirect_reg->File) {
1103    case TGSI_FILE_ADDRESS:
1104       rel = LLVMBuildLoad(builder,
1105                           bld->addr[indirect_reg->Index][swizzle],
1106                           "load addr reg");
1107       /* ADDR LLVM values already have LLVM integer type. */
1108       break;
1109    case TGSI_FILE_TEMPORARY:
1110       rel = lp_get_temp_ptr_soa(bld, indirect_reg->Index, swizzle);
1111       rel = LLVMBuildLoad(builder, rel, "load temp reg");
1112       /* TEMP LLVM values always have LLVM float type, but for indirection, the
1113        * value actually stored is expected to be an integer */
1114       rel = LLVMBuildBitCast(builder, rel, uint_bld->vec_type, "");
1115       break;
1116    default:
1117       assert(0);
1118       rel = uint_bld->zero;
1119    }
1120
1121    index = lp_build_add(uint_bld, base, rel);
1122
1123    /*
1124     * emit_fetch_constant handles constant buffer overflow so this code
1125     * is pointless for them.
1126     * Furthermore the D3D10 spec in section 6.5 says:
1127     * If the constant buffer bound to a slot is larger than the size
1128     * declared in the shader for that slot, implementations are allowed
1129     * to return incorrect data (not necessarily 0) for indices that are
1130     * larger than the declared size but smaller than the buffer size.
1131     */
1132    if (reg_file != TGSI_FILE_CONSTANT) {
1133       max_index = lp_build_const_int_vec(bld->bld_base.base.gallivm,
1134                                          uint_bld->type,
1135                                          bld->bld_base.info->file_max[reg_file]);
1136
1137       assert(!uint_bld->type.sign);
1138       index = lp_build_min(uint_bld, index, max_index);
1139    }
1140
1141    return index;
1142 }
1143
1144 static struct lp_build_context *
1145 stype_to_fetch(struct lp_build_tgsi_context * bld_base,
1146                enum tgsi_opcode_type stype)
1147 {
1148    struct lp_build_context *bld_fetch;
1149
1150    switch (stype) {
1151    case TGSI_TYPE_FLOAT:
1152    case TGSI_TYPE_UNTYPED:
1153       bld_fetch = &bld_base->base;
1154       break;
1155    case TGSI_TYPE_UNSIGNED:
1156       bld_fetch = &bld_base->uint_bld;
1157       break;
1158    case TGSI_TYPE_SIGNED:
1159       bld_fetch = &bld_base->int_bld;
1160       break;
1161    case TGSI_TYPE_VOID:
1162    case TGSI_TYPE_DOUBLE:
1163    default:
1164       assert(0);
1165       bld_fetch = NULL;
1166       break;
1167    }
1168    return bld_fetch;
1169 }
1170
1171 static LLVMValueRef
1172 get_soa_array_offsets(struct lp_build_context *uint_bld,
1173                       LLVMValueRef indirect_index,
1174                       unsigned chan_index,
1175                       boolean need_perelement_offset)
1176 {
1177    struct gallivm_state *gallivm = uint_bld->gallivm;
1178    LLVMValueRef chan_vec =
1179       lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
1180    LLVMValueRef length_vec =
1181       lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
1182    LLVMValueRef index_vec;
1183
1184    /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
1185    index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1186    index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
1187    index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
1188
1189    if (need_perelement_offset) {
1190       LLVMValueRef pixel_offsets;
1191       int i;
1192      /* build pixel offset vector: {0, 1, 2, 3, ...} */
1193       pixel_offsets = uint_bld->undef;
1194       for (i = 0; i < uint_bld->type.length; i++) {
1195          LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1196          pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
1197                                                 ii, ii, "");
1198       }
1199       index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
1200    }
1201    return index_vec;
1202 }
1203
1204 static LLVMValueRef
1205 emit_fetch_constant(
1206    struct lp_build_tgsi_context * bld_base,
1207    const struct tgsi_full_src_register * reg,
1208    enum tgsi_opcode_type stype,
1209    unsigned swizzle)
1210 {
1211    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1212    struct gallivm_state *gallivm = bld_base->base.gallivm;
1213    LLVMBuilderRef builder = gallivm->builder;
1214    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1215    unsigned dimension = 0;
1216    LLVMValueRef consts_ptr;
1217    LLVMValueRef num_consts;
1218    LLVMValueRef res;
1219
1220    /* XXX: Handle fetching xyzw components as a vector */
1221    assert(swizzle != ~0);
1222
1223    if (reg->Register.Dimension) {
1224       assert(!reg->Dimension.Indirect);
1225       dimension = reg->Dimension.Index;
1226       assert(dimension < LP_MAX_TGSI_CONST_BUFFERS);
1227    }
1228
1229    consts_ptr = bld->consts[dimension];
1230    num_consts = bld->consts_sizes[dimension];
1231
1232    if (reg->Register.Indirect) {
1233       LLVMValueRef indirect_index;
1234       LLVMValueRef swizzle_vec =
1235          lp_build_const_int_vec(gallivm, uint_bld->type, swizzle);
1236       LLVMValueRef index_vec;  /* index into the const buffer */
1237       LLVMValueRef overflow_mask;
1238
1239       indirect_index = get_indirect_index(bld,
1240                                           reg->Register.File,
1241                                           reg->Register.Index,
1242                                           &reg->Indirect);
1243
1244       /* All fetches are from the same constant buffer, so
1245        * we need to propagate the size to a vector to do a
1246        * vector comparison */
1247       num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
1248       /* Construct a boolean vector telling us which channels
1249        * overflow the bound constant buffer */
1250       overflow_mask = LLVMBuildICmp(builder, LLVMIntUGE,
1251                                     indirect_index,
1252                                     num_consts, "");
1253
1254       /* index_vec = indirect_index * 4 + swizzle */
1255       index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1256       index_vec = lp_build_add(uint_bld, index_vec, swizzle_vec);
1257
1258       /* Gather values from the constant buffer */
1259       res = build_gather(&bld_base->base, consts_ptr, index_vec,
1260                          &overflow_mask);
1261    }
1262    else {
1263       LLVMValueRef index;  /* index into the const buffer */
1264       LLVMValueRef scalar, scalar_ptr;
1265
1266       index = lp_build_const_int32(gallivm, reg->Register.Index * 4 + swizzle);
1267
1268       scalar_ptr = LLVMBuildGEP(builder, consts_ptr,
1269                                 &index, 1, "");
1270       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1271       res = lp_build_broadcast_scalar(&bld_base->base, scalar);
1272    }
1273
1274    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED) {
1275       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1276       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1277    }
1278
1279    return res;
1280 }
1281
1282 static LLVMValueRef
1283 emit_fetch_immediate(
1284    struct lp_build_tgsi_context * bld_base,
1285    const struct tgsi_full_src_register * reg,
1286    enum tgsi_opcode_type stype,
1287    unsigned swizzle)
1288 {
1289    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1290    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1291    LLVMBuilderRef builder = gallivm->builder;
1292    LLVMValueRef res = NULL;
1293
1294    if (bld->use_immediates_array || reg->Register.Indirect) {
1295       LLVMValueRef imms_array;
1296       LLVMTypeRef fptr_type;
1297
1298       /* cast imms_array pointer to float* */
1299       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1300       imms_array = LLVMBuildBitCast(builder, bld->imms_array, fptr_type, "");
1301
1302       if (reg->Register.Indirect) {
1303          LLVMValueRef indirect_index;
1304          LLVMValueRef index_vec;  /* index into the immediate register array */
1305
1306          indirect_index = get_indirect_index(bld,
1307                                              reg->Register.File,
1308                                              reg->Register.Index,
1309                                              &reg->Indirect);
1310          /*
1311           * Unlike for other reg classes, adding pixel offsets is unnecessary -
1312           * immediates are stored as full vectors (FIXME??? - might be better
1313           * to store them the same as constants) but all elements are the same
1314           * in any case.
1315           */
1316          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1317                                            indirect_index,
1318                                            swizzle,
1319                                            FALSE);
1320
1321          /* Gather values from the immediate register array */
1322          res = build_gather(&bld_base->base, imms_array, index_vec, NULL);
1323       } else {
1324          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1325                                         reg->Register.Index * 4 + swizzle);
1326          LLVMValueRef imms_ptr =  LLVMBuildGEP(builder,
1327                                                 bld->imms_array, &lindex, 1, "");
1328          res = LLVMBuildLoad(builder, imms_ptr, "");
1329       }
1330    }
1331    else {
1332       res = bld->immediates[reg->Register.Index][swizzle];
1333    }
1334
1335    if (stype == TGSI_TYPE_UNSIGNED) {
1336       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1337    } else if (stype == TGSI_TYPE_SIGNED) {
1338       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1339    }
1340    return res;
1341 }
1342
1343 static LLVMValueRef
1344 emit_fetch_input(
1345    struct lp_build_tgsi_context * bld_base,
1346    const struct tgsi_full_src_register * reg,
1347    enum tgsi_opcode_type stype,
1348    unsigned swizzle)
1349 {
1350    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1351    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1352    LLVMBuilderRef builder = gallivm->builder;
1353    LLVMValueRef res;
1354
1355    if (reg->Register.Indirect) {
1356       LLVMValueRef indirect_index;
1357       LLVMValueRef index_vec;  /* index into the input reg array */
1358       LLVMValueRef inputs_array;
1359       LLVMTypeRef fptr_type;
1360
1361       indirect_index = get_indirect_index(bld,
1362                                           reg->Register.File,
1363                                           reg->Register.Index,
1364                                           &reg->Indirect);
1365
1366       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1367                                         indirect_index,
1368                                         swizzle,
1369                                         TRUE);
1370
1371       /* cast inputs_array pointer to float* */
1372       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1373       inputs_array = LLVMBuildBitCast(builder, bld->inputs_array, fptr_type, "");
1374
1375       /* Gather values from the input register array */
1376       res = build_gather(&bld_base->base, inputs_array, index_vec, NULL);
1377    } else {
1378       if (bld->indirect_files & (1 << TGSI_FILE_INPUT)) {
1379          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1380                                         reg->Register.Index * 4 + swizzle);
1381          LLVMValueRef input_ptr =  LLVMBuildGEP(builder,
1382                                                 bld->inputs_array, &lindex, 1, "");
1383          res = LLVMBuildLoad(builder, input_ptr, "");
1384       }
1385       else {
1386          res = bld->inputs[reg->Register.Index][swizzle];
1387       }
1388    }
1389
1390    assert(res);
1391
1392    if (stype == TGSI_TYPE_UNSIGNED) {
1393       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1394    } else if (stype == TGSI_TYPE_SIGNED) {
1395       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1396    }
1397
1398    return res;
1399 }
1400
1401
1402 static LLVMValueRef
1403 emit_fetch_gs_input(
1404    struct lp_build_tgsi_context * bld_base,
1405    const struct tgsi_full_src_register * reg,
1406    enum tgsi_opcode_type stype,
1407    unsigned swizzle)
1408 {
1409    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1410    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1411    LLVMBuilderRef builder = gallivm->builder;
1412    LLVMValueRef attrib_index = NULL;
1413    LLVMValueRef vertex_index = NULL;
1414    LLVMValueRef swizzle_index = lp_build_const_int32(gallivm, swizzle);
1415    LLVMValueRef res;
1416
1417    if (reg->Register.Indirect) {
1418       attrib_index = get_indirect_index(bld,
1419                                         reg->Register.File,
1420                                         reg->Register.Index,
1421                                         &reg->Indirect);
1422    } else {
1423       attrib_index = lp_build_const_int32(gallivm, reg->Register.Index);
1424    }
1425
1426    if (reg->Dimension.Indirect) {
1427       vertex_index = get_indirect_index(bld,
1428                                         reg->Register.File,
1429                                         reg->Dimension.Index,
1430                                         &reg->DimIndirect);
1431    } else {
1432       vertex_index = lp_build_const_int32(gallivm, reg->Dimension.Index);
1433    }
1434
1435    res = bld->gs_iface->fetch_input(bld->gs_iface, bld_base,
1436                                     reg->Dimension.Indirect,
1437                                     vertex_index,
1438                                     reg->Register.Indirect,
1439                                     attrib_index,
1440                                     swizzle_index);
1441
1442    assert(res);
1443
1444    if (stype == TGSI_TYPE_UNSIGNED) {
1445       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1446    } else if (stype == TGSI_TYPE_SIGNED) {
1447       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1448    }
1449
1450    return res;
1451 }
1452
1453 static LLVMValueRef
1454 emit_fetch_temporary(
1455    struct lp_build_tgsi_context * bld_base,
1456    const struct tgsi_full_src_register * reg,
1457    enum tgsi_opcode_type stype,
1458    unsigned swizzle)
1459 {
1460    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1461    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1462    LLVMBuilderRef builder = gallivm->builder;
1463    LLVMValueRef res;
1464
1465    if (reg->Register.Indirect) {
1466       LLVMValueRef indirect_index;
1467       LLVMValueRef index_vec;  /* index into the temp reg array */
1468       LLVMValueRef temps_array;
1469       LLVMTypeRef fptr_type;
1470
1471       indirect_index = get_indirect_index(bld,
1472                                           reg->Register.File,
1473                                           reg->Register.Index,
1474                                           &reg->Indirect);
1475
1476       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1477                                         indirect_index,
1478                                         swizzle,
1479                                         TRUE);
1480
1481       /* cast temps_array pointer to float* */
1482       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1483       temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1484
1485       /* Gather values from the temporary register array */
1486       res = build_gather(&bld_base->base, temps_array, index_vec, NULL);
1487    }
1488    else {
1489       LLVMValueRef temp_ptr;
1490       temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle);
1491       res = LLVMBuildLoad(builder, temp_ptr, "");
1492    }
1493
1494    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED) {
1495       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1496       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1497    }
1498
1499    return res;
1500 }
1501
1502 static LLVMValueRef
1503 emit_fetch_system_value(
1504    struct lp_build_tgsi_context * bld_base,
1505    const struct tgsi_full_src_register * reg,
1506    enum tgsi_opcode_type stype,
1507    unsigned swizzle)
1508 {
1509    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1510    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1511    const struct tgsi_shader_info *info = bld->bld_base.info;
1512    LLVMBuilderRef builder = gallivm->builder;
1513    LLVMValueRef res;
1514    enum tgsi_opcode_type atype; // Actual type of the value
1515
1516    assert(!reg->Register.Indirect);
1517
1518    switch (info->system_value_semantic_name[reg->Register.Index]) {
1519    case TGSI_SEMANTIC_INSTANCEID:
1520       res = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
1521       atype = TGSI_TYPE_UNSIGNED;
1522       break;
1523
1524    case TGSI_SEMANTIC_VERTEXID:
1525       res = bld->system_values.vertex_id;
1526       atype = TGSI_TYPE_UNSIGNED;
1527       break;
1528
1529    case TGSI_SEMANTIC_PRIMID:
1530       res = bld->system_values.prim_id;
1531       atype = TGSI_TYPE_UNSIGNED;
1532       break;
1533
1534    default:
1535       assert(!"unexpected semantic in emit_fetch_system_value");
1536       res = bld_base->base.zero;
1537       atype = TGSI_TYPE_FLOAT;
1538       break;
1539    }
1540
1541    if (atype != stype) {
1542       if (stype == TGSI_TYPE_FLOAT) {
1543          res = LLVMBuildBitCast(builder, res, bld_base->base.vec_type, "");
1544       } else if (stype == TGSI_TYPE_UNSIGNED) {
1545          res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1546       } else if (stype == TGSI_TYPE_SIGNED) {
1547          res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1548       }
1549    }
1550
1551    return res;
1552 }
1553
1554 /**
1555  * Register fetch with derivatives.
1556  */
1557 static void
1558 emit_fetch_deriv(
1559    struct lp_build_tgsi_soa_context *bld,
1560    LLVMValueRef src,
1561    LLVMValueRef *res,
1562    LLVMValueRef *ddx,
1563    LLVMValueRef *ddy)
1564 {
1565    if(res)
1566       *res = src;
1567
1568    /* TODO: use interpolation coeffs for inputs */
1569
1570    if(ddx)
1571       *ddx = lp_build_ddx(&bld->bld_base.base, src);
1572
1573    if(ddy)
1574       *ddy = lp_build_ddy(&bld->bld_base.base, src);
1575 }
1576
1577
1578 /**
1579  * Predicate.
1580  */
1581 static void
1582 emit_fetch_predicate(
1583    struct lp_build_tgsi_soa_context *bld,
1584    const struct tgsi_full_instruction *inst,
1585    LLVMValueRef *pred)
1586 {
1587    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1588    unsigned index;
1589    unsigned char swizzles[4];
1590    LLVMValueRef unswizzled[4] = {NULL, NULL, NULL, NULL};
1591    LLVMValueRef value;
1592    unsigned chan;
1593
1594    if (!inst->Instruction.Predicate) {
1595       TGSI_FOR_EACH_CHANNEL( chan ) {
1596          pred[chan] = NULL;
1597       }
1598       return;
1599    }
1600
1601    swizzles[0] = inst->Predicate.SwizzleX;
1602    swizzles[1] = inst->Predicate.SwizzleY;
1603    swizzles[2] = inst->Predicate.SwizzleZ;
1604    swizzles[3] = inst->Predicate.SwizzleW;
1605
1606    index = inst->Predicate.Index;
1607    assert(index < LP_MAX_TGSI_PREDS);
1608
1609    TGSI_FOR_EACH_CHANNEL( chan ) {
1610       unsigned swizzle = swizzles[chan];
1611
1612       /*
1613        * Only fetch the predicate register channels that are actually listed
1614        * in the swizzles
1615        */
1616       if (!unswizzled[swizzle]) {
1617          value = LLVMBuildLoad(builder,
1618                                bld->preds[index][swizzle], "");
1619
1620          /*
1621           * Convert the value to an integer mask.
1622           *
1623           * TODO: Short-circuit this comparison -- a D3D setp_xx instructions
1624           * is needlessly causing two comparisons due to storing the intermediate
1625           * result as float vector instead of an integer mask vector.
1626           */
1627          value = lp_build_compare(bld->bld_base.base.gallivm,
1628                                   bld->bld_base.base.type,
1629                                   PIPE_FUNC_NOTEQUAL,
1630                                   value,
1631                                   bld->bld_base.base.zero);
1632          if (inst->Predicate.Negate) {
1633             value = LLVMBuildNot(builder, value, "");
1634          }
1635
1636          unswizzled[swizzle] = value;
1637       } else {
1638          value = unswizzled[swizzle];
1639       }
1640
1641       pred[chan] = value;
1642    }
1643 }
1644
1645
1646 /**
1647  * Register store.
1648  */
1649 static void
1650 emit_store_chan(
1651    struct lp_build_tgsi_context *bld_base,
1652    const struct tgsi_full_instruction *inst,
1653    unsigned index,
1654    unsigned chan_index,
1655    LLVMValueRef pred,
1656    LLVMValueRef value)
1657 {
1658    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1659    struct gallivm_state *gallivm = bld_base->base.gallivm;
1660    LLVMBuilderRef builder = gallivm->builder;
1661    const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1662    struct lp_build_context *float_bld = &bld_base->base;
1663    struct lp_build_context *int_bld = &bld_base->int_bld;
1664    LLVMValueRef indirect_index = NULL;
1665    enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode);
1666
1667    /*
1668     * Apply saturation.
1669     *
1670     * It is always assumed to be float.
1671     */
1672    switch( inst->Instruction.Saturate ) {
1673    case TGSI_SAT_NONE:
1674       break;
1675
1676    case TGSI_SAT_ZERO_ONE:
1677       assert(dtype == TGSI_TYPE_FLOAT ||
1678              dtype == TGSI_TYPE_UNTYPED);
1679       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1680       value = lp_build_clamp_zero_one_nanzero(float_bld, value);
1681       break;
1682
1683    case TGSI_SAT_MINUS_PLUS_ONE:
1684       assert(dtype == TGSI_TYPE_FLOAT ||
1685              dtype == TGSI_TYPE_UNTYPED);
1686       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1687       /* This will give -1.0 for NaN which is probably not what we want. */
1688       value = lp_build_max_ext(float_bld, value,
1689                                lp_build_const_vec(gallivm, float_bld->type, -1.0),
1690                                GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
1691       value = lp_build_min(float_bld, value, float_bld->one);
1692       break;
1693
1694    default:
1695       assert(0);
1696    }
1697
1698    if (reg->Register.Indirect) {
1699       indirect_index = get_indirect_index(bld,
1700                                           reg->Register.File,
1701                                           reg->Register.Index,
1702                                           &reg->Indirect);
1703    } else {
1704       assert(reg->Register.Index <=
1705                              bld_base->info->file_max[reg->Register.File]);
1706    }
1707
1708    if (DEBUG_EXECUTION) {
1709       emit_dump_reg(gallivm, reg->Register.File, reg->Register.Index, chan_index, value);
1710    }
1711
1712    switch( reg->Register.File ) {
1713    case TGSI_FILE_OUTPUT:
1714       /* Outputs are always stored as floats */
1715       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1716
1717       if (reg->Register.Indirect) {
1718          LLVMValueRef index_vec;  /* indexes into the output registers */
1719          LLVMValueRef outputs_array;
1720          LLVMTypeRef fptr_type;
1721
1722          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1723                                            indirect_index,
1724                                            chan_index,
1725                                            TRUE);
1726
1727          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1728          outputs_array = LLVMBuildBitCast(builder, bld->outputs_array, fptr_type, "");
1729
1730          /* Scatter store values into output registers */
1731          emit_mask_scatter(bld, outputs_array, index_vec, value,
1732                            &bld->exec_mask, pred);
1733       }
1734       else {
1735          LLVMValueRef out_ptr = lp_get_output_ptr(bld, reg->Register.Index,
1736                                                   chan_index);
1737          lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, out_ptr);
1738       }
1739       break;
1740
1741    case TGSI_FILE_TEMPORARY:
1742       /* Temporaries are always stored as floats */
1743       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1744
1745       if (reg->Register.Indirect) {
1746          LLVMValueRef index_vec;  /* indexes into the temp registers */
1747          LLVMValueRef temps_array;
1748          LLVMTypeRef fptr_type;
1749
1750          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1751                                            indirect_index,
1752                                            chan_index,
1753                                            TRUE);
1754
1755          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1756          temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1757
1758          /* Scatter store values into temp registers */
1759          emit_mask_scatter(bld, temps_array, index_vec, value,
1760                            &bld->exec_mask, pred);
1761       }
1762       else {
1763          LLVMValueRef temp_ptr;
1764          temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, chan_index);
1765          lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, temp_ptr);
1766       }
1767       break;
1768
1769    case TGSI_FILE_ADDRESS:
1770       assert(dtype == TGSI_TYPE_SIGNED);
1771       assert(LLVMTypeOf(value) == int_bld->vec_type);
1772       value = LLVMBuildBitCast(builder, value, int_bld->vec_type, "");
1773       lp_exec_mask_store(&bld->exec_mask, int_bld, pred, value,
1774                          bld->addr[reg->Register.Index][chan_index]);
1775       break;
1776
1777    case TGSI_FILE_PREDICATE:
1778       assert(LLVMTypeOf(value) == float_bld->vec_type);
1779       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1780       lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value,
1781                          bld->preds[reg->Register.Index][chan_index]);
1782       break;
1783
1784    default:
1785       assert( 0 );
1786    }
1787
1788    (void)dtype;
1789 }
1790
1791 /*
1792  * Called at the beginning of the translation of each TGSI instruction, to
1793  * emit some debug code.
1794  */
1795 static void
1796 emit_debug(
1797    struct lp_build_tgsi_context * bld_base,
1798    const struct tgsi_full_instruction * inst,
1799    const struct tgsi_opcode_info * info)
1800
1801 {
1802    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1803
1804    if (DEBUG_EXECUTION) {
1805       /*
1806        * Dump the TGSI instruction.
1807        */
1808
1809       struct gallivm_state *gallivm = bld_base->base.gallivm;
1810       char buf[512];
1811       buf[0] = '$';
1812       buf[1] = ' ';
1813       tgsi_dump_instruction_str(inst, bld_base->pc, &buf[2], sizeof buf - 2);
1814       lp_build_printf(gallivm, buf);
1815
1816       /* Dump the execution mask.
1817        */
1818       if (bld->exec_mask.has_mask) {
1819          lp_build_print_value(gallivm, "    mask = ", bld->exec_mask.exec_mask);
1820       }
1821    }
1822 }
1823
1824 static void
1825 emit_store(
1826    struct lp_build_tgsi_context * bld_base,
1827    const struct tgsi_full_instruction * inst,
1828    const struct tgsi_opcode_info * info,
1829    LLVMValueRef dst[4])
1830
1831 {
1832    unsigned chan_index;
1833    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1834
1835    if(info->num_dst) {
1836       LLVMValueRef pred[TGSI_NUM_CHANNELS];
1837
1838       emit_fetch_predicate( bld, inst, pred );
1839
1840       TGSI_FOR_EACH_DST0_ENABLED_CHANNEL( inst, chan_index ) {
1841          emit_store_chan(bld_base, inst, 0, chan_index, pred[chan_index], dst[chan_index]);
1842       }
1843    }
1844 }
1845
1846 static unsigned
1847 tgsi_to_pipe_tex_target(unsigned tgsi_target)
1848 {
1849    switch (tgsi_target) {
1850    case TGSI_TEXTURE_BUFFER:
1851       return PIPE_BUFFER;
1852    case TGSI_TEXTURE_1D:
1853    case TGSI_TEXTURE_SHADOW1D:
1854       return PIPE_TEXTURE_1D;
1855    case TGSI_TEXTURE_2D:
1856    case TGSI_TEXTURE_SHADOW2D:
1857    case TGSI_TEXTURE_2D_MSAA:
1858       return PIPE_TEXTURE_2D;
1859    case TGSI_TEXTURE_3D:
1860       return PIPE_TEXTURE_3D;
1861    case TGSI_TEXTURE_CUBE:
1862    case TGSI_TEXTURE_SHADOWCUBE:
1863       return PIPE_TEXTURE_CUBE;
1864    case TGSI_TEXTURE_RECT:
1865    case TGSI_TEXTURE_SHADOWRECT:
1866       return PIPE_TEXTURE_RECT;
1867    case TGSI_TEXTURE_1D_ARRAY:
1868    case TGSI_TEXTURE_SHADOW1D_ARRAY:
1869       return PIPE_TEXTURE_1D_ARRAY;
1870    case TGSI_TEXTURE_2D_ARRAY:
1871    case TGSI_TEXTURE_SHADOW2D_ARRAY:
1872    case TGSI_TEXTURE_2D_ARRAY_MSAA:
1873       return PIPE_TEXTURE_2D_ARRAY;
1874    case TGSI_TEXTURE_CUBE_ARRAY:
1875    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
1876       return PIPE_TEXTURE_CUBE_ARRAY;
1877    default:
1878       assert(0);
1879       return PIPE_BUFFER;
1880    }
1881 }
1882
1883
1884 static enum lp_sampler_lod_property
1885 lp_build_lod_property(
1886    struct lp_build_tgsi_context *bld_base,
1887    const struct tgsi_full_instruction *inst,
1888    unsigned src_op)
1889 {
1890    const struct tgsi_full_src_register *reg = &inst->Src[src_op];
1891    enum lp_sampler_lod_property lod_property;
1892
1893    /*
1894     * Not much we can do here. We could try catching inputs declared
1895     * with constant interpolation but not sure it's worth it - since for
1896     * TEX opcodes as well as FETCH/LD the lod comes from same reg as
1897     * the coords, so it could only work for SAMPLE/TXQ/SVIEWINFO), just
1898     * like the constant/immediate recognition below.
1899     * What seems to be of more value would be to recognize temps holding
1900     * broadcasted scalars but no way we can do it.
1901     * Tried asking llvm but without any success (using LLVMIsConstant
1902     * even though this isn't exactly what we'd need), even as simple as
1903     * IMM[0] UINT32 (0,-1,0,0)
1904     * MOV TEMP[0] IMM[0].yyyy
1905     * SVIEWINFO TEMP[1], TEMP[0].xxxx, SVIEWINFO[0]
1906     * doesn't work.
1907     * This means there's ZERO chance this will ever catch a scalar lod
1908     * with traditional tex opcodes as well as texel fetches, since the lod
1909     * comes from the same reg as coords (except some test shaders using
1910     * constant coords maybe).
1911     * There's at least hope for sample opcodes as well as size queries.
1912     */
1913    if (reg->Register.File == TGSI_FILE_CONSTANT ||
1914        reg->Register.File == TGSI_FILE_IMMEDIATE) {
1915       lod_property = LP_SAMPLER_LOD_SCALAR;
1916    }
1917    else if (bld_base->info->processor == TGSI_PROCESSOR_FRAGMENT) {
1918       if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
1919          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1920       }
1921       else {
1922          lod_property = LP_SAMPLER_LOD_PER_QUAD;
1923       }
1924    }
1925    else {
1926       /* never use scalar (per-quad) lod the results are just too wrong. */
1927       lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1928    }
1929    return lod_property;
1930 }
1931
1932
1933 /**
1934  * High-level instruction translators.
1935  */
1936
1937 static void
1938 emit_tex( struct lp_build_tgsi_soa_context *bld,
1939           const struct tgsi_full_instruction *inst,
1940           enum lp_build_tex_modifier modifier,
1941           LLVMValueRef *texel)
1942 {
1943    unsigned unit;
1944    LLVMValueRef lod_bias, explicit_lod;
1945    LLVMValueRef oow = NULL;
1946    LLVMValueRef coords[5];
1947    LLVMValueRef offsets[3] = { NULL };
1948    struct lp_derivatives derivs;
1949    struct lp_derivatives *deriv_ptr = NULL;
1950    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
1951    unsigned num_derivs, num_offsets, i;
1952    unsigned shadow_coord = 0;
1953    unsigned layer_coord = 0;
1954
1955    if (!bld->sampler) {
1956       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
1957       for (i = 0; i < 4; i++) {
1958          texel[i] = bld->bld_base.base.undef;
1959       }
1960       return;
1961    }
1962
1963    switch (inst->Texture.Texture) {
1964    case TGSI_TEXTURE_1D_ARRAY:
1965       layer_coord = 1;
1966       /* fallthrough */
1967    case TGSI_TEXTURE_1D:
1968       num_offsets = 1;
1969       num_derivs = 1;
1970       break;
1971    case TGSI_TEXTURE_2D_ARRAY:
1972       layer_coord = 2;
1973       /* fallthrough */
1974    case TGSI_TEXTURE_2D:
1975    case TGSI_TEXTURE_RECT:
1976       num_offsets = 2;
1977       num_derivs = 2;
1978       break;
1979    case TGSI_TEXTURE_SHADOW1D_ARRAY:
1980       layer_coord = 1;
1981       /* fallthrough */
1982    case TGSI_TEXTURE_SHADOW1D:
1983       shadow_coord = 2;
1984       num_offsets = 1;
1985       num_derivs = 1;
1986       break;
1987    case TGSI_TEXTURE_SHADOW2D_ARRAY:
1988       layer_coord = 2;
1989       shadow_coord = 3;
1990       num_offsets = 2;
1991       num_derivs = 2;
1992       break;
1993    case TGSI_TEXTURE_SHADOW2D:
1994    case TGSI_TEXTURE_SHADOWRECT:
1995       shadow_coord = 2;
1996       num_offsets = 2;
1997       num_derivs = 2;
1998       break;
1999    case TGSI_TEXTURE_CUBE:
2000       num_offsets = 2;
2001       num_derivs = 3;
2002       break;
2003    case TGSI_TEXTURE_3D:
2004       num_offsets = 3;
2005       num_derivs = 3;
2006       break;
2007    case TGSI_TEXTURE_SHADOWCUBE:
2008       shadow_coord = 3;
2009       num_offsets = 2;
2010       num_derivs = 3;
2011       break;
2012    case TGSI_TEXTURE_CUBE_ARRAY:
2013    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2014    case TGSI_TEXTURE_2D_MSAA:
2015    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2016    default:
2017       assert(0);
2018       return;
2019    }
2020
2021    /* Note lod and especially projected are illegal in a LOT of cases */
2022    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2023        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2024       LLVMValueRef lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2025       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2026          lod_bias = lod;
2027          explicit_lod = NULL;
2028       }
2029       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2030          lod_bias = NULL;
2031          explicit_lod = lod;
2032       }
2033       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2034    }
2035    else {
2036       lod_bias = NULL;
2037       explicit_lod = NULL;
2038    }
2039
2040    if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED) {
2041       oow = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2042       oow = lp_build_rcp(&bld->bld_base.base, oow);
2043    }
2044
2045    for (i = 0; i < num_derivs; i++) {
2046       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2047       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2048          coords[i] = lp_build_mul(&bld->bld_base.base, coords[i], oow);
2049    }
2050    for (i = num_derivs; i < 5; i++) {
2051       coords[i] = bld->bld_base.base.undef;
2052    }
2053
2054    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2055    if (layer_coord) {
2056       coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2057       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2058          coords[2] = lp_build_mul(&bld->bld_base.base, coords[2], oow);
2059    }
2060    /* Shadow coord occupies always 5th slot. */
2061    if (shadow_coord) {
2062       coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 0, shadow_coord);
2063       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2064          coords[4] = lp_build_mul(&bld->bld_base.base, coords[4], oow);
2065    }
2066
2067    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2068       unsigned dim;
2069       for (dim = 0; dim < num_derivs; ++dim) {
2070          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 1, dim);
2071          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 2, dim);
2072       }
2073       deriv_ptr = &derivs;
2074       unit = inst->Src[3].Register.Index;
2075       /*
2076        * could also check all src regs if constant but I doubt such
2077        * cases exist in practice.
2078        */
2079       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2080          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2081             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2082          }
2083          else {
2084             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2085          }
2086       }
2087       else {
2088          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2089       }
2090    } else {
2091       unit = inst->Src[1].Register.Index;
2092    }
2093
2094    /* some advanced gather instructions (txgo) would require 4 offsets */
2095    if (inst->Texture.NumOffsets == 1) {
2096       unsigned dim;
2097       for (dim = 0; dim < num_offsets; dim++) {
2098          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2099       }
2100    }
2101
2102    bld->sampler->emit_fetch_texel(bld->sampler,
2103                                   bld->bld_base.base.gallivm,
2104                                   bld->bld_base.base.type,
2105                                   FALSE,
2106                                   unit, unit,
2107                                   coords,
2108                                   offsets,
2109                                   deriv_ptr,
2110                                   lod_bias, explicit_lod, lod_property,
2111                                   texel);
2112 }
2113
2114 static void
2115 emit_sample(struct lp_build_tgsi_soa_context *bld,
2116             const struct tgsi_full_instruction *inst,
2117             enum lp_build_tex_modifier modifier,
2118             boolean compare,
2119             LLVMValueRef *texel)
2120 {
2121    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2122    unsigned texture_unit, sampler_unit;
2123    LLVMValueRef lod_bias, explicit_lod;
2124    LLVMValueRef coords[5];
2125    LLVMValueRef offsets[3] = { NULL };
2126    struct lp_derivatives derivs;
2127    struct lp_derivatives *deriv_ptr = NULL;
2128    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2129
2130    unsigned num_offsets, num_derivs, i;
2131    unsigned layer_coord = 0;
2132
2133    if (!bld->sampler) {
2134       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2135       for (i = 0; i < 4; i++) {
2136          texel[i] = bld->bld_base.base.undef;
2137       }
2138       return;
2139    }
2140
2141    /*
2142     * unlike old-style tex opcodes the texture/sampler indices
2143     * always come from src1 and src2 respectively.
2144     */
2145    texture_unit = inst->Src[1].Register.Index;
2146    sampler_unit = inst->Src[2].Register.Index;
2147
2148    /*
2149     * Note inst->Texture.Texture will contain the number of offsets,
2150     * however the target information is NOT there and comes from the
2151     * declared sampler views instead.
2152     */
2153    switch (bld->sv[texture_unit].Resource) {
2154    case TGSI_TEXTURE_1D:
2155       num_offsets = 1;
2156       num_derivs = 1;
2157       break;
2158    case TGSI_TEXTURE_1D_ARRAY:
2159       layer_coord = 1;
2160       num_offsets = 1;
2161       num_derivs = 1;
2162       break;
2163    case TGSI_TEXTURE_2D:
2164    case TGSI_TEXTURE_RECT:
2165       num_offsets = 2;
2166       num_derivs = 2;
2167       break;
2168    case TGSI_TEXTURE_2D_ARRAY:
2169       layer_coord = 2;
2170       num_offsets = 2;
2171       num_derivs = 2;
2172       break;
2173    case TGSI_TEXTURE_CUBE:
2174       num_offsets = 2;
2175       num_derivs = 3;
2176       break;
2177    case TGSI_TEXTURE_3D:
2178       num_offsets = 3;
2179       num_derivs = 3;
2180       break;
2181    case TGSI_TEXTURE_CUBE_ARRAY:
2182       layer_coord = 3;
2183       num_offsets = 2;
2184       num_derivs = 3;
2185       break;
2186    default:
2187       assert(0);
2188       return;
2189    }
2190
2191    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2192        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2193       LLVMValueRef lod = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2194       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2195          lod_bias = lod;
2196          explicit_lod = NULL;
2197       }
2198       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2199          lod_bias = NULL;
2200          explicit_lod = lod;
2201       }
2202       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2203    }
2204    else if (modifier == LP_BLD_TEX_MODIFIER_LOD_ZERO) {
2205       lod_bias = NULL;
2206       /* XXX might be better to explicitly pass the level zero information */
2207       explicit_lod = lp_build_const_vec(gallivm, bld->bld_base.base.type, 0.0F);
2208    }
2209    else {
2210       lod_bias = NULL;
2211       explicit_lod = NULL;
2212    }
2213
2214    for (i = 0; i < num_derivs; i++) {
2215       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2216    }
2217    for (i = num_derivs; i < 5; i++) {
2218       coords[i] = bld->bld_base.base.undef;
2219    }
2220
2221    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2222    if (layer_coord) {
2223       if (layer_coord == 3)
2224          coords[3] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2225       else
2226          coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2227    }
2228    /* Shadow coord occupies always 5th slot. */
2229    if (compare) {
2230       coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2231    }
2232
2233    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2234       unsigned dim;
2235       for (dim = 0; dim < num_derivs; ++dim) {
2236          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 3, dim);
2237          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 4, dim);
2238       }
2239       deriv_ptr = &derivs;
2240       /*
2241        * could also check all src regs if constant but I doubt such
2242        * cases exist in practice.
2243        */
2244       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2245          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2246             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2247          }
2248          else {
2249             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2250          }
2251       }
2252       else {
2253          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2254       }
2255    }
2256
2257    /* some advanced gather instructions (txgo) would require 4 offsets */
2258    if (inst->Texture.NumOffsets == 1) {
2259       unsigned dim;
2260       for (dim = 0; dim < num_offsets; dim++) {
2261          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2262       }
2263    }
2264
2265    bld->sampler->emit_fetch_texel(bld->sampler,
2266                                   bld->bld_base.base.gallivm,
2267                                   bld->bld_base.base.type,
2268                                   FALSE,
2269                                   texture_unit, sampler_unit,
2270                                   coords,
2271                                   offsets,
2272                                   deriv_ptr,
2273                                   lod_bias, explicit_lod, lod_property,
2274                                   texel);
2275
2276    if (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2277        inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2278        inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2279        inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA) {
2280       unsigned char swizzles[4];
2281       swizzles[0] = inst->Src[1].Register.SwizzleX;
2282       swizzles[1] = inst->Src[1].Register.SwizzleY;
2283       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2284       swizzles[3] = inst->Src[1].Register.SwizzleW;
2285
2286       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2287    }
2288 }
2289
2290 static void
2291 emit_fetch_texels( struct lp_build_tgsi_soa_context *bld,
2292                    const struct tgsi_full_instruction *inst,
2293                    LLVMValueRef *texel,
2294                    boolean is_samplei)
2295 {
2296    unsigned unit, target;
2297    LLVMValueRef coord_undef = LLVMGetUndef(bld->bld_base.base.int_vec_type);
2298    LLVMValueRef explicit_lod = NULL;
2299    LLVMValueRef coords[3];
2300    LLVMValueRef offsets[3] = { NULL };
2301    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2302    unsigned dims, i;
2303    unsigned layer_coord = 0;
2304
2305    if (!bld->sampler) {
2306       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2307       for (i = 0; i < 4; i++) {
2308          texel[i] = coord_undef;
2309       }
2310       return;
2311    }
2312
2313    unit = inst->Src[1].Register.Index;
2314
2315    if (is_samplei) {
2316       target = bld->sv[unit].Resource;
2317    }
2318    else {
2319       target = inst->Texture.Texture;
2320    }
2321
2322    switch (target) {
2323    case TGSI_TEXTURE_1D:
2324    case TGSI_TEXTURE_BUFFER:
2325       dims = 1;
2326       break;
2327    case TGSI_TEXTURE_1D_ARRAY:
2328       layer_coord = 1;
2329       dims = 1;
2330       break;
2331    case TGSI_TEXTURE_2D:
2332    case TGSI_TEXTURE_RECT:
2333       dims = 2;
2334       break;
2335    case TGSI_TEXTURE_2D_ARRAY:
2336       layer_coord = 2;
2337       dims = 2;
2338       break;
2339    case TGSI_TEXTURE_3D:
2340       dims = 3;
2341       break;
2342    default:
2343       assert(0);
2344       return;
2345    }
2346
2347    /* always have lod except for buffers ? */
2348    if (target != TGSI_TEXTURE_BUFFER) {
2349       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2350       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2351    }
2352
2353    for (i = 0; i < dims; i++) {
2354       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2355    }
2356    for (i = dims; i < 3; i++) {
2357       coords[i] = coord_undef;
2358    }
2359    if (layer_coord)
2360       coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2361
2362    if (inst->Texture.NumOffsets == 1) {
2363       unsigned dim;
2364       for (dim = 0; dim < dims; dim++) {
2365          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2366       }
2367    }
2368
2369    bld->sampler->emit_fetch_texel(bld->sampler,
2370                                   bld->bld_base.base.gallivm,
2371                                   bld->bld_base.base.type,
2372                                   TRUE,
2373                                   unit, unit,
2374                                   coords,
2375                                   offsets,
2376                                   NULL,
2377                                   NULL, explicit_lod, lod_property,
2378                                   texel);
2379
2380    if (is_samplei &&
2381        (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2382         inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2383         inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2384         inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA)) {
2385       unsigned char swizzles[4];
2386       swizzles[0] = inst->Src[1].Register.SwizzleX;
2387       swizzles[1] = inst->Src[1].Register.SwizzleY;
2388       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2389       swizzles[3] = inst->Src[1].Register.SwizzleW;
2390
2391       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2392    }
2393 }
2394
2395 static void
2396 emit_size_query( struct lp_build_tgsi_soa_context *bld,
2397                  const struct tgsi_full_instruction *inst,
2398                  LLVMValueRef *sizes_out,
2399                  boolean is_sviewinfo)
2400 {
2401    LLVMValueRef explicit_lod;
2402    enum lp_sampler_lod_property lod_property;
2403    unsigned has_lod;
2404    unsigned i;
2405    unsigned unit = inst->Src[1].Register.Index;
2406    unsigned target, pipe_target;
2407
2408    if (is_sviewinfo) {
2409       target = bld->sv[unit].Resource;
2410    }
2411    else {
2412       target = inst->Texture.Texture;
2413    }
2414    switch (target) {
2415    case TGSI_TEXTURE_BUFFER:
2416    case TGSI_TEXTURE_RECT:
2417    case TGSI_TEXTURE_SHADOWRECT:
2418       has_lod = 0;
2419       break;
2420    default:
2421       has_lod = 1;
2422       break;
2423    }
2424
2425    if (!bld->sampler) {
2426       _debug_printf("warning: found texture query instruction but no sampler generator supplied\n");
2427       for (i = 0; i < 4; i++)
2428          sizes_out[i] = bld->bld_base.int_bld.undef;
2429       return;
2430    }
2431
2432    if (has_lod) {
2433       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 0);
2434       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2435    }
2436    else {
2437       explicit_lod = NULL;
2438       lod_property = LP_SAMPLER_LOD_SCALAR;
2439    }
2440
2441
2442    pipe_target = tgsi_to_pipe_tex_target(target);
2443
2444    bld->sampler->emit_size_query(bld->sampler,
2445                                  bld->bld_base.base.gallivm,
2446                                  bld->bld_base.int_bld.type,
2447                                  unit, pipe_target,
2448                                  is_sviewinfo,
2449                                  lod_property,
2450                                  explicit_lod,
2451                                  sizes_out);
2452 }
2453
2454 static boolean
2455 near_end_of_shader(struct lp_build_tgsi_soa_context *bld,
2456                    int pc)
2457 {
2458    int i;
2459
2460    for (i = 0; i < 5; i++) {
2461       unsigned opcode;
2462
2463       if (pc + i >= bld->bld_base.info->num_instructions)
2464          return TRUE;
2465
2466       opcode = bld->bld_base.instructions[pc + i].Instruction.Opcode;
2467
2468       if (opcode == TGSI_OPCODE_END)
2469          return TRUE;
2470
2471       if (opcode == TGSI_OPCODE_TEX ||
2472           opcode == TGSI_OPCODE_TXP ||
2473           opcode == TGSI_OPCODE_TXD ||
2474           opcode == TGSI_OPCODE_TXB ||
2475           opcode == TGSI_OPCODE_TXL ||
2476           opcode == TGSI_OPCODE_TXF ||
2477           opcode == TGSI_OPCODE_TXQ ||
2478           opcode == TGSI_OPCODE_CAL ||
2479           opcode == TGSI_OPCODE_CALLNZ ||
2480           opcode == TGSI_OPCODE_IF ||
2481           opcode == TGSI_OPCODE_UIF ||
2482           opcode == TGSI_OPCODE_BGNLOOP ||
2483           opcode == TGSI_OPCODE_SWITCH)
2484          return FALSE;
2485    }
2486
2487    return TRUE;
2488 }
2489
2490
2491
2492 /**
2493  * Kill fragment if any of the src register values are negative.
2494  */
2495 static void
2496 emit_kill_if(
2497    struct lp_build_tgsi_soa_context *bld,
2498    const struct tgsi_full_instruction *inst,
2499    int pc)
2500 {
2501    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2502    const struct tgsi_full_src_register *reg = &inst->Src[0];
2503    LLVMValueRef terms[TGSI_NUM_CHANNELS];
2504    LLVMValueRef mask;
2505    unsigned chan_index;
2506
2507    memset(&terms, 0, sizeof terms);
2508
2509    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2510       unsigned swizzle;
2511
2512       /* Unswizzle channel */
2513       swizzle = tgsi_util_get_full_src_register_swizzle( reg, chan_index );
2514
2515       /* Check if the component has not been already tested. */
2516       assert(swizzle < TGSI_NUM_CHANNELS);
2517       if( !terms[swizzle] )
2518          /* TODO: change the comparison operator instead of setting the sign */
2519          terms[swizzle] =  lp_build_emit_fetch(&bld->bld_base, inst, 0, chan_index );
2520    }
2521
2522    mask = NULL;
2523    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2524       if(terms[chan_index]) {
2525          LLVMValueRef chan_mask;
2526
2527          /*
2528           * If term < 0 then mask = 0 else mask = ~0.
2529           */
2530          chan_mask = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GEQUAL, terms[chan_index], bld->bld_base.base.zero);
2531
2532          if(mask)
2533             mask = LLVMBuildAnd(builder, mask, chan_mask, "");
2534          else
2535             mask = chan_mask;
2536       }
2537    }
2538
2539    if (bld->exec_mask.has_mask) {
2540       LLVMValueRef invmask;
2541       invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2542       mask = LLVMBuildOr(builder, mask, invmask, "");
2543    }
2544
2545    lp_build_mask_update(bld->mask, mask);
2546    if (!near_end_of_shader(bld, pc))
2547       lp_build_mask_check(bld->mask);
2548 }
2549
2550
2551 /**
2552  * Unconditional fragment kill.
2553  * The only predication is the execution mask which will apply if
2554  * we're inside a loop or conditional.
2555  */
2556 static void
2557 emit_kill(struct lp_build_tgsi_soa_context *bld,
2558           int pc)
2559 {
2560    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2561    LLVMValueRef mask;
2562
2563    /* For those channels which are "alive", disable fragment shader
2564     * execution.
2565     */
2566    if (bld->exec_mask.has_mask) {
2567       mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2568    }
2569    else {
2570       LLVMValueRef zero = LLVMConstNull(bld->bld_base.base.int_vec_type);
2571       mask = zero;
2572    }
2573
2574    lp_build_mask_update(bld->mask, mask);
2575
2576    if (!near_end_of_shader(bld, pc))
2577       lp_build_mask_check(bld->mask);
2578 }
2579
2580
2581 /**
2582  * Emit code which will dump the value of all the temporary registers
2583  * to stdout.
2584  */
2585 static void
2586 emit_dump_file(struct lp_build_tgsi_soa_context *bld,
2587                unsigned file)
2588 {
2589    const struct tgsi_shader_info *info = bld->bld_base.info;
2590    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2591    LLVMBuilderRef builder = gallivm->builder;
2592    LLVMValueRef reg_ptr;
2593    int index;
2594    int max_index = info->file_max[file];
2595
2596    /*
2597     * Some register files, particularly constants, can be very large,
2598     * and dumping everything could make this unusably slow.
2599     */
2600    max_index = MIN2(max_index, 32);
2601
2602    for (index = 0; index <= max_index; index++) {
2603       LLVMValueRef res;
2604       unsigned mask;
2605       int chan;
2606
2607       if (index < 8 * sizeof(unsigned) &&
2608           (info->file_mask[file] & (1 << index)) == 0)  {
2609          /* This was not declared.*/
2610          continue;
2611       }
2612
2613       if (file == TGSI_FILE_INPUT) {
2614          mask = info->input_usage_mask[index];
2615       } else {
2616          mask = TGSI_WRITEMASK_XYZW;
2617       }
2618
2619       for (chan = 0; chan < 4; chan++) {
2620          if ((mask & (1 << chan)) == 0) {
2621             /* This channel is not used.*/
2622             continue;
2623          }
2624
2625          if (file == TGSI_FILE_CONSTANT) {
2626             struct tgsi_full_src_register reg;
2627             memset(&reg, 0, sizeof reg);
2628             reg.Register.File = file;
2629             reg.Register.Index = index;
2630             reg.Register.SwizzleX = 0;
2631             reg.Register.SwizzleY = 1;
2632             reg.Register.SwizzleZ = 2;
2633             reg.Register.SwizzleW = 3;
2634
2635             res = bld->bld_base.emit_fetch_funcs[file](&bld->bld_base, &reg, TGSI_TYPE_FLOAT, chan);
2636             if (!res) {
2637                continue;
2638             }
2639          } else if (file == TGSI_FILE_INPUT) {
2640             res = bld->inputs[index][chan];
2641             if (!res) {
2642                continue;
2643             }
2644          } else if (file == TGSI_FILE_TEMPORARY) {
2645             reg_ptr = lp_get_temp_ptr_soa(bld, index, chan);
2646             assert(reg_ptr);
2647             res = LLVMBuildLoad(builder, reg_ptr, "");
2648          } else if (file == TGSI_FILE_OUTPUT) {
2649             reg_ptr = lp_get_output_ptr(bld, index, chan);
2650             assert(reg_ptr);
2651             res = LLVMBuildLoad(builder, reg_ptr, "");
2652          } else {
2653             assert(0);
2654             continue;
2655          }
2656
2657          emit_dump_reg(gallivm, file, index, chan, res);
2658       }
2659    }
2660 }
2661
2662
2663
2664 void
2665 lp_emit_declaration_soa(
2666    struct lp_build_tgsi_context *bld_base,
2667    const struct tgsi_full_declaration *decl)
2668 {
2669    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2670    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2671    LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
2672    const unsigned first = decl->Range.First;
2673    const unsigned last = decl->Range.Last;
2674    unsigned idx, i;
2675
2676    assert(last <= bld->bld_base.info->file_max[decl->Declaration.File]);
2677
2678    switch (decl->Declaration.File) {
2679    case TGSI_FILE_TEMPORARY:
2680       if (!(bld->indirect_files & (1 << TGSI_FILE_TEMPORARY))) {
2681          assert(last < LP_MAX_INLINED_TEMPS);
2682          for (idx = first; idx <= last; ++idx) {
2683             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2684                bld->temps[idx][i] = lp_build_alloca(gallivm, vec_type, "temp");
2685          }
2686       }
2687       break;
2688
2689    case TGSI_FILE_OUTPUT:
2690       if (!(bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
2691          for (idx = first; idx <= last; ++idx) {
2692             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2693                bld->outputs[idx][i] = lp_build_alloca(gallivm,
2694                                                       vec_type, "output");
2695          }
2696       }
2697       break;
2698
2699    case TGSI_FILE_ADDRESS:
2700       /* ADDR registers are only allocated with an integer LLVM IR type,
2701        * as they are guaranteed to always have integers.
2702        * XXX: Not sure if this exception is worthwhile (or the whole idea of
2703        * an ADDR register for that matter).
2704        */
2705       assert(last < LP_MAX_TGSI_ADDRS);
2706       for (idx = first; idx <= last; ++idx) {
2707          assert(idx < LP_MAX_TGSI_ADDRS);
2708          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2709             bld->addr[idx][i] = lp_build_alloca(gallivm, bld_base->base.int_vec_type, "addr");
2710       }
2711       break;
2712
2713    case TGSI_FILE_PREDICATE:
2714       assert(last < LP_MAX_TGSI_PREDS);
2715       for (idx = first; idx <= last; ++idx) {
2716          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2717             bld->preds[idx][i] = lp_build_alloca(gallivm, vec_type,
2718                                                  "predicate");
2719       }
2720       break;
2721
2722    case TGSI_FILE_SAMPLER_VIEW:
2723       /*
2724        * The target stored here MUST match whatever there actually
2725        * is in the set sampler views (what about return type?).
2726        */
2727       assert(last < PIPE_MAX_SHADER_SAMPLER_VIEWS);
2728       for (idx = first; idx <= last; ++idx) {
2729          bld->sv[idx] = decl->SamplerView;
2730       }
2731       break;
2732
2733    case TGSI_FILE_CONSTANT:
2734    {
2735       /*
2736        * We could trivially fetch the per-buffer pointer when fetching the
2737        * constant, relying on llvm to figure out it's always the same pointer
2738        * anyway. However, doing so results in a huge (more than factor of 10)
2739        * slowdown in llvm compilation times for some (but not all) shaders
2740        * (more specifically, the IR optimization spends way more time in
2741        * DominatorTree::dominates). At least with llvm versions 3.1, 3.3.
2742        */
2743       unsigned idx2D = decl->Dim.Index2D;
2744       LLVMValueRef index2D = lp_build_const_int32(gallivm, idx2D);
2745       assert(idx2D < LP_MAX_TGSI_CONST_BUFFERS);
2746       bld->consts[idx2D] =
2747          lp_build_array_get(gallivm, bld->consts_ptr, index2D);
2748       bld->consts_sizes[idx2D] =
2749          lp_build_array_get(gallivm, bld->const_sizes_ptr, index2D);
2750    }
2751       break;
2752
2753    default:
2754       /* don't need to declare other vars */
2755       break;
2756    }
2757 }
2758
2759
2760 void lp_emit_immediate_soa(
2761    struct lp_build_tgsi_context *bld_base,
2762    const struct tgsi_full_immediate *imm)
2763 {
2764    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2765    struct gallivm_state * gallivm = bld_base->base.gallivm;
2766    LLVMValueRef imms[4];
2767    unsigned i;
2768    const uint size = imm->Immediate.NrTokens - 1;
2769    assert(size <= 4);
2770    switch (imm->Immediate.DataType) {
2771    case TGSI_IMM_FLOAT32:
2772       for( i = 0; i < size; ++i )
2773          imms[i] =
2774                lp_build_const_vec(gallivm, bld_base->base.type, imm->u[i].Float);
2775
2776       break;
2777    case TGSI_IMM_UINT32:
2778       for( i = 0; i < size; ++i ) {
2779          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->uint_bld.type, imm->u[i].Uint);
2780          imms[i] = LLVMConstBitCast(tmp, bld_base->base.vec_type);
2781       }
2782
2783       break;
2784    case TGSI_IMM_INT32:
2785       for( i = 0; i < size; ++i ) {
2786          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->int_bld.type, imm->u[i].Int);
2787          imms[i] = LLVMConstBitCast(tmp, bld_base->base.vec_type);
2788       }
2789
2790       break;
2791    }
2792    for( i = size; i < 4; ++i )
2793       imms[i] = bld_base->base.undef;
2794
2795    if (bld->use_immediates_array) {
2796       unsigned index = bld->num_immediates;
2797       struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2798       LLVMBuilderRef builder = gallivm->builder;
2799
2800       assert(bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE));
2801       for (i = 0; i < 4; ++i ) {
2802          LLVMValueRef lindex = lp_build_const_int32(
2803                   bld->bld_base.base.gallivm, index * 4 + i);
2804          LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
2805                                              bld->imms_array, &lindex, 1, "");
2806          LLVMBuildStore(builder, imms[i], imm_ptr);
2807       }
2808    } else {
2809       /* simply copy the immediate values into the next immediates[] slot */
2810       unsigned i;
2811       const uint size = imm->Immediate.NrTokens - 1;
2812       assert(size <= 4);
2813       assert(bld->num_immediates < LP_MAX_INLINED_IMMEDIATES);
2814
2815       for(i = 0; i < 4; ++i )
2816          bld->immediates[bld->num_immediates][i] = imms[i];
2817
2818       if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
2819          unsigned index = bld->num_immediates;
2820          struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2821          LLVMBuilderRef builder = gallivm->builder;
2822          for (i = 0; i < 4; ++i ) {
2823             LLVMValueRef lindex = lp_build_const_int32(
2824                      bld->bld_base.base.gallivm, index * 4 + i);
2825             LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
2826                                                 bld->imms_array, &lindex, 1, "");
2827             LLVMBuildStore(builder,
2828                            bld->immediates[index][i],
2829                            imm_ptr);
2830          }
2831       }
2832    }
2833
2834    bld->num_immediates++;
2835 }
2836
2837 static void
2838 ddx_emit(
2839    const struct lp_build_tgsi_action * action,
2840    struct lp_build_tgsi_context * bld_base,
2841    struct lp_build_emit_data * emit_data)
2842 {
2843    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2844
2845    emit_fetch_deriv(bld, emit_data->args[0], NULL,
2846                     &emit_data->output[emit_data->chan], NULL);
2847 }
2848
2849 static void
2850 ddy_emit(
2851    const struct lp_build_tgsi_action * action,
2852    struct lp_build_tgsi_context * bld_base,
2853    struct lp_build_emit_data * emit_data)
2854 {
2855    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2856
2857    emit_fetch_deriv(bld, emit_data->args[0], NULL, NULL,
2858                     &emit_data->output[emit_data->chan]);
2859 }
2860
2861 static void
2862 kill_emit(
2863    const struct lp_build_tgsi_action * action,
2864    struct lp_build_tgsi_context * bld_base,
2865    struct lp_build_emit_data * emit_data)
2866 {
2867    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2868
2869    emit_kill(bld, bld_base->pc - 1);
2870 }
2871
2872 static void
2873 kill_if_emit(
2874    const struct lp_build_tgsi_action * action,
2875    struct lp_build_tgsi_context * bld_base,
2876    struct lp_build_emit_data * emit_data)
2877 {
2878    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2879
2880    emit_kill_if(bld, emit_data->inst, bld_base->pc - 1);
2881 }
2882
2883 static void
2884 tex_emit(
2885    const struct lp_build_tgsi_action * action,
2886    struct lp_build_tgsi_context * bld_base,
2887    struct lp_build_emit_data * emit_data)
2888 {
2889    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2890
2891    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE, emit_data->output);
2892 }
2893
2894 static void
2895 txb_emit(
2896    const struct lp_build_tgsi_action * action,
2897    struct lp_build_tgsi_context * bld_base,
2898    struct lp_build_emit_data * emit_data)
2899 {
2900    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2901
2902    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
2903             emit_data->output);
2904 }
2905
2906 static void
2907 txd_emit(
2908    const struct lp_build_tgsi_action * action,
2909    struct lp_build_tgsi_context * bld_base,
2910    struct lp_build_emit_data * emit_data)
2911 {
2912    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2913
2914    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
2915             emit_data->output);
2916 }
2917
2918 static void
2919 txl_emit(
2920    const struct lp_build_tgsi_action * action,
2921    struct lp_build_tgsi_context * bld_base,
2922    struct lp_build_emit_data * emit_data)
2923 {
2924    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2925
2926    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
2927             emit_data->output);
2928 }
2929
2930 static void
2931 txp_emit(
2932    const struct lp_build_tgsi_action * action,
2933    struct lp_build_tgsi_context * bld_base,
2934    struct lp_build_emit_data * emit_data)
2935 {
2936    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2937
2938    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_PROJECTED,
2939             emit_data->output);
2940 }
2941
2942 static void
2943 txq_emit(
2944    const struct lp_build_tgsi_action * action,
2945    struct lp_build_tgsi_context * bld_base,
2946    struct lp_build_emit_data * emit_data)
2947 {
2948    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2949
2950    emit_size_query(bld, emit_data->inst, emit_data->output, FALSE);
2951 }
2952
2953 static void
2954 txf_emit(
2955    const struct lp_build_tgsi_action * action,
2956    struct lp_build_tgsi_context * bld_base,
2957    struct lp_build_emit_data * emit_data)
2958 {
2959    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2960
2961    emit_fetch_texels(bld, emit_data->inst, emit_data->output, FALSE);
2962 }
2963
2964 static void
2965 sample_i_emit(
2966    const struct lp_build_tgsi_action * action,
2967    struct lp_build_tgsi_context * bld_base,
2968    struct lp_build_emit_data * emit_data)
2969 {
2970    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2971
2972    emit_fetch_texels(bld, emit_data->inst, emit_data->output, TRUE);
2973 }
2974
2975 static void
2976 sample_emit(
2977    const struct lp_build_tgsi_action * action,
2978    struct lp_build_tgsi_context * bld_base,
2979    struct lp_build_emit_data * emit_data)
2980 {
2981    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2982
2983    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
2984                FALSE, emit_data->output);
2985 }
2986
2987 static void
2988 sample_b_emit(
2989    const struct lp_build_tgsi_action * action,
2990    struct lp_build_tgsi_context * bld_base,
2991    struct lp_build_emit_data * emit_data)
2992 {
2993    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2994
2995    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
2996                FALSE, emit_data->output);
2997 }
2998
2999 static void
3000 sample_c_emit(
3001    const struct lp_build_tgsi_action * action,
3002    struct lp_build_tgsi_context * bld_base,
3003    struct lp_build_emit_data * emit_data)
3004 {
3005    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3006
3007    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
3008                TRUE, emit_data->output);
3009 }
3010
3011 static void
3012 sample_c_lz_emit(
3013    const struct lp_build_tgsi_action * action,
3014    struct lp_build_tgsi_context * bld_base,
3015    struct lp_build_emit_data * emit_data)
3016 {
3017    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3018
3019    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_ZERO,
3020                TRUE, emit_data->output);
3021 }
3022
3023 static void
3024 sample_d_emit(
3025    const struct lp_build_tgsi_action * action,
3026    struct lp_build_tgsi_context * bld_base,
3027    struct lp_build_emit_data * emit_data)
3028 {
3029    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3030
3031    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
3032                FALSE, emit_data->output);
3033 }
3034
3035 static void
3036 sample_l_emit(
3037    const struct lp_build_tgsi_action * action,
3038    struct lp_build_tgsi_context * bld_base,
3039    struct lp_build_emit_data * emit_data)
3040 {
3041    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3042
3043    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
3044                FALSE, emit_data->output);
3045 }
3046
3047 static void
3048 sviewinfo_emit(
3049    const struct lp_build_tgsi_action * action,
3050    struct lp_build_tgsi_context * bld_base,
3051    struct lp_build_emit_data * emit_data)
3052 {
3053    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3054
3055    emit_size_query(bld, emit_data->inst, emit_data->output, TRUE);
3056 }
3057
3058 static LLVMValueRef
3059 mask_vec(struct lp_build_tgsi_context *bld_base)
3060 {
3061    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3062    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3063    struct lp_exec_mask *exec_mask = &bld->exec_mask;
3064
3065    if (!exec_mask->has_mask) {
3066       return lp_build_mask_value(bld->mask);
3067    }
3068    return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
3069                        exec_mask->exec_mask, "");
3070 }
3071
3072 static void
3073 increment_vec_ptr_by_mask(struct lp_build_tgsi_context * bld_base,
3074                           LLVMValueRef ptr,
3075                           LLVMValueRef mask)
3076 {
3077    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3078    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3079
3080    current_vec = LLVMBuildSub(builder, current_vec, mask, "");
3081
3082    LLVMBuildStore(builder, current_vec, ptr);
3083 }
3084
3085 static void
3086 clear_uint_vec_ptr_from_mask(struct lp_build_tgsi_context * bld_base,
3087                              LLVMValueRef ptr,
3088                              LLVMValueRef mask)
3089 {
3090    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3091    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3092
3093    current_vec = lp_build_select(&bld_base->uint_bld,
3094                                  mask,
3095                                  bld_base->uint_bld.zero,
3096                                  current_vec);
3097
3098    LLVMBuildStore(builder, current_vec, ptr);
3099 }
3100
3101 static LLVMValueRef
3102 clamp_mask_to_max_output_vertices(struct lp_build_tgsi_soa_context * bld,
3103                                   LLVMValueRef current_mask_vec,
3104                                   LLVMValueRef total_emitted_vertices_vec)
3105 {
3106    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3107    struct lp_build_context *int_bld = &bld->bld_base.int_bld;
3108    LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
3109                                         total_emitted_vertices_vec,
3110                                         bld->max_output_vertices_vec);
3111
3112    return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
3113 }
3114
3115 static void
3116 emit_vertex(
3117    const struct lp_build_tgsi_action * action,
3118    struct lp_build_tgsi_context * bld_base,
3119    struct lp_build_emit_data * emit_data)
3120 {
3121    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3122    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3123
3124    if (bld->gs_iface->emit_vertex) {
3125       LLVMValueRef mask = mask_vec(bld_base);
3126       LLVMValueRef total_emitted_vertices_vec =
3127          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3128       mask = clamp_mask_to_max_output_vertices(bld, mask,
3129                                                total_emitted_vertices_vec);
3130       gather_outputs(bld);
3131       bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base,
3132                                  bld->outputs,
3133                                  total_emitted_vertices_vec);
3134       increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr,
3135                                 mask);
3136       increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr,
3137                                 mask);
3138 #if DUMP_GS_EMITS
3139       lp_build_print_value(bld->bld_base.base.gallivm,
3140                            " +++ emit vertex masked ones = ",
3141                            mask);
3142       lp_build_print_value(bld->bld_base.base.gallivm,
3143                            " +++ emit vertex emitted = ",
3144                            total_emitted_vertices_vec);
3145 #endif
3146    }
3147 }
3148
3149
3150 static void
3151 end_primitive_masked(struct lp_build_tgsi_context * bld_base,
3152                      LLVMValueRef mask)
3153 {
3154    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3155    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3156
3157    if (bld->gs_iface->end_primitive) {
3158       struct lp_build_context *uint_bld = &bld_base->uint_bld;
3159       LLVMValueRef emitted_vertices_vec =
3160          LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr, "");
3161       LLVMValueRef emitted_prims_vec =
3162          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3163
3164       LLVMValueRef emitted_mask = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3165                                                emitted_vertices_vec,
3166                                                uint_bld->zero);
3167       /* We need to combine the current execution mask with the mask
3168          telling us which, if any, execution slots actually have
3169          unemitted primitives, this way we make sure that end_primitives
3170          executes only on the paths that have unflushed vertices */
3171       mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
3172
3173       bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base,
3174                                    emitted_vertices_vec,
3175                                    emitted_prims_vec);
3176
3177 #if DUMP_GS_EMITS
3178       lp_build_print_value(bld->bld_base.base.gallivm,
3179                            " +++ end prim masked ones = ",
3180                            mask);
3181       lp_build_print_value(bld->bld_base.base.gallivm,
3182                            " +++ end prim emitted verts1 = ",
3183                            emitted_vertices_vec);
3184       lp_build_print_value(bld->bld_base.base.gallivm,
3185                            " +++ end prim emitted prims1 = ",
3186                            LLVMBuildLoad(builder,
3187                                          bld->emitted_prims_vec_ptr, ""));
3188 #endif
3189       increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr,
3190                                 mask);
3191       clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr,
3192                                    mask);
3193 #if DUMP_GS_EMITS
3194       lp_build_print_value(bld->bld_base.base.gallivm,
3195                            " +++ end prim emitted verts2 = ",
3196                            LLVMBuildLoad(builder,
3197                                          bld->emitted_vertices_vec_ptr, ""));
3198 #endif
3199    }
3200
3201 }
3202
3203 static void
3204 end_primitive(
3205    const struct lp_build_tgsi_action * action,
3206    struct lp_build_tgsi_context * bld_base,
3207    struct lp_build_emit_data * emit_data)
3208 {
3209    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3210
3211    if (bld->gs_iface->end_primitive) {
3212       LLVMValueRef mask = mask_vec(bld_base);
3213       end_primitive_masked(bld_base, mask);
3214    }
3215 }
3216
3217 static void
3218 cal_emit(
3219    const struct lp_build_tgsi_action * action,
3220    struct lp_build_tgsi_context * bld_base,
3221    struct lp_build_emit_data * emit_data)
3222 {
3223    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3224
3225    lp_exec_mask_call(&bld->exec_mask, emit_data->inst->Label.Label,
3226                      &bld_base->pc);
3227 }
3228
3229 static void
3230 ret_emit(
3231    const struct lp_build_tgsi_action * action,
3232    struct lp_build_tgsi_context * bld_base,
3233    struct lp_build_emit_data * emit_data)
3234 {
3235    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3236
3237    lp_exec_mask_ret(&bld->exec_mask, &bld_base->pc);
3238 }
3239
3240 static void
3241 brk_emit(
3242    const struct lp_build_tgsi_action * action,
3243    struct lp_build_tgsi_context * bld_base,
3244    struct lp_build_emit_data * emit_data)
3245 {
3246    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3247
3248    lp_exec_break(&bld->exec_mask, bld_base);
3249 }
3250
3251 static void
3252 breakc_emit(
3253    const struct lp_build_tgsi_action * action,
3254    struct lp_build_tgsi_context * bld_base,
3255    struct lp_build_emit_data * emit_data)
3256 {
3257    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3258    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3259    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3260    LLVMValueRef unsigned_cond =
3261       LLVMBuildBitCast(builder, emit_data->args[0], uint_bld->vec_type, "");
3262    LLVMValueRef cond = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3263                                     unsigned_cond,
3264                                     uint_bld->zero);
3265
3266    lp_exec_break_condition(&bld->exec_mask, cond);
3267 }
3268
3269 static void
3270 if_emit(
3271    const struct lp_build_tgsi_action * action,
3272    struct lp_build_tgsi_context * bld_base,
3273    struct lp_build_emit_data * emit_data)
3274 {
3275    LLVMValueRef tmp;
3276    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3277
3278    tmp = lp_build_cmp(&bld_base->base, PIPE_FUNC_NOTEQUAL,
3279                       emit_data->args[0], bld->bld_base.base.zero);
3280    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3281 }
3282
3283 static void
3284 uif_emit(
3285    const struct lp_build_tgsi_action * action,
3286    struct lp_build_tgsi_context * bld_base,
3287    struct lp_build_emit_data * emit_data)
3288 {
3289    LLVMValueRef tmp;
3290    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3291    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3292
3293    tmp = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3294                       emit_data->args[0], uint_bld->zero);
3295    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3296 }
3297
3298 static void
3299 case_emit(
3300    const struct lp_build_tgsi_action * action,
3301    struct lp_build_tgsi_context * bld_base,
3302    struct lp_build_emit_data * emit_data)
3303 {
3304    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3305
3306    lp_exec_case(&bld->exec_mask, emit_data->args[0]);
3307 }
3308
3309 static void
3310 default_emit(
3311    const struct lp_build_tgsi_action * action,
3312    struct lp_build_tgsi_context * bld_base,
3313    struct lp_build_emit_data * emit_data)
3314 {
3315    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3316
3317    lp_exec_default(&bld->exec_mask, bld_base);
3318 }
3319
3320 static void
3321 switch_emit(
3322    const struct lp_build_tgsi_action * action,
3323    struct lp_build_tgsi_context * bld_base,
3324    struct lp_build_emit_data * emit_data)
3325 {
3326    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3327
3328    lp_exec_switch(&bld->exec_mask, emit_data->args[0]);
3329 }
3330
3331 static void
3332 endswitch_emit(
3333    const struct lp_build_tgsi_action * action,
3334    struct lp_build_tgsi_context * bld_base,
3335    struct lp_build_emit_data * emit_data)
3336 {
3337    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3338
3339    lp_exec_endswitch(&bld->exec_mask, bld_base);
3340 }
3341
3342 static void
3343 bgnloop_emit(
3344    const struct lp_build_tgsi_action * action,
3345    struct lp_build_tgsi_context * bld_base,
3346    struct lp_build_emit_data * emit_data)
3347 {
3348    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3349
3350    lp_exec_bgnloop(&bld->exec_mask);
3351 }
3352
3353 static void
3354 bgnsub_emit(
3355    const struct lp_build_tgsi_action * action,
3356    struct lp_build_tgsi_context * bld_base,
3357    struct lp_build_emit_data * emit_data)
3358 {
3359    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3360
3361    lp_exec_mask_bgnsub(&bld->exec_mask);
3362 }
3363
3364 static void
3365 else_emit(
3366    const struct lp_build_tgsi_action * action,
3367    struct lp_build_tgsi_context * bld_base,
3368    struct lp_build_emit_data * emit_data)
3369 {
3370    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3371
3372    lp_exec_mask_cond_invert(&bld->exec_mask);
3373 }
3374
3375 static void
3376 endif_emit(
3377    const struct lp_build_tgsi_action * action,
3378    struct lp_build_tgsi_context * bld_base,
3379    struct lp_build_emit_data * emit_data)
3380 {
3381    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3382
3383    lp_exec_mask_cond_pop(&bld->exec_mask);
3384 }
3385
3386 static void
3387 endloop_emit(
3388    const struct lp_build_tgsi_action * action,
3389    struct lp_build_tgsi_context * bld_base,
3390    struct lp_build_emit_data * emit_data)
3391 {
3392    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3393
3394    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
3395 }
3396
3397 static void
3398 endsub_emit(
3399    const struct lp_build_tgsi_action * action,
3400    struct lp_build_tgsi_context * bld_base,
3401    struct lp_build_emit_data * emit_data)
3402 {
3403    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3404
3405    lp_exec_mask_endsub(&bld->exec_mask, &bld_base->pc);
3406 }
3407
3408 static void
3409 cont_emit(
3410    const struct lp_build_tgsi_action * action,
3411    struct lp_build_tgsi_context * bld_base,
3412    struct lp_build_emit_data * emit_data)
3413 {
3414    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3415
3416    lp_exec_continue(&bld->exec_mask);
3417 }
3418
3419 /* XXX: Refactor and move it to lp_bld_tgsi_action.c
3420  *
3421  * XXX: What do the comments about xmm registers mean?  Maybe they are left over
3422  * from old code, but there is no garauntee that LLVM will use those registers
3423  * for this code.
3424  *
3425  * XXX: There should be no calls to lp_build_emit_fetch in this function.  This
3426  * should be handled by the emit_data->fetch_args function. */
3427 static void
3428 nrm_emit(
3429    const struct lp_build_tgsi_action * action,
3430    struct lp_build_tgsi_context * bld_base,
3431    struct lp_build_emit_data * emit_data)
3432 {
3433    LLVMValueRef tmp0, tmp1;
3434    LLVMValueRef tmp4 = NULL;
3435    LLVMValueRef tmp5 = NULL;
3436    LLVMValueRef tmp6 = NULL;
3437    LLVMValueRef tmp7 = NULL;
3438    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3439
3440    uint dims = (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_NRM) ? 3 : 4;
3441
3442   if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X) ||
3443       TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Y) ||
3444       TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Z) ||
3445       (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_W) && dims == 4)) {
3446
3447       /* NOTE: Cannot use xmm regs 2/3 here (see emit_rsqrt() above). */
3448
3449       /* xmm4 = src.x */
3450       /* xmm0 = src.x * src.x */
3451       tmp0 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_X);
3452       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X)) {
3453          tmp4 = tmp0;
3454       }
3455       tmp0 = lp_build_mul( &bld->bld_base.base, tmp0, tmp0);
3456
3457       /* xmm5 = src.y */
3458       /* xmm0 = xmm0 + src.y * src.y */
3459       tmp1 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_Y);
3460       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Y)) {
3461          tmp5 = tmp1;
3462       }
3463       tmp1 = lp_build_mul( &bld->bld_base.base, tmp1, tmp1);
3464       tmp0 = lp_build_add( &bld->bld_base.base, tmp0, tmp1);
3465
3466       /* xmm6 = src.z */
3467       /* xmm0 = xmm0 + src.z * src.z */
3468       tmp1 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_Z);
3469       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Z)) {
3470          tmp6 = tmp1;
3471       }
3472       tmp1 = lp_build_mul( &bld->bld_base.base, tmp1, tmp1);
3473       tmp0 = lp_build_add( &bld->bld_base.base, tmp0, tmp1);
3474
3475       if (dims == 4) {
3476          /* xmm7 = src.w */
3477          /* xmm0 = xmm0 + src.w * src.w */
3478          tmp1 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_W);
3479          if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_W)) {
3480             tmp7 = tmp1;
3481          }
3482          tmp1 = lp_build_mul( &bld->bld_base.base, tmp1, tmp1);
3483          tmp0 = lp_build_add( &bld->bld_base.base, tmp0, tmp1);
3484       }
3485       /* xmm1 = 1 / sqrt(xmm0) */
3486       tmp1 = lp_build_rsqrt( &bld->bld_base.base, tmp0);
3487        /* dst.x = xmm1 * src.x */
3488       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X)) {
3489          emit_data->output[TGSI_CHAN_X] = lp_build_mul( &bld->bld_base.base, tmp4, tmp1);
3490       }
3491       /* dst.y = xmm1 * src.y */
3492       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Y)) {
3493          emit_data->output[TGSI_CHAN_Y] = lp_build_mul( &bld->bld_base.base, tmp5, tmp1);
3494       }
3495
3496       /* dst.z = xmm1 * src.z */
3497       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Z)) {
3498          emit_data->output[TGSI_CHAN_Z] = lp_build_mul( &bld->bld_base.base, tmp6, tmp1);
3499       }
3500       /* dst.w = xmm1 * src.w */
3501       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X) && dims == 4) {
3502          emit_data->output[TGSI_CHAN_W] = lp_build_mul( &bld->bld_base.base, tmp7, tmp1);
3503       }
3504    }
3505
3506    /* dst.w = 1.0 */
3507    if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_W) && dims == 3) {
3508        emit_data->output[TGSI_CHAN_W] = bld->bld_base.base.one;
3509    }
3510 }
3511
3512 static void emit_prologue(struct lp_build_tgsi_context * bld_base)
3513 {
3514    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3515    struct gallivm_state * gallivm = bld_base->base.gallivm;
3516
3517    if (bld->indirect_files & (1 << TGSI_FILE_TEMPORARY)) {
3518       LLVMValueRef array_size =
3519          lp_build_const_int32(gallivm,
3520                          bld_base->info->file_max[TGSI_FILE_TEMPORARY] * 4 + 4);
3521       bld->temps_array = lp_build_array_alloca(gallivm,
3522                                               bld_base->base.vec_type, array_size,
3523                                               "temp_array");
3524    }
3525
3526    if (bld->indirect_files & (1 << TGSI_FILE_OUTPUT)) {
3527       LLVMValueRef array_size =
3528          lp_build_const_int32(gallivm,
3529                             bld_base->info->file_max[TGSI_FILE_OUTPUT] * 4 + 4);
3530       bld->outputs_array = lp_build_array_alloca(gallivm,
3531                                                 bld_base->base.vec_type, array_size,
3532                                                 "output_array");
3533    }
3534
3535    if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
3536       LLVMValueRef array_size =
3537          lp_build_const_int32(gallivm,
3538                          bld_base->info->file_max[TGSI_FILE_IMMEDIATE] * 4 + 4);
3539       bld->imms_array = lp_build_array_alloca(gallivm,
3540                                               bld_base->base.vec_type, array_size,
3541                                               "imms_array");
3542    }
3543
3544    /* If we have indirect addressing in inputs we need to copy them into
3545     * our alloca array to be able to iterate over them */
3546    if (bld->indirect_files & (1 << TGSI_FILE_INPUT) && !bld->gs_iface) {
3547       unsigned index, chan;
3548       LLVMTypeRef vec_type = bld_base->base.vec_type;
3549       LLVMValueRef array_size = lp_build_const_int32(gallivm,
3550             bld_base->info->file_max[TGSI_FILE_INPUT]*4 + 4);
3551       bld->inputs_array = lp_build_array_alloca(gallivm,
3552                                                vec_type, array_size,
3553                                                "input_array");
3554
3555       assert(bld_base->info->num_inputs
3556                         <= bld_base->info->file_max[TGSI_FILE_INPUT] + 1);
3557
3558       for (index = 0; index < bld_base->info->num_inputs; ++index) {
3559          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3560             LLVMValueRef lindex =
3561                lp_build_const_int32(gallivm, index * 4 + chan);
3562             LLVMValueRef input_ptr =
3563                LLVMBuildGEP(gallivm->builder, bld->inputs_array,
3564                             &lindex, 1, "");
3565             LLVMValueRef value = bld->inputs[index][chan];
3566             if (value)
3567                LLVMBuildStore(gallivm->builder, value, input_ptr);
3568          }
3569       }
3570    }
3571
3572    if (bld->gs_iface) {
3573       struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
3574       bld->emitted_prims_vec_ptr =
3575          lp_build_alloca(gallivm,
3576                          uint_bld->vec_type,
3577                          "emitted_prims_ptr");
3578       bld->emitted_vertices_vec_ptr =
3579          lp_build_alloca(gallivm,
3580                          uint_bld->vec_type,
3581                          "emitted_vertices_ptr");
3582       bld->total_emitted_vertices_vec_ptr =
3583          lp_build_alloca(gallivm,
3584                          uint_bld->vec_type,
3585                          "total_emitted_vertices_ptr");
3586
3587       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3588                      bld->emitted_prims_vec_ptr);
3589       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3590                      bld->emitted_vertices_vec_ptr);
3591       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3592                      bld->total_emitted_vertices_vec_ptr);
3593    }
3594
3595    if (DEBUG_EXECUTION) {
3596       lp_build_printf(gallivm, "\n");
3597       emit_dump_file(bld, TGSI_FILE_CONSTANT);
3598       if (!bld->gs_iface)
3599          emit_dump_file(bld, TGSI_FILE_INPUT);
3600    }
3601 }
3602
3603 static void emit_epilogue(struct lp_build_tgsi_context * bld_base)
3604 {
3605    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3606    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3607
3608    if (DEBUG_EXECUTION) {
3609       /* for debugging */
3610       if (0) {
3611          emit_dump_file(bld, TGSI_FILE_TEMPORARY);
3612       }
3613       emit_dump_file(bld, TGSI_FILE_OUTPUT);
3614       lp_build_printf(bld_base->base.gallivm, "\n");
3615    }
3616
3617    /* If we have indirect addressing in outputs we need to copy our alloca array
3618     * to the outputs slots specified by the caller */
3619    if (bld->gs_iface) {
3620       LLVMValueRef total_emitted_vertices_vec;
3621       LLVMValueRef emitted_prims_vec;
3622       /* implicit end_primitives, needed in case there are any unflushed
3623          vertices in the cache. Note must not call end_primitive here
3624          since the exec_mask is not valid at this point. */
3625       end_primitive_masked(bld_base, lp_build_mask_value(bld->mask));
3626
3627       total_emitted_vertices_vec =
3628          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3629       emitted_prims_vec =
3630          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3631
3632       bld->gs_iface->gs_epilogue(bld->gs_iface,
3633                                  &bld->bld_base,
3634                                  total_emitted_vertices_vec,
3635                                  emitted_prims_vec);
3636    } else {
3637       gather_outputs(bld);
3638    }
3639 }
3640
3641 void
3642 lp_build_tgsi_soa(struct gallivm_state *gallivm,
3643                   const struct tgsi_token *tokens,
3644                   struct lp_type type,
3645                   struct lp_build_mask_context *mask,
3646                   LLVMValueRef consts_ptr,
3647                   LLVMValueRef const_sizes_ptr,
3648                   const struct lp_bld_tgsi_system_values *system_values,
3649                   const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
3650                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
3651                   struct lp_build_sampler_soa *sampler,
3652                   const struct tgsi_shader_info *info,
3653                   const struct lp_build_tgsi_gs_iface *gs_iface)
3654 {
3655    struct lp_build_tgsi_soa_context bld;
3656
3657    struct lp_type res_type;
3658
3659    assert(type.length <= LP_MAX_VECTOR_LENGTH);
3660    memset(&res_type, 0, sizeof res_type);
3661    res_type.width = type.width;
3662    res_type.length = type.length;
3663    res_type.sign = 1;
3664
3665    /* Setup build context */
3666    memset(&bld, 0, sizeof bld);
3667    lp_build_context_init(&bld.bld_base.base, gallivm, type);
3668    lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
3669    lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
3670    lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
3671    bld.mask = mask;
3672    bld.inputs = inputs;
3673    bld.outputs = outputs;
3674    bld.consts_ptr = consts_ptr;
3675    bld.const_sizes_ptr = const_sizes_ptr;
3676    bld.sampler = sampler;
3677    bld.bld_base.info = info;
3678    bld.indirect_files = info->indirect_files;
3679
3680    /*
3681     * If the number of temporaries is rather large then we just
3682     * allocate them as an array right from the start and treat
3683     * like indirect temporaries.
3684     */
3685    if (info->file_max[TGSI_FILE_TEMPORARY] >= LP_MAX_INLINED_TEMPS) {
3686       bld.indirect_files |= (1 << TGSI_FILE_TEMPORARY);
3687    }
3688    /*
3689     * For performance reason immediates are always backed in a static
3690     * array, but if their number is too great, we have to use just
3691     * a dynamically allocated array.
3692     */
3693    bld.use_immediates_array =
3694          (info->file_max[TGSI_FILE_IMMEDIATE] >= LP_MAX_INLINED_IMMEDIATES);
3695    if (bld.use_immediates_array) {
3696       bld.indirect_files |= (1 << TGSI_FILE_IMMEDIATE);
3697    }
3698
3699
3700    bld.bld_base.soa = TRUE;
3701    bld.bld_base.emit_debug = emit_debug;
3702    bld.bld_base.emit_fetch_funcs[TGSI_FILE_CONSTANT] = emit_fetch_constant;
3703    bld.bld_base.emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = emit_fetch_immediate;
3704    bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_input;
3705    bld.bld_base.emit_fetch_funcs[TGSI_FILE_TEMPORARY] = emit_fetch_temporary;
3706    bld.bld_base.emit_fetch_funcs[TGSI_FILE_SYSTEM_VALUE] = emit_fetch_system_value;
3707    bld.bld_base.emit_store = emit_store;
3708
3709    bld.bld_base.emit_declaration = lp_emit_declaration_soa;
3710    bld.bld_base.emit_immediate = lp_emit_immediate_soa;
3711
3712    bld.bld_base.emit_prologue = emit_prologue;
3713    bld.bld_base.emit_epilogue = emit_epilogue;
3714
3715    /* Set opcode actions */
3716    lp_set_default_actions_cpu(&bld.bld_base);
3717
3718    bld.bld_base.op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit;
3719    bld.bld_base.op_actions[TGSI_OPCODE_BGNSUB].emit = bgnsub_emit;
3720    bld.bld_base.op_actions[TGSI_OPCODE_BRK].emit = brk_emit;
3721    bld.bld_base.op_actions[TGSI_OPCODE_BREAKC].emit = breakc_emit;
3722    bld.bld_base.op_actions[TGSI_OPCODE_CAL].emit = cal_emit;
3723    bld.bld_base.op_actions[TGSI_OPCODE_CASE].emit = case_emit;
3724    bld.bld_base.op_actions[TGSI_OPCODE_CONT].emit = cont_emit;
3725    bld.bld_base.op_actions[TGSI_OPCODE_DDX].emit = ddx_emit;
3726    bld.bld_base.op_actions[TGSI_OPCODE_DDY].emit = ddy_emit;
3727    bld.bld_base.op_actions[TGSI_OPCODE_DEFAULT].emit = default_emit;
3728    bld.bld_base.op_actions[TGSI_OPCODE_ELSE].emit = else_emit;
3729    bld.bld_base.op_actions[TGSI_OPCODE_ENDIF].emit = endif_emit;
3730    bld.bld_base.op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit;
3731    bld.bld_base.op_actions[TGSI_OPCODE_ENDSUB].emit = endsub_emit;
3732    bld.bld_base.op_actions[TGSI_OPCODE_ENDSWITCH].emit = endswitch_emit;
3733    bld.bld_base.op_actions[TGSI_OPCODE_IF].emit = if_emit;
3734    bld.bld_base.op_actions[TGSI_OPCODE_UIF].emit = uif_emit;
3735    bld.bld_base.op_actions[TGSI_OPCODE_KILL_IF].emit = kill_if_emit;
3736    bld.bld_base.op_actions[TGSI_OPCODE_KILL].emit = kill_emit;
3737    bld.bld_base.op_actions[TGSI_OPCODE_NRM].emit = nrm_emit;
3738    bld.bld_base.op_actions[TGSI_OPCODE_NRM4].emit = nrm_emit;
3739    bld.bld_base.op_actions[TGSI_OPCODE_RET].emit = ret_emit;
3740    bld.bld_base.op_actions[TGSI_OPCODE_SWITCH].emit = switch_emit;
3741    bld.bld_base.op_actions[TGSI_OPCODE_TEX].emit = tex_emit;
3742    bld.bld_base.op_actions[TGSI_OPCODE_TXB].emit = txb_emit;
3743    bld.bld_base.op_actions[TGSI_OPCODE_TXD].emit = txd_emit;
3744    bld.bld_base.op_actions[TGSI_OPCODE_TXL].emit = txl_emit;
3745    bld.bld_base.op_actions[TGSI_OPCODE_TXP].emit = txp_emit;
3746    bld.bld_base.op_actions[TGSI_OPCODE_TXQ].emit = txq_emit;
3747    bld.bld_base.op_actions[TGSI_OPCODE_TXF].emit = txf_emit;
3748    /* DX10 sampling ops */
3749    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE].emit = sample_emit;
3750    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_B].emit = sample_b_emit;
3751    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C].emit = sample_c_emit;
3752    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C_LZ].emit = sample_c_lz_emit;
3753    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_D].emit = sample_d_emit;
3754    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_I].emit = sample_i_emit;
3755    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_L].emit = sample_l_emit;
3756    bld.bld_base.op_actions[TGSI_OPCODE_SVIEWINFO].emit = sviewinfo_emit;
3757
3758    if (gs_iface) {
3759       /* There's no specific value for this because it should always
3760        * be set, but apps using ext_geometry_shader4 quite often
3761        * were forgetting so we're using MAX_VERTEX_VARYING from
3762        * that spec even though we could debug_assert if it's not
3763        * set, but that's a lot uglier. */
3764       uint max_output_vertices = 32;
3765       uint i = 0;
3766       /* inputs are always indirect with gs */
3767       bld.indirect_files |= (1 << TGSI_FILE_INPUT);
3768       bld.gs_iface = gs_iface;
3769       bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_gs_input;
3770       bld.bld_base.op_actions[TGSI_OPCODE_EMIT].emit = emit_vertex;
3771       bld.bld_base.op_actions[TGSI_OPCODE_ENDPRIM].emit = end_primitive;
3772
3773       for (i = 0; i < info->num_properties; ++i) {
3774          if (info->properties[i].name ==
3775              TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES) {
3776             max_output_vertices = info->properties[i].data[0];
3777          }
3778       }
3779       bld.max_output_vertices_vec =
3780          lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
3781                                 max_output_vertices);
3782    }
3783
3784    lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
3785
3786    bld.system_values = *system_values;
3787
3788    lp_build_tgsi_llvm(&bld.bld_base, tokens);
3789
3790    if (0) {
3791       LLVMBasicBlockRef block = LLVMGetInsertBlock(gallivm->builder);
3792       LLVMValueRef function = LLVMGetBasicBlockParent(block);
3793       debug_printf("11111111111111111111111111111 \n");
3794       tgsi_dump(tokens, 0);
3795       lp_debug_dump_value(function);
3796       debug_printf("2222222222222222222222222222 \n");
3797    }
3798
3799    if (0) {
3800       LLVMModuleRef module = LLVMGetGlobalParent(
3801          LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
3802       LLVMDumpModule(module);
3803
3804    }
3805    lp_exec_mask_fini(&bld.exec_mask);
3806 }