src/gallium/auxiliary/gallivm/lp_bld_tgsi_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * Copyright 2007-2008 VMware, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28
  29 /**
  30  * @file
  31  * TGSI to LLVM IR translation -- SoA.
  32  *
  33  * @author Jose Fonseca <jfonseca@vmware.com>
  34  *
  35  * Based on tgsi_sse2.c code written by Michal Krol, Keith Whitwell,
  36  * Brian Paul, and others.
  37  */
  38
  39 #include "pipe/p_config.h"
  40 #include "pipe/p_shader_tokens.h"
  41 #include "util/u_debug.h"
  42 #include "util/u_math.h"
  43 #include "util/u_memory.h"
  44 #include "tgsi/tgsi_dump.h"
  45 #include "tgsi/tgsi_exec.h"
  46 #include "tgsi/tgsi_info.h"
  47 #include "tgsi/tgsi_parse.h"
  48 #include "tgsi/tgsi_util.h"
  49 #include "tgsi/tgsi_scan.h"
  50 #include "tgsi/tgsi_strings.h"
  51 #include "lp_bld_tgsi_action.h"
  52 #include "lp_bld_type.h"
  53 #include "lp_bld_const.h"
  54 #include "lp_bld_arit.h"
  55 #include "lp_bld_bitarit.h"
  56 #include "lp_bld_gather.h"
  57 #include "lp_bld_init.h"
  58 #include "lp_bld_logic.h"
  59 #include "lp_bld_swizzle.h"
  60 #include "lp_bld_flow.h"
  61 #include "lp_bld_quad.h"
  62 #include "lp_bld_tgsi.h"
  63 #include "lp_bld_limits.h"
  64 #include "lp_bld_debug.h"
  65 #include "lp_bld_printf.h"
  66 #include "lp_bld_sample.h"
  67 #include "lp_bld_struct.h"
  68
  69 /* SM 4.0 says that subroutines can nest 32 deep and
  70  * we need one more for our main function */
  71 #define LP_MAX_NUM_FUNCS 33
  72
  73 #define DUMP_GS_EMITS 0
  74
  75 /*
  76  * If non-zero, the generated LLVM IR will print intermediate results on every TGSI
  77  * instruction.
  78  *
  79  * TODO:
  80  * - take execution masks in consideration
  81  * - debug control-flow instructions
  82  */
  83 #define DEBUG_EXECUTION 0
  84
  85
  86 /*
  87  * Emit code to print a register value.
  88  */
  89 static void
  90 emit_dump_reg(struct gallivm_state *gallivm,
  91               unsigned file,
  92               unsigned index,
  93               unsigned chan,
  94               LLVMValueRef value)
  95 {
  96    char buf[32];
  97
  98    util_snprintf(buf, sizeof buf, "    %s[%u].%c = ",
  99                  tgsi_file_name(file),
 100                  index, "xyzw"[chan]);
 101
 102    lp_build_print_value(gallivm, buf, value);
 103 }
 104
 105 /*
 106  * Return the context for the current function.
 107  * (always 'main', if shader doesn't do any function calls)
 108  */
 109 static INLINE struct function_ctx *
 110 func_ctx(struct lp_exec_mask *mask)
 111 {
 112    assert(mask->function_stack_size > 0);
 113    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 114    return &mask->function_stack[mask->function_stack_size - 1];
 115 }
 116
 117 /*
 118  * Returns true if we're in a loop.
 119  * It's global, meaning that it returns true even if there's
 120  * no loop inside the current function, but we were inside
 121  * a loop inside another function, from which this one was called.
 122  */
 123 static INLINE boolean
 124 mask_has_loop(struct lp_exec_mask *mask)
 125 {
 126    int i;
 127    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 128       const struct function_ctx *ctx = &mask->function_stack[i];
 129       if (ctx->loop_stack_size > 0)
 130          return TRUE;
 131    }
 132    return FALSE;
 133 }
 134
 135 /*
 136  * Returns true if we're inside a switch statement.
 137  * It's global, meaning that it returns true even if there's
 138  * no switch in the current function, but we were inside
 139  * a switch inside another function, from which this one was called.
 140  */
 141 static INLINE boolean
 142 mask_has_switch(struct lp_exec_mask *mask)
 143 {
 144    int i;
 145    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 146       const struct function_ctx *ctx = &mask->function_stack[i];
 147       if (ctx->switch_stack_size > 0)
 148          return TRUE;
 149    }
 150    return FALSE;
 151 }
 152
 153 /*
 154  * Returns true if we're inside a conditional.
 155  * It's global, meaning that it returns true even if there's
 156  * no conditional in the current function, but we were inside
 157  * a conditional inside another function, from which this one was called.
 158  */
 159 static INLINE boolean
 160 mask_has_cond(struct lp_exec_mask *mask)
 161 {
 162    int i;
 163    for (i = mask->function_stack_size - 1; i >= 0; --i) {
 164       const struct function_ctx *ctx = &mask->function_stack[i];
 165       if (ctx->cond_stack_size > 0)
 166          return TRUE;
 167    }
 168    return FALSE;
 169 }
 170
 171
 172 /*
 173  * Initialize a function context at the specified index.
 174  */
 175 static void
 176 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
 177 {
 178    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 179    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 180    struct function_ctx *ctx =  &mask->function_stack[function_idx];
 181
 182    ctx->cond_stack_size = 0;
 183    ctx->loop_stack_size = 0;
 184    ctx->switch_stack_size = 0;
 185
 186    if (function_idx == 0) {
 187       ctx->ret_mask = mask->ret_mask;
 188    }
 189
 190    ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
 191                                        int_type, "looplimiter");
 192    LLVMBuildStore(
 193       builder,
 194       LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
 195       ctx->loop_limiter);
 196 }
 197
 198 static void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
 199 {
 200    mask->bld = bld;
 201    mask->has_mask = FALSE;
 202    mask->ret_in_main = FALSE;
 203    /* For the main function */
 204    mask->function_stack_size = 1;
 205
 206    mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
 207    mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
 208          mask->cond_mask = mask->switch_mask =
 209          LLVMConstAllOnes(mask->int_vec_type);
 210
 211    mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
 212                                  sizeof(mask->function_stack[0]));
 213    lp_exec_mask_function_init(mask, 0);
 214 }
 215
 216 static void
 217 lp_exec_mask_fini(struct lp_exec_mask *mask)
 218 {
 219    FREE(mask->function_stack);
 220 }
 221
 222 static void lp_exec_mask_update(struct lp_exec_mask *mask)
 223 {
 224    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 225    boolean has_loop_mask = mask_has_loop(mask);
 226    boolean has_cond_mask = mask_has_cond(mask);
 227    boolean has_switch_mask = mask_has_switch(mask);
 228    boolean has_ret_mask = mask->function_stack_size > 1 ||
 229          mask->ret_in_main;
 230
 231    if (has_loop_mask) {
 232       /*for loops we need to update the entire mask at runtime */
 233       LLVMValueRef tmp;
 234       assert(mask->break_mask);
 235       tmp = LLVMBuildAnd(builder,
 236                          mask->cont_mask,
 237                          mask->break_mask,
 238                          "maskcb");
 239       mask->exec_mask = LLVMBuildAnd(builder,
 240                                      mask->cond_mask,
 241                                      tmp,
 242                                      "maskfull");
 243    } else
 244       mask->exec_mask = mask->cond_mask;
 245
 246    if (has_switch_mask) {
 247       mask->exec_mask = LLVMBuildAnd(builder,
 248                                      mask->exec_mask,
 249                                      mask->switch_mask,
 250                                      "switchmask");
 251    }
 252
 253    if (has_ret_mask) {
 254       mask->exec_mask = LLVMBuildAnd(builder,
 255                                      mask->exec_mask,
 256                                      mask->ret_mask,
 257                                      "callmask");
 258    }
 259
 260    mask->has_mask = (has_cond_mask ||
 261                      has_loop_mask ||
 262                      has_switch_mask ||
 263                      has_ret_mask);
 264 }
 265
 266 static void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
 267                                    LLVMValueRef val)
 268 {
 269    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 270    struct function_ctx *ctx = func_ctx(mask);
 271
 272    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
 273       ctx->cond_stack_size++;
 274       return;
 275    }
 276    if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
 277       assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
 278    }
 279    ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
 280    assert(LLVMTypeOf(val) == mask->int_vec_type);
 281    mask->cond_mask = LLVMBuildAnd(builder,
 282                                   mask->cond_mask,
 283                                   val,
 284                                   "");
 285    lp_exec_mask_update(mask);
 286 }
 287
 288 static void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
 289 {
 290    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 291    struct function_ctx *ctx = func_ctx(mask);
 292    LLVMValueRef prev_mask;
 293    LLVMValueRef inv_mask;
 294
 295    assert(ctx->cond_stack_size);
 296    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 297       return;
 298    prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
 299    if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
 300       assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
 301    }
 302
 303    inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");
 304
 305    mask->cond_mask = LLVMBuildAnd(builder,
 306                                   inv_mask,
 307                                   prev_mask, "");
 308    lp_exec_mask_update(mask);
 309 }
 310
 311 static void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
 312 {
 313    struct function_ctx *ctx = func_ctx(mask);
 314    assert(ctx->cond_stack_size);
 315    --ctx->cond_stack_size;
 316    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 317       return;
 318    mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
 319    lp_exec_mask_update(mask);
 320 }
 321
 322 static void lp_exec_bgnloop(struct lp_exec_mask *mask)
 323 {
 324    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 325    struct function_ctx *ctx = func_ctx(mask);
 326
 327    if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
 328       ++ctx->loop_stack_size;
 329       return;
 330    }
 331
 332    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 333       ctx->break_type;
 334    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;
 335
 336    ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
 337    ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
 338    ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
 339    ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
 340    ++ctx->loop_stack_size;
 341
 342    ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
 343    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 344
 345    ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");
 346
 347    LLVMBuildBr(builder, ctx->loop_block);
 348    LLVMPositionBuilderAtEnd(builder, ctx->loop_block);
 349
 350    mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
 351
 352    lp_exec_mask_update(mask);
 353 }
 354
 355 static void lp_exec_break(struct lp_exec_mask *mask,
 356                           struct lp_build_tgsi_context * bld_base)
 357 {
 358    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 359    struct function_ctx *ctx = func_ctx(mask);
 360
 361    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 362       LLVMValueRef exec_mask = LLVMBuildNot(builder,
 363                                             mask->exec_mask,
 364                                             "break");
 365
 366       mask->break_mask = LLVMBuildAnd(builder,
 367                                       mask->break_mask,
 368                                       exec_mask, "break_full");
 369    }
 370    else {
 371       unsigned opcode = bld_base->instructions[bld_base->pc + 1].Instruction.Opcode;
 372       boolean break_always = (opcode == TGSI_OPCODE_ENDSWITCH ||
 373                               opcode == TGSI_OPCODE_CASE);
 374
 375
 376       if (ctx->switch_in_default) {
 377          /*
 378           * stop default execution but only if this is an unconditional switch.
 379           * (The condition here is not perfect since dead code after break is
 380           * allowed but should be sufficient since false negatives are just
 381           * unoptimized - so we don't have to pre-evaluate that).
 382           */
 383          if(break_always && ctx->switch_pc) {
 384             bld_base->pc = ctx->switch_pc;
 385             return;
 386          }
 387       }
 388
 389       if (break_always) {
 390          mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
 391       }
 392       else {
 393          LLVMValueRef exec_mask = LLVMBuildNot(builder,
 394                                                mask->exec_mask,
 395                                                "break");
 396          mask->switch_mask = LLVMBuildAnd(builder,
 397                                           mask->switch_mask,
 398                                           exec_mask, "break_switch");
 399       }
 400    }
 401
 402    lp_exec_mask_update(mask);
 403 }
 404
 405 static void lp_exec_break_condition(struct lp_exec_mask *mask,
 406                                     LLVMValueRef cond)
 407 {
 408    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 409    struct function_ctx *ctx = func_ctx(mask);
 410    LLVMValueRef cond_mask = LLVMBuildAnd(builder,
 411                                          mask->exec_mask,
 412                                          cond, "cond_mask");
 413    cond_mask = LLVMBuildNot(builder, cond_mask, "break_cond");
 414
 415    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 416       mask->break_mask = LLVMBuildAnd(builder,
 417                                       mask->break_mask,
 418                                       cond_mask, "breakc_full");
 419    }
 420    else {
 421       mask->switch_mask = LLVMBuildAnd(builder,
 422                                        mask->switch_mask,
 423                                        cond_mask, "breakc_switch");
 424    }
 425
 426    lp_exec_mask_update(mask);
 427 }
 428
 429 static void lp_exec_continue(struct lp_exec_mask *mask)
 430 {
 431    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 432    LLVMValueRef exec_mask = LLVMBuildNot(builder,
 433                                          mask->exec_mask,
 434                                          "");
 435
 436    mask->cont_mask = LLVMBuildAnd(builder,
 437                                   mask->cont_mask,
 438                                   exec_mask, "");
 439
 440    lp_exec_mask_update(mask);
 441 }
 442
 443
 444 static void lp_exec_endloop(struct gallivm_state *gallivm,
 445                             struct lp_exec_mask *mask)
 446 {
 447    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 448    struct function_ctx *ctx = func_ctx(mask);
 449    LLVMBasicBlockRef endloop;
 450    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 451    LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
 452                                                mask->bld->type.width *
 453                                                mask->bld->type.length);
 454    LLVMValueRef i1cond, i2cond, icond, limiter;
 455
 456    assert(mask->break_mask);
 457
 458
 459    assert(ctx->loop_stack_size);
 460    if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 461       --ctx->loop_stack_size;
 462       return;
 463    }
 464
 465    /*
 466     * Restore the cont_mask, but don't pop
 467     */
 468    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
 469    lp_exec_mask_update(mask);
 470
 471    /*
 472     * Unlike the continue mask, the break_mask must be preserved across loop
 473     * iterations
 474     */
 475    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 476
 477    /* Decrement the loop limiter */
 478    limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");
 479
 480    limiter = LLVMBuildSub(
 481       builder,
 482       limiter,
 483       LLVMConstInt(int_type, 1, false),
 484       "");
 485
 486    LLVMBuildStore(builder, limiter, ctx->loop_limiter);
 487
 488    /* i1cond = (mask != 0) */
 489    i1cond = LLVMBuildICmp(
 490       builder,
 491       LLVMIntNE,
 492       LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
 493       LLVMConstNull(reg_type), "i1cond");
 494
 495    /* i2cond = (looplimiter > 0) */
 496    i2cond = LLVMBuildICmp(
 497       builder,
 498       LLVMIntSGT,
 499       limiter,
 500       LLVMConstNull(int_type), "i2cond");
 501
 502    /* if( i1cond && i2cond ) */
 503    icond = LLVMBuildAnd(builder, i1cond, i2cond, "");
 504
 505    endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");
 506
 507    LLVMBuildCondBr(builder,
 508                    icond, ctx->loop_block, endloop);
 509
 510    LLVMPositionBuilderAtEnd(builder, endloop);
 511
 512    assert(ctx->loop_stack_size);
 513    --ctx->loop_stack_size;
 514    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
 515    mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
 516    ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
 517    ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
 518    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
 519          ctx->switch_stack_size];
 520
 521    lp_exec_mask_update(mask);
 522 }
 523
 524 static void lp_exec_switch(struct lp_exec_mask *mask,
 525                            LLVMValueRef switchval)
 526 {
 527    struct function_ctx *ctx = func_ctx(mask);
 528
 529    if (ctx->switch_stack_size >= LP_MAX_TGSI_NESTING ||
 530        ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 531       ctx->switch_stack_size++;
 532       return;
 533    }
 534
 535    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 536       ctx->break_type;
 537    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_SWITCH;
 538
 539    ctx->switch_stack[ctx->switch_stack_size].switch_mask = mask->switch_mask;
 540    ctx->switch_stack[ctx->switch_stack_size].switch_val = ctx->switch_val;
 541    ctx->switch_stack[ctx->switch_stack_size].switch_mask_default = ctx->switch_mask_default;
 542    ctx->switch_stack[ctx->switch_stack_size].switch_in_default = ctx->switch_in_default;
 543    ctx->switch_stack[ctx->switch_stack_size].switch_pc = ctx->switch_pc;
 544    ctx->switch_stack_size++;
 545
 546    mask->switch_mask = LLVMConstNull(mask->int_vec_type);
 547    ctx->switch_val = switchval;
 548    ctx->switch_mask_default = LLVMConstNull(mask->int_vec_type);
 549    ctx->switch_in_default = false;
 550    ctx->switch_pc = 0;
 551
 552    lp_exec_mask_update(mask);
 553 }
 554
 555 static void lp_exec_endswitch(struct lp_exec_mask *mask,
 556                               struct lp_build_tgsi_context * bld_base)
 557 {
 558    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 559    struct function_ctx *ctx = func_ctx(mask);
 560
 561    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 562       ctx->switch_stack_size--;
 563       return;
 564    }
 565
 566    /* check if there's deferred default if so do it now */
 567    if (ctx->switch_pc && !ctx->switch_in_default) {
 568       LLVMValueRef prevmask, defaultmask;
 569       unsigned tmp_pc;
 570       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 571       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 572       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 573       ctx->switch_in_default = true;
 574
 575       lp_exec_mask_update(mask);
 576
 577       assert(bld_base->instructions[ctx->switch_pc - 1].Instruction.Opcode ==
 578              TGSI_OPCODE_DEFAULT);
 579
 580       tmp_pc = bld_base->pc;
 581       bld_base->pc = ctx->switch_pc;
 582       /*
 583        * re-purpose switch_pc to point to here again, since we stop execution of
 584        * the deferred default after next break.
 585        */
 586       ctx->switch_pc = tmp_pc - 1;
 587
 588       return;
 589    }
 590
 591    else if (ctx->switch_pc && ctx->switch_in_default) {
 592       assert(bld_base->pc == ctx->switch_pc + 1);
 593    }
 594
 595    ctx->switch_stack_size--;
 596    mask->switch_mask = ctx->switch_stack[ctx->switch_stack_size].switch_mask;
 597    ctx->switch_val = ctx->switch_stack[ctx->switch_stack_size].switch_val;
 598    ctx->switch_mask_default = ctx->switch_stack[ctx->switch_stack_size].switch_mask_default;
 599    ctx->switch_in_default = ctx->switch_stack[ctx->switch_stack_size].switch_in_default;
 600    ctx->switch_pc = ctx->switch_stack[ctx->switch_stack_size].switch_pc;
 601
 602    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size];
 603
 604    lp_exec_mask_update(mask);
 605 }
 606
 607 static void lp_exec_case(struct lp_exec_mask *mask,
 608                          LLVMValueRef caseval)
 609 {
 610    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 611    struct function_ctx *ctx = func_ctx(mask);
 612
 613    LLVMValueRef casemask, prevmask;
 614
 615    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 616       return;
 617    }
 618
 619    /* skipping case mask evaluation here is NOT optional (not in all cases anyway). */
 620    if (!ctx->switch_in_default) {
 621       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 622       casemask = lp_build_cmp(mask->bld, PIPE_FUNC_EQUAL, caseval, ctx->switch_val);
 623       ctx->switch_mask_default = LLVMBuildOr(builder, casemask,
 624                                              ctx->switch_mask_default, "sw_default_mask");
 625       casemask = LLVMBuildOr(builder, casemask, mask->switch_mask, "");
 626       mask->switch_mask = LLVMBuildAnd(builder, casemask, prevmask, "sw_mask");
 627
 628       lp_exec_mask_update(mask);
 629    }
 630 }
 631
 632 /*
 633  * Analyse default statement in a switch.
 634  * \return true if default is last statement, false otherwise
 635  * \param default_pc_start contains pc of instruction to jump to
 636  *                         if default wasn't last but there's no
 637  *                         fallthrough into default.
 638  */
 639 static boolean default_analyse_is_last(struct lp_exec_mask *mask,
 640                                        struct lp_build_tgsi_context * bld_base,
 641                                        int *default_pc_start)
 642 {
 643    unsigned pc = bld_base->pc;
 644    struct function_ctx *ctx = func_ctx(mask);
 645    unsigned curr_switch_stack = ctx->switch_stack_size;
 646
 647    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 648       return false;
 649    }
 650
 651    /* skip over case statements which are together with default */
 652    while (bld_base->instructions[pc].Instruction.Opcode == TGSI_OPCODE_CASE) {
 653       pc++;
 654    }
 655
 656    while (pc != -1 && pc < bld_base->num_instructions) {
 657       unsigned opcode = bld_base->instructions[pc].Instruction.Opcode;
 658       switch (opcode) {
 659       case TGSI_OPCODE_CASE:
 660          if (curr_switch_stack == ctx->switch_stack_size) {
 661             *default_pc_start = pc - 1;
 662             return false;
 663          }
 664          break;
 665       case TGSI_OPCODE_SWITCH:
 666          curr_switch_stack++;
 667          break;
 668       case TGSI_OPCODE_ENDSWITCH:
 669          if (curr_switch_stack == ctx->switch_stack_size) {
 670             *default_pc_start = pc - 1;
 671             return true;
 672          }
 673          curr_switch_stack--;
 674          break;
 675       }
 676       pc++;
 677    }
 678    /* should never arrive here */
 679    assert(0);
 680    return true;
 681 }
 682
 683 static void lp_exec_default(struct lp_exec_mask *mask,
 684                             struct lp_build_tgsi_context * bld_base)
 685 {
 686    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 687    struct function_ctx *ctx = func_ctx(mask);
 688
 689    int default_exec_pc;
 690    boolean default_is_last;
 691
 692    if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {
 693       return;
 694    }
 695
 696    /*
 697     * This is a messy opcode, because it may not be always at the end and
 698     * there can be fallthrough in and out of it.
 699     */
 700
 701    default_is_last = default_analyse_is_last(mask, bld_base, &default_exec_pc);
 702    /*
 703     * If it is last statement in switch (note that case statements appearing
 704     * "at the same time" as default don't change that) everything is just fine,
 705     * update switch mask and go on. This means we can handle default with
 706     * fallthrough INTO it without overhead, if it is last.
 707     */
 708    if (default_is_last) {
 709       LLVMValueRef prevmask, defaultmask;
 710       prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;
 711       defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");
 712       defaultmask = LLVMBuildOr(builder, defaultmask, mask->switch_mask, "");
 713       mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");
 714       ctx->switch_in_default = true;
 715
 716       lp_exec_mask_update(mask);
 717    }
 718    else {
 719       /*
 720        * Technically, "case" immediately before default isn't really a
 721        * fallthrough, however we still have to count them as such as we
 722        * already have updated the masks.
 723        * If that happens in practice could add a switch optimizer pass
 724        * which just gets rid of all case statements appearing together with
 725        * default (or could do switch analysis at switch start time instead).
 726        */
 727       unsigned opcode = bld_base->instructions[bld_base->pc - 1].Instruction.Opcode;
 728       boolean ft_into = (opcode != TGSI_OPCODE_BRK ||
 729                          opcode != TGSI_OPCODE_SWITCH);
 730       /*
 731        * If it is not last statement and there was no fallthrough into it,
 732        * we record the PC and continue execution at next case (again, those
 733        * case encountered at the same time don't count). At endswitch
 734        * time, we update switchmask, and go back executing the code we skipped
 735        * until the next break (possibly re-executing some code with changed mask
 736        * if there was a fallthrough out of default).
 737        * Finally, if it is not last statement and there was a fallthrough into it,
 738        * do the same as with the former case, except instead of skipping the code
 739        * just execute it without updating the mask, then go back and re-execute.
 740        */
 741       ctx->switch_pc = bld_base->pc;
 742       if (!ft_into) {
 743          bld_base->pc = default_exec_pc;
 744       }
 745    }
 746 }
 747
 748
 749 /* stores val into an address pointed to by dst_ptr.
 750  * mask->exec_mask is used to figure out which bits of val
 751  * should be stored into the address
 752  * (0 means don't store this bit, 1 means do store).
 753  */
 754 static void lp_exec_mask_store(struct lp_exec_mask *mask,
 755                                struct lp_build_context *bld_store,
 756                                LLVMValueRef pred,
 757                                LLVMValueRef val,
 758                                LLVMValueRef dst_ptr)
 759 {
 760    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 761
 762    assert(lp_check_value(bld_store->type, val));
 763    assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
 764    assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val));
 765
 766    /* Mix the predicate and execution mask */
 767    if (mask->has_mask) {
 768       if (pred) {
 769          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
 770       } else {
 771          pred = mask->exec_mask;
 772       }
 773    }
 774
 775    if (pred) {
 776       LLVMValueRef res, dst;
 777
 778       dst = LLVMBuildLoad(builder, dst_ptr, "");
 779       res = lp_build_select(bld_store, pred, val, dst);
 780       LLVMBuildStore(builder, res, dst_ptr);
 781    } else
 782       LLVMBuildStore(builder, val, dst_ptr);
 783 }
 784
 785 static void lp_exec_mask_call(struct lp_exec_mask *mask,
 786                               int func,
 787                               int *pc)
 788 {
 789    if (mask->function_stack_size >= LP_MAX_NUM_FUNCS) {
 790       return;
 791    }
 792
 793    lp_exec_mask_function_init(mask, mask->function_stack_size);
 794    mask->function_stack[mask->function_stack_size].pc = *pc;
 795    mask->function_stack[mask->function_stack_size].ret_mask = mask->ret_mask;
 796    mask->function_stack_size++;
 797    *pc = func;
 798 }
 799
 800 static void lp_exec_mask_ret(struct lp_exec_mask *mask, int *pc)
 801 {
 802    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 803    struct function_ctx *ctx = func_ctx(mask);
 804    LLVMValueRef exec_mask;
 805
 806    if (ctx->cond_stack_size == 0 &&
 807        ctx->loop_stack_size == 0 &&
 808        ctx->switch_stack_size == 0 &&
 809        mask->function_stack_size == 1) {
 810       /* returning from main() */
 811       *pc = -1;
 812       return;
 813    }
 814
 815    if (mask->function_stack_size == 1) {
 816       /*
 817        * This requires special handling since we need to ensure
 818        * we don't drop the mask even if we have no call stack
 819        * (e.g. after a ret in a if clause after the endif)
 820        */
 821       mask->ret_in_main = TRUE;
 822    }
 823
 824    exec_mask = LLVMBuildNot(builder,
 825                             mask->exec_mask,
 826                             "ret");
 827
 828    mask->ret_mask = LLVMBuildAnd(builder,
 829                                  mask->ret_mask,
 830                                  exec_mask, "ret_full");
 831
 832    lp_exec_mask_update(mask);
 833 }
 834
 835 static void lp_exec_mask_bgnsub(struct lp_exec_mask *mask)
 836 {
 837 }
 838
 839 static void lp_exec_mask_endsub(struct lp_exec_mask *mask, int *pc)
 840 {
 841    struct function_ctx *ctx;
 842
 843    assert(mask->function_stack_size > 1);
 844    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
 845
 846    ctx = func_ctx(mask);
 847    mask->function_stack_size--;
 848
 849    *pc = ctx->pc;
 850    mask->ret_mask = ctx->ret_mask;
 851
 852    lp_exec_mask_update(mask);
 853 }
 854
 855
 856 static LLVMValueRef
 857 get_file_ptr(struct lp_build_tgsi_soa_context *bld,
 858              unsigned file,
 859              unsigned index,
 860              unsigned chan)
 861 {
 862    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 863    LLVMValueRef (*array_of_vars)[TGSI_NUM_CHANNELS];
 864    LLVMValueRef var_of_array;
 865
 866    switch (file) {
 867    case TGSI_FILE_TEMPORARY:
 868       array_of_vars = bld->temps;
 869       var_of_array = bld->temps_array;
 870       break;
 871    case TGSI_FILE_OUTPUT:
 872       array_of_vars = bld->outputs;
 873       var_of_array = bld->outputs_array;
 874       break;
 875    default:
 876       assert(0);
 877       return NULL;
 878    }
 879
 880    assert(chan < 4);
 881
 882    if (bld->indirect_files & (1 << file)) {
 883       LLVMValueRef lindex = lp_build_const_int32(bld->bld_base.base.gallivm, index * 4 + chan);
 884       return LLVMBuildGEP(builder, var_of_array, &lindex, 1, "");
 885    }
 886    else {
 887       assert(index <= bld->bld_base.info->file_max[file]);
 888       return array_of_vars[index][chan];
 889    }
 890 }
 891
 892
 893 /**
 894  * Return pointer to a temporary register channel (src or dest).
 895  * Note that indirect addressing cannot be handled here.
 896  * \param index  which temporary register
 897  * \param chan  which channel of the temp register.
 898  */
 899 LLVMValueRef
 900 lp_get_temp_ptr_soa(struct lp_build_tgsi_soa_context *bld,
 901              unsigned index,
 902              unsigned chan)
 903 {
 904    return get_file_ptr(bld, TGSI_FILE_TEMPORARY, index, chan);
 905 }
 906
 907 /**
 908  * Return pointer to a output register channel (src or dest).
 909  * Note that indirect addressing cannot be handled here.
 910  * \param index  which output register
 911  * \param chan  which channel of the output register.
 912  */
 913 LLVMValueRef
 914 lp_get_output_ptr(struct lp_build_tgsi_soa_context *bld,
 915                unsigned index,
 916                unsigned chan)
 917 {
 918    return get_file_ptr(bld, TGSI_FILE_OUTPUT, index, chan);
 919 }
 920
 921 /*
 922  * If we have indirect addressing in outputs copy our alloca array
 923  * to the outputs slots specified by the caller to make sure
 924  * our outputs are delivered consistently via the same interface.
 925  */
 926 static void
 927 gather_outputs(struct lp_build_tgsi_soa_context * bld)
 928 {
 929    if ((bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
 930       unsigned index, chan;
 931       assert(bld->bld_base.info->num_outputs <=
 932              bld->bld_base.info->file_max[TGSI_FILE_OUTPUT] + 1);
 933       for (index = 0; index < bld->bld_base.info->num_outputs; ++index) {
 934          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 935             bld->outputs[index][chan] = lp_get_output_ptr(bld, index, chan);
 936          }
 937       }
 938    }
 939 }
 940
 941 /**
 942  * Gather vector.
 943  * XXX the lp_build_gather() function should be capable of doing this
 944  * with a little work.
 945  */
 946 static LLVMValueRef
 947 build_gather(struct lp_build_context *bld,
 948              LLVMValueRef base_ptr,
 949              LLVMValueRef indexes,
 950              LLVMValueRef *overflow_mask)
 951 {
 952    LLVMBuilderRef builder = bld->gallivm->builder;
 953    LLVMValueRef res = bld->undef;
 954    unsigned i;
 955    LLVMValueRef temp_ptr;
 956
 957    if (overflow_mask) {
 958       temp_ptr = lp_build_alloca(
 959          bld->gallivm,
 960          lp_build_vec_type(bld->gallivm, bld->type), "");
 961    }
 962
 963    /*
 964     * Loop over elements of index_vec, load scalar value, insert it into 'res'.
 965     */
 966    for (i = 0; i < bld->type.length; i++) {
 967       LLVMValueRef ii = lp_build_const_int32(bld->gallivm, i);
 968       LLVMValueRef index = LLVMBuildExtractElement(builder,
 969                                                    indexes, ii, "");
 970       LLVMValueRef scalar_ptr, scalar;
 971       LLVMValueRef overflow;
 972       struct lp_build_if_state if_ctx;
 973
 974       /*
 975        * overflow_mask is a boolean vector telling us which channels
 976        * in the vector overflowed. We use the overflow behavior for
 977        * constant buffers which is defined as:
 978        * Out of bounds access to constant buffer returns 0 in all
 979        * componenets. Out of bounds behavior is always with respect
 980        * to the size of the buffer bound at that slot.
 981        */
 982       if (overflow_mask) {
 983          overflow = LLVMBuildExtractElement(builder, *overflow_mask,
 984                                             ii, "");
 985          lp_build_if(&if_ctx, bld->gallivm, overflow);
 986          {
 987             LLVMValueRef val = LLVMBuildLoad(builder, temp_ptr, "");
 988             val = LLVMBuildInsertElement(
 989                builder, val,
 990                LLVMConstNull(LLVMFloatTypeInContext(bld->gallivm->context)),
 991                ii, "");
 992             LLVMBuildStore(builder, val, temp_ptr);
 993          }
 994          lp_build_else(&if_ctx);
 995          {
 996             LLVMValueRef val = LLVMBuildLoad(builder, temp_ptr, "");
 997
 998             scalar_ptr = LLVMBuildGEP(builder, base_ptr,
 999                                       &index, 1, "gather_ptr");
1000             scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1001
1002             val = LLVMBuildInsertElement(builder, val, scalar, ii, "");
1003
1004             LLVMBuildStore(builder, val, temp_ptr);
1005          }
1006          lp_build_endif(&if_ctx);
1007       } else {
1008          scalar_ptr = LLVMBuildGEP(builder, base_ptr,
1009                                    &index, 1, "gather_ptr");
1010          scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1011
1012          res = LLVMBuildInsertElement(builder, res, scalar, ii, "");
1013       }
1014    }
1015
1016    if (overflow_mask) {
1017       res = LLVMBuildLoad(builder, temp_ptr, "gather_val");
1018    }
1019
1020    return res;
1021 }
1022
1023
1024 /**
1025  * Scatter/store vector.
1026  */
1027 static void
1028 emit_mask_scatter(struct lp_build_tgsi_soa_context *bld,
1029                   LLVMValueRef base_ptr,
1030                   LLVMValueRef indexes,
1031                   LLVMValueRef values,
1032                   struct lp_exec_mask *mask,
1033                   LLVMValueRef pred)
1034 {
1035    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1036    LLVMBuilderRef builder = gallivm->builder;
1037    unsigned i;
1038
1039    /* Mix the predicate and execution mask */
1040    if (mask->has_mask) {
1041       if (pred) {
1042          pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");
1043       }
1044       else {
1045          pred = mask->exec_mask;
1046       }
1047    }
1048
1049    /*
1050     * Loop over elements of index_vec, store scalar value.
1051     */
1052    for (i = 0; i < bld->bld_base.base.type.length; i++) {
1053       LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1054       LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
1055       LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
1056       LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
1057       LLVMValueRef scalar_pred = pred ?
1058          LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
1059
1060       if (0)
1061          lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
1062                          ii, val, index, scalar_ptr);
1063
1064       if (scalar_pred) {
1065          LLVMValueRef real_val, dst_val;
1066          dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
1067          real_val = lp_build_select(&bld->elem_bld, scalar_pred, val, dst_val);
1068          LLVMBuildStore(builder, real_val, scalar_ptr);
1069       }
1070       else {
1071          LLVMBuildStore(builder, val, scalar_ptr);
1072       }
1073    }
1074 }
1075
1076
1077 /**
1078  * Read the current value of the ADDR register, convert the floats to
1079  * ints, add the base index and return the vector of offsets.
1080  * The offsets will be used to index into the constant buffer or
1081  * temporary register file.
1082  */
1083 static LLVMValueRef
1084 get_indirect_index(struct lp_build_tgsi_soa_context *bld,
1085                    unsigned reg_file, unsigned reg_index,
1086                    const struct tgsi_ind_register *indirect_reg)
1087 {
1088    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1089    struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
1090    /* always use X component of address register */
1091    unsigned swizzle = indirect_reg->Swizzle;
1092    LLVMValueRef base;
1093    LLVMValueRef rel;
1094    LLVMValueRef max_index;
1095    LLVMValueRef index;
1096
1097    assert(bld->indirect_files & (1 << reg_file));
1098
1099    base = lp_build_const_int_vec(bld->bld_base.base.gallivm, uint_bld->type, reg_index);
1100
1101    assert(swizzle < 4);
1102    switch (indirect_reg->File) {
1103    case TGSI_FILE_ADDRESS:
1104       rel = LLVMBuildLoad(builder,
1105                           bld->addr[indirect_reg->Index][swizzle],
1106                           "load addr reg");
1107       /* ADDR LLVM values already have LLVM integer type. */
1108       break;
1109    case TGSI_FILE_TEMPORARY:
1110       rel = lp_get_temp_ptr_soa(bld, indirect_reg->Index, swizzle);
1111       rel = LLVMBuildLoad(builder, rel, "load temp reg");
1112       /* TEMP LLVM values always have LLVM float type, but for indirection, the
1113        * value actually stored is expected to be an integer */
1114       rel = LLVMBuildBitCast(builder, rel, uint_bld->vec_type, "");
1115       break;
1116    default:
1117       assert(0);
1118       rel = uint_bld->zero;
1119    }
1120
1121    index = lp_build_add(uint_bld, base, rel);
1122
1123    /*
1124     * emit_fetch_constant handles constant buffer overflow so this code
1125     * is pointless for them.
1126     * Furthermore the D3D10 spec in section 6.5 says:
1127     * If the constant buffer bound to a slot is larger than the size
1128     * declared in the shader for that slot, implementations are allowed
1129     * to return incorrect data (not necessarily 0) for indices that are
1130     * larger than the declared size but smaller than the buffer size.
1131     */
1132    if (reg_file != TGSI_FILE_CONSTANT) {
1133       max_index = lp_build_const_int_vec(bld->bld_base.base.gallivm,
1134                                          uint_bld->type,
1135                                          bld->bld_base.info->file_max[reg_file]);
1136
1137       assert(!uint_bld->type.sign);
1138       index = lp_build_min(uint_bld, index, max_index);
1139    }
1140
1141    return index;
1142 }
1143
1144 static struct lp_build_context *
1145 stype_to_fetch(struct lp_build_tgsi_context * bld_base,
1146                enum tgsi_opcode_type stype)
1147 {
1148    struct lp_build_context *bld_fetch;
1149
1150    switch (stype) {
1151    case TGSI_TYPE_FLOAT:
1152    case TGSI_TYPE_UNTYPED:
1153       bld_fetch = &bld_base->base;
1154       break;
1155    case TGSI_TYPE_UNSIGNED:
1156       bld_fetch = &bld_base->uint_bld;
1157       break;
1158    case TGSI_TYPE_SIGNED:
1159       bld_fetch = &bld_base->int_bld;
1160       break;
1161    case TGSI_TYPE_VOID:
1162    case TGSI_TYPE_DOUBLE:
1163    default:
1164       assert(0);
1165       bld_fetch = NULL;
1166       break;
1167    }
1168    return bld_fetch;
1169 }
1170
1171 static LLVMValueRef
1172 get_soa_array_offsets(struct lp_build_context *uint_bld,
1173                       LLVMValueRef indirect_index,
1174                       unsigned chan_index,
1175                       boolean need_perelement_offset)
1176 {
1177    struct gallivm_state *gallivm = uint_bld->gallivm;
1178    LLVMValueRef chan_vec =
1179       lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
1180    LLVMValueRef length_vec =
1181       lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
1182    LLVMValueRef index_vec;
1183
1184    /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
1185    index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1186    index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
1187    index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
1188
1189    if (need_perelement_offset) {
1190       LLVMValueRef pixel_offsets;
1191       int i;
1192      /* build pixel offset vector: {0, 1, 2, 3, ...} */
1193       pixel_offsets = uint_bld->undef;
1194       for (i = 0; i < uint_bld->type.length; i++) {
1195          LLVMValueRef ii = lp_build_const_int32(gallivm, i);
1196          pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
1197                                                 ii, ii, "");
1198       }
1199       index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
1200    }
1201    return index_vec;
1202 }
1203
1204 static LLVMValueRef
1205 emit_fetch_constant(
1206    struct lp_build_tgsi_context * bld_base,
1207    const struct tgsi_full_src_register * reg,
1208    enum tgsi_opcode_type stype,
1209    unsigned swizzle)
1210 {
1211    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1212    struct gallivm_state *gallivm = bld_base->base.gallivm;
1213    LLVMBuilderRef builder = gallivm->builder;
1214    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1215    unsigned dimension = 0;
1216    LLVMValueRef dimension_index;
1217    LLVMValueRef consts_ptr;
1218    LLVMValueRef num_consts;
1219    LLVMValueRef res;
1220
1221    /* XXX: Handle fetching xyzw components as a vector */
1222    assert(swizzle != ~0);
1223
1224    if (reg->Register.Dimension) {
1225       assert(!reg->Dimension.Indirect);
1226       dimension = reg->Dimension.Index;
1227       assert(dimension < LP_MAX_TGSI_CONST_BUFFERS);
1228    }
1229
1230    dimension_index = lp_build_const_int32(gallivm, dimension);
1231    consts_ptr =
1232       lp_build_array_get(gallivm, bld->consts_ptr, dimension_index);
1233    num_consts =
1234       lp_build_array_get(gallivm, bld->const_sizes_ptr, dimension_index);
1235
1236    if (reg->Register.Indirect) {
1237       LLVMValueRef indirect_index;
1238       LLVMValueRef swizzle_vec =
1239          lp_build_const_int_vec(gallivm, uint_bld->type, swizzle);
1240       LLVMValueRef index_vec;  /* index into the const buffer */
1241       LLVMValueRef overflow_mask;
1242
1243       indirect_index = get_indirect_index(bld,
1244                                           reg->Register.File,
1245                                           reg->Register.Index,
1246                                           &reg->Indirect);
1247
1248       /* All fetches are from the same constant buffer, so
1249        * we need to propagate the size to a vector to do a
1250        * vector comparison */
1251       num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
1252       /* Construct a boolean vector telling us which channels
1253        * overflow the bound constant buffer */
1254       overflow_mask = LLVMBuildICmp(builder, LLVMIntUGE,
1255                                     indirect_index,
1256                                     num_consts, "");
1257
1258       /* index_vec = indirect_index * 4 + swizzle */
1259       index_vec = lp_build_shl_imm(uint_bld, indirect_index, 2);
1260       index_vec = lp_build_add(uint_bld, index_vec, swizzle_vec);
1261
1262       /* Gather values from the constant buffer */
1263       res = build_gather(&bld_base->base, consts_ptr, index_vec,
1264                          &overflow_mask);
1265    }
1266    else {
1267       LLVMValueRef index;  /* index into the const buffer */
1268       LLVMValueRef scalar, scalar_ptr;
1269
1270       index = lp_build_const_int32(gallivm, reg->Register.Index * 4 + swizzle);
1271
1272       scalar_ptr = LLVMBuildGEP(builder, consts_ptr,
1273                                 &index, 1, "");
1274       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
1275       res = lp_build_broadcast_scalar(&bld_base->base, scalar);
1276    }
1277
1278    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED) {
1279       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1280       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1281    }
1282
1283    return res;
1284 }
1285
1286 static LLVMValueRef
1287 emit_fetch_immediate(
1288    struct lp_build_tgsi_context * bld_base,
1289    const struct tgsi_full_src_register * reg,
1290    enum tgsi_opcode_type stype,
1291    unsigned swizzle)
1292 {
1293    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1294    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1295    LLVMBuilderRef builder = gallivm->builder;
1296    LLVMValueRef res = NULL;
1297
1298    if (reg->Register.Indirect) {
1299       LLVMValueRef indirect_index;
1300       LLVMValueRef index_vec;  /* index into the immediate register array */
1301       LLVMValueRef imms_array;
1302       LLVMTypeRef fptr_type;
1303
1304       indirect_index = get_indirect_index(bld,
1305                                           reg->Register.File,
1306                                           reg->Register.Index,
1307                                           &reg->Indirect);
1308       /*
1309        * Unlike for other reg classes, adding pixel offsets is unnecessary -
1310        * immediates are stored as full vectors (FIXME??? - might be better
1311        * to store them the same as constants) but all elements are the same
1312        * in any case.
1313        */
1314       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1315                                         indirect_index,
1316                                         swizzle,
1317                                         FALSE);
1318
1319       /* cast imms_array pointer to float* */
1320       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1321       imms_array = LLVMBuildBitCast(builder, bld->imms_array, fptr_type, "");
1322
1323       /* Gather values from the immediate register array */
1324       res = build_gather(&bld_base->base, imms_array, index_vec, NULL);
1325    }
1326    else {
1327       res = bld->immediates[reg->Register.Index][swizzle];
1328    }
1329
1330    if (stype == TGSI_TYPE_UNSIGNED) {
1331       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1332    } else if (stype == TGSI_TYPE_SIGNED) {
1333       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1334    }
1335    return res;
1336 }
1337
1338 static LLVMValueRef
1339 emit_fetch_input(
1340    struct lp_build_tgsi_context * bld_base,
1341    const struct tgsi_full_src_register * reg,
1342    enum tgsi_opcode_type stype,
1343    unsigned swizzle)
1344 {
1345    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1346    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1347    LLVMBuilderRef builder = gallivm->builder;
1348    LLVMValueRef res;
1349
1350    if (reg->Register.Indirect) {
1351       LLVMValueRef indirect_index;
1352       LLVMValueRef index_vec;  /* index into the input reg array */
1353       LLVMValueRef inputs_array;
1354       LLVMTypeRef fptr_type;
1355
1356       indirect_index = get_indirect_index(bld,
1357                                           reg->Register.File,
1358                                           reg->Register.Index,
1359                                           &reg->Indirect);
1360
1361       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1362                                         indirect_index,
1363                                         swizzle,
1364                                         TRUE);
1365
1366       /* cast inputs_array pointer to float* */
1367       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1368       inputs_array = LLVMBuildBitCast(builder, bld->inputs_array, fptr_type, "");
1369
1370       /* Gather values from the input register array */
1371       res = build_gather(&bld_base->base, inputs_array, index_vec, NULL);
1372    } else {
1373       if (bld->indirect_files & (1 << TGSI_FILE_INPUT)) {
1374          LLVMValueRef lindex = lp_build_const_int32(gallivm,
1375                                         reg->Register.Index * 4 + swizzle);
1376          LLVMValueRef input_ptr =  LLVMBuildGEP(builder,
1377                                                 bld->inputs_array, &lindex, 1, "");
1378          res = LLVMBuildLoad(builder, input_ptr, "");
1379       }
1380       else {
1381          res = bld->inputs[reg->Register.Index][swizzle];
1382       }
1383    }
1384
1385    assert(res);
1386
1387    if (stype == TGSI_TYPE_UNSIGNED) {
1388       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1389    } else if (stype == TGSI_TYPE_SIGNED) {
1390       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1391    }
1392
1393    return res;
1394 }
1395
1396
1397 static LLVMValueRef
1398 emit_fetch_gs_input(
1399    struct lp_build_tgsi_context * bld_base,
1400    const struct tgsi_full_src_register * reg,
1401    enum tgsi_opcode_type stype,
1402    unsigned swizzle)
1403 {
1404    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1405    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1406    LLVMBuilderRef builder = gallivm->builder;
1407    LLVMValueRef attrib_index = NULL;
1408    LLVMValueRef vertex_index = NULL;
1409    LLVMValueRef swizzle_index = lp_build_const_int32(gallivm, swizzle);
1410    LLVMValueRef res;
1411
1412    if (reg->Register.Indirect) {
1413       attrib_index = get_indirect_index(bld,
1414                                         reg->Register.File,
1415                                         reg->Register.Index,
1416                                         &reg->Indirect);
1417    } else {
1418       attrib_index = lp_build_const_int32(gallivm, reg->Register.Index);
1419    }
1420
1421    if (reg->Dimension.Indirect) {
1422       vertex_index = get_indirect_index(bld,
1423                                         reg->Register.File,
1424                                         reg->Dimension.Index,
1425                                         &reg->DimIndirect);
1426    } else {
1427       vertex_index = lp_build_const_int32(gallivm, reg->Dimension.Index);
1428    }
1429
1430    res = bld->gs_iface->fetch_input(bld->gs_iface, bld_base,
1431                                     reg->Dimension.Indirect,
1432                                     vertex_index,
1433                                     reg->Register.Indirect,
1434                                     attrib_index,
1435                                     swizzle_index);
1436
1437    assert(res);
1438
1439    if (stype == TGSI_TYPE_UNSIGNED) {
1440       res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1441    } else if (stype == TGSI_TYPE_SIGNED) {
1442       res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1443    }
1444
1445    return res;
1446 }
1447
1448 static LLVMValueRef
1449 emit_fetch_temporary(
1450    struct lp_build_tgsi_context * bld_base,
1451    const struct tgsi_full_src_register * reg,
1452    enum tgsi_opcode_type stype,
1453    unsigned swizzle)
1454 {
1455    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1456    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1457    LLVMBuilderRef builder = gallivm->builder;
1458    LLVMValueRef res;
1459
1460    if (reg->Register.Indirect) {
1461       LLVMValueRef indirect_index;
1462       LLVMValueRef index_vec;  /* index into the temp reg array */
1463       LLVMValueRef temps_array;
1464       LLVMTypeRef fptr_type;
1465
1466       indirect_index = get_indirect_index(bld,
1467                                           reg->Register.File,
1468                                           reg->Register.Index,
1469                                           &reg->Indirect);
1470
1471       index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1472                                         indirect_index,
1473                                         swizzle,
1474                                         TRUE);
1475
1476       /* cast temps_array pointer to float* */
1477       fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1478       temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1479
1480       /* Gather values from the temporary register array */
1481       res = build_gather(&bld_base->base, temps_array, index_vec, NULL);
1482    }
1483    else {
1484       LLVMValueRef temp_ptr;
1485       temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle);
1486       res = LLVMBuildLoad(builder, temp_ptr, "");
1487    }
1488
1489    if (stype == TGSI_TYPE_SIGNED || stype == TGSI_TYPE_UNSIGNED) {
1490       struct lp_build_context *bld_fetch = stype_to_fetch(bld_base, stype);
1491       res = LLVMBuildBitCast(builder, res, bld_fetch->vec_type, "");
1492    }
1493
1494    return res;
1495 }
1496
1497 static LLVMValueRef
1498 emit_fetch_system_value(
1499    struct lp_build_tgsi_context * bld_base,
1500    const struct tgsi_full_src_register * reg,
1501    enum tgsi_opcode_type stype,
1502    unsigned swizzle)
1503 {
1504    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1505    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
1506    const struct tgsi_shader_info *info = bld->bld_base.info;
1507    LLVMBuilderRef builder = gallivm->builder;
1508    LLVMValueRef res;
1509    enum tgsi_opcode_type atype; // Actual type of the value
1510
1511    assert(!reg->Register.Indirect);
1512
1513    switch (info->system_value_semantic_name[reg->Register.Index]) {
1514    case TGSI_SEMANTIC_INSTANCEID:
1515       res = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
1516       atype = TGSI_TYPE_UNSIGNED;
1517       break;
1518
1519    case TGSI_SEMANTIC_VERTEXID:
1520       res = bld->system_values.vertex_id;
1521       atype = TGSI_TYPE_UNSIGNED;
1522       break;
1523
1524    case TGSI_SEMANTIC_PRIMID:
1525       res = bld->system_values.prim_id;
1526       atype = TGSI_TYPE_UNSIGNED;
1527       break;
1528
1529    default:
1530       assert(!"unexpected semantic in emit_fetch_system_value");
1531       res = bld_base->base.zero;
1532       atype = TGSI_TYPE_FLOAT;
1533       break;
1534    }
1535
1536    if (atype != stype) {
1537       if (stype == TGSI_TYPE_FLOAT) {
1538          res = LLVMBuildBitCast(builder, res, bld_base->base.vec_type, "");
1539       } else if (stype == TGSI_TYPE_UNSIGNED) {
1540          res = LLVMBuildBitCast(builder, res, bld_base->uint_bld.vec_type, "");
1541       } else if (stype == TGSI_TYPE_SIGNED) {
1542          res = LLVMBuildBitCast(builder, res, bld_base->int_bld.vec_type, "");
1543       }
1544    }
1545
1546    return res;
1547 }
1548
1549 /**
1550  * Register fetch with derivatives.
1551  */
1552 static void
1553 emit_fetch_deriv(
1554    struct lp_build_tgsi_soa_context *bld,
1555    LLVMValueRef src,
1556    LLVMValueRef *res,
1557    LLVMValueRef *ddx,
1558    LLVMValueRef *ddy)
1559 {
1560    if(res)
1561       *res = src;
1562
1563    /* TODO: use interpolation coeffs for inputs */
1564
1565    if(ddx)
1566       *ddx = lp_build_ddx(&bld->bld_base.base, src);
1567
1568    if(ddy)
1569       *ddy = lp_build_ddy(&bld->bld_base.base, src);
1570 }
1571
1572
1573 /**
1574  * Predicate.
1575  */
1576 static void
1577 emit_fetch_predicate(
1578    struct lp_build_tgsi_soa_context *bld,
1579    const struct tgsi_full_instruction *inst,
1580    LLVMValueRef *pred)
1581 {
1582    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1583    unsigned index;
1584    unsigned char swizzles[4];
1585    LLVMValueRef unswizzled[4] = {NULL, NULL, NULL, NULL};
1586    LLVMValueRef value;
1587    unsigned chan;
1588
1589    if (!inst->Instruction.Predicate) {
1590       TGSI_FOR_EACH_CHANNEL( chan ) {
1591          pred[chan] = NULL;
1592       }
1593       return;
1594    }
1595
1596    swizzles[0] = inst->Predicate.SwizzleX;
1597    swizzles[1] = inst->Predicate.SwizzleY;
1598    swizzles[2] = inst->Predicate.SwizzleZ;
1599    swizzles[3] = inst->Predicate.SwizzleW;
1600
1601    index = inst->Predicate.Index;
1602    assert(index < LP_MAX_TGSI_PREDS);
1603
1604    TGSI_FOR_EACH_CHANNEL( chan ) {
1605       unsigned swizzle = swizzles[chan];
1606
1607       /*
1608        * Only fetch the predicate register channels that are actually listed
1609        * in the swizzles
1610        */
1611       if (!unswizzled[swizzle]) {
1612          value = LLVMBuildLoad(builder,
1613                                bld->preds[index][swizzle], "");
1614
1615          /*
1616           * Convert the value to an integer mask.
1617           *
1618           * TODO: Short-circuit this comparison -- a D3D setp_xx instructions
1619           * is needlessly causing two comparisons due to storing the intermediate
1620           * result as float vector instead of an integer mask vector.
1621           */
1622          value = lp_build_compare(bld->bld_base.base.gallivm,
1623                                   bld->bld_base.base.type,
1624                                   PIPE_FUNC_NOTEQUAL,
1625                                   value,
1626                                   bld->bld_base.base.zero);
1627          if (inst->Predicate.Negate) {
1628             value = LLVMBuildNot(builder, value, "");
1629          }
1630
1631          unswizzled[swizzle] = value;
1632       } else {
1633          value = unswizzled[swizzle];
1634       }
1635
1636       pred[chan] = value;
1637    }
1638 }
1639
1640
1641 /**
1642  * Register store.
1643  */
1644 static void
1645 emit_store_chan(
1646    struct lp_build_tgsi_context *bld_base,
1647    const struct tgsi_full_instruction *inst,
1648    unsigned index,
1649    unsigned chan_index,
1650    LLVMValueRef pred,
1651    LLVMValueRef value)
1652 {
1653    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1654    struct gallivm_state *gallivm = bld_base->base.gallivm;
1655    LLVMBuilderRef builder = gallivm->builder;
1656    const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1657    struct lp_build_context *float_bld = &bld_base->base;
1658    struct lp_build_context *int_bld = &bld_base->int_bld;
1659    LLVMValueRef indirect_index = NULL;
1660    enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode);
1661
1662    /*
1663     * Apply saturation.
1664     *
1665     * It is always assumed to be float.
1666     */
1667    switch( inst->Instruction.Saturate ) {
1668    case TGSI_SAT_NONE:
1669       break;
1670
1671    case TGSI_SAT_ZERO_ONE:
1672       assert(dtype == TGSI_TYPE_FLOAT ||
1673              dtype == TGSI_TYPE_UNTYPED);
1674       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1675       value = lp_build_clamp_zero_one_nanzero(float_bld, value);
1676       break;
1677
1678    case TGSI_SAT_MINUS_PLUS_ONE:
1679       assert(dtype == TGSI_TYPE_FLOAT ||
1680              dtype == TGSI_TYPE_UNTYPED);
1681       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1682       /* This will give -1.0 for NaN which is probably not what we want. */
1683       value = lp_build_max_ext(float_bld, value,
1684                                lp_build_const_vec(gallivm, float_bld->type, -1.0),
1685                                GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
1686       value = lp_build_min(float_bld, value, float_bld->one);
1687       break;
1688
1689    default:
1690       assert(0);
1691    }
1692
1693    if (reg->Register.Indirect) {
1694       indirect_index = get_indirect_index(bld,
1695                                           reg->Register.File,
1696                                           reg->Register.Index,
1697                                           &reg->Indirect);
1698    } else {
1699       assert(reg->Register.Index <=
1700                              bld_base->info->file_max[reg->Register.File]);
1701    }
1702
1703    if (DEBUG_EXECUTION) {
1704       emit_dump_reg(gallivm, reg->Register.File, reg->Register.Index, chan_index, value);
1705    }
1706
1707    switch( reg->Register.File ) {
1708    case TGSI_FILE_OUTPUT:
1709       /* Outputs are always stored as floats */
1710       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1711
1712       if (reg->Register.Indirect) {
1713          LLVMValueRef index_vec;  /* indexes into the output registers */
1714          LLVMValueRef outputs_array;
1715          LLVMTypeRef fptr_type;
1716
1717          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1718                                            indirect_index,
1719                                            chan_index,
1720                                            TRUE);
1721
1722          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1723          outputs_array = LLVMBuildBitCast(builder, bld->outputs_array, fptr_type, "");
1724
1725          /* Scatter store values into output registers */
1726          emit_mask_scatter(bld, outputs_array, index_vec, value,
1727                            &bld->exec_mask, pred);
1728       }
1729       else {
1730          LLVMValueRef out_ptr = lp_get_output_ptr(bld, reg->Register.Index,
1731                                                   chan_index);
1732          lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, out_ptr);
1733       }
1734       break;
1735
1736    case TGSI_FILE_TEMPORARY:
1737       /* Temporaries are always stored as floats */
1738       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1739
1740       if (reg->Register.Indirect) {
1741          LLVMValueRef index_vec;  /* indexes into the temp registers */
1742          LLVMValueRef temps_array;
1743          LLVMTypeRef fptr_type;
1744
1745          index_vec = get_soa_array_offsets(&bld_base->uint_bld,
1746                                            indirect_index,
1747                                            chan_index,
1748                                            TRUE);
1749
1750          fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
1751          temps_array = LLVMBuildBitCast(builder, bld->temps_array, fptr_type, "");
1752
1753          /* Scatter store values into temp registers */
1754          emit_mask_scatter(bld, temps_array, index_vec, value,
1755                            &bld->exec_mask, pred);
1756       }
1757       else {
1758          LLVMValueRef temp_ptr;
1759          temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, chan_index);
1760          lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value, temp_ptr);
1761       }
1762       break;
1763
1764    case TGSI_FILE_ADDRESS:
1765       assert(dtype == TGSI_TYPE_SIGNED);
1766       assert(LLVMTypeOf(value) == int_bld->vec_type);
1767       value = LLVMBuildBitCast(builder, value, int_bld->vec_type, "");
1768       lp_exec_mask_store(&bld->exec_mask, int_bld, pred, value,
1769                          bld->addr[reg->Register.Index][chan_index]);
1770       break;
1771
1772    case TGSI_FILE_PREDICATE:
1773       assert(LLVMTypeOf(value) == float_bld->vec_type);
1774       value = LLVMBuildBitCast(builder, value, float_bld->vec_type, "");
1775       lp_exec_mask_store(&bld->exec_mask, float_bld, pred, value,
1776                          bld->preds[reg->Register.Index][chan_index]);
1777       break;
1778
1779    default:
1780       assert( 0 );
1781    }
1782
1783    (void)dtype;
1784 }
1785
1786 /*
1787  * Called at the beginning of the translation of each TGSI instruction, to
1788  * emit some debug code.
1789  */
1790 static void
1791 emit_debug(
1792    struct lp_build_tgsi_context * bld_base,
1793    const struct tgsi_full_instruction * inst,
1794    const struct tgsi_opcode_info * info)
1795
1796 {
1797    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1798
1799    if (DEBUG_EXECUTION) {
1800       /*
1801        * Dump the TGSI instruction.
1802        */
1803
1804       struct gallivm_state *gallivm = bld_base->base.gallivm;
1805       char buf[512];
1806       buf[0] = '$';
1807       buf[1] = ' ';
1808       tgsi_dump_instruction_str(inst, bld_base->pc, &buf[2], sizeof buf - 2);
1809       lp_build_printf(gallivm, buf);
1810
1811       /* Dump the execution mask.
1812        */
1813       if (bld->exec_mask.has_mask) {
1814          lp_build_print_value(gallivm, "    mask = ", bld->exec_mask.exec_mask);
1815       }
1816    }
1817 }
1818
1819 static void
1820 emit_store(
1821    struct lp_build_tgsi_context * bld_base,
1822    const struct tgsi_full_instruction * inst,
1823    const struct tgsi_opcode_info * info,
1824    LLVMValueRef dst[4])
1825
1826 {
1827    unsigned chan_index;
1828    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
1829
1830    if(info->num_dst) {
1831       LLVMValueRef pred[TGSI_NUM_CHANNELS];
1832
1833       emit_fetch_predicate( bld, inst, pred );
1834
1835       TGSI_FOR_EACH_DST0_ENABLED_CHANNEL( inst, chan_index ) {
1836          emit_store_chan(bld_base, inst, 0, chan_index, pred[chan_index], dst[chan_index]);
1837       }
1838    }
1839 }
1840
1841 static unsigned
1842 tgsi_to_pipe_tex_target(unsigned tgsi_target)
1843 {
1844    switch (tgsi_target) {
1845    case TGSI_TEXTURE_BUFFER:
1846       return PIPE_BUFFER;
1847    case TGSI_TEXTURE_1D:
1848    case TGSI_TEXTURE_SHADOW1D:
1849       return PIPE_TEXTURE_1D;
1850    case TGSI_TEXTURE_2D:
1851    case TGSI_TEXTURE_SHADOW2D:
1852    case TGSI_TEXTURE_2D_MSAA:
1853       return PIPE_TEXTURE_2D;
1854    case TGSI_TEXTURE_3D:
1855       return PIPE_TEXTURE_3D;
1856    case TGSI_TEXTURE_CUBE:
1857    case TGSI_TEXTURE_SHADOWCUBE:
1858       return PIPE_TEXTURE_CUBE;
1859    case TGSI_TEXTURE_RECT:
1860    case TGSI_TEXTURE_SHADOWRECT:
1861       return PIPE_TEXTURE_RECT;
1862    case TGSI_TEXTURE_1D_ARRAY:
1863    case TGSI_TEXTURE_SHADOW1D_ARRAY:
1864       return PIPE_TEXTURE_1D_ARRAY;
1865    case TGSI_TEXTURE_2D_ARRAY:
1866    case TGSI_TEXTURE_SHADOW2D_ARRAY:
1867    case TGSI_TEXTURE_2D_ARRAY_MSAA:
1868       return PIPE_TEXTURE_2D_ARRAY;
1869    case TGSI_TEXTURE_CUBE_ARRAY:
1870    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
1871       return PIPE_TEXTURE_CUBE_ARRAY;
1872    default:
1873       assert(0);
1874       return PIPE_BUFFER;
1875    }
1876 }
1877
1878
1879 static enum lp_sampler_lod_property
1880 lp_build_lod_property(
1881    struct lp_build_tgsi_context *bld_base,
1882    const struct tgsi_full_instruction *inst,
1883    unsigned src_op)
1884 {
1885    const struct tgsi_full_src_register *reg = &inst->Src[src_op];
1886    enum lp_sampler_lod_property lod_property;
1887
1888    /*
1889     * Not much we can do here. We could try catching inputs declared
1890     * with constant interpolation but not sure it's worth it - since for
1891     * TEX opcodes as well as FETCH/LD the lod comes from same reg as
1892     * the coords, so it could only work for SAMPLE/TXQ/SVIEWINFO), just
1893     * like the constant/immediate recognition below.
1894     * What seems to be of more value would be to recognize temps holding
1895     * broadcasted scalars but no way we can do it.
1896     * Tried asking llvm but without any success (using LLVMIsConstant
1897     * even though this isn't exactly what we'd need), even as simple as
1898     * IMM[0] UINT32 (0,-1,0,0)
1899     * MOV TEMP[0] IMM[0].yyyy
1900     * SVIEWINFO TEMP[1], TEMP[0].xxxx, SVIEWINFO[0]
1901     * doesn't work.
1902     * This means there's ZERO chance this will ever catch a scalar lod
1903     * with traditional tex opcodes as well as texel fetches, since the lod
1904     * comes from the same reg as coords (except some test shaders using
1905     * constant coords maybe).
1906     * There's at least hope for sample opcodes as well as size queries.
1907     */
1908    if (reg->Register.File == TGSI_FILE_CONSTANT ||
1909        reg->Register.File == TGSI_FILE_IMMEDIATE) {
1910       lod_property = LP_SAMPLER_LOD_SCALAR;
1911    }
1912    else if (bld_base->info->processor == TGSI_PROCESSOR_FRAGMENT) {
1913       if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
1914          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1915       }
1916       else {
1917          lod_property = LP_SAMPLER_LOD_PER_QUAD;
1918       }
1919    }
1920    else {
1921       /* never use scalar (per-quad) lod the results are just too wrong. */
1922       lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1923    }
1924    return lod_property;
1925 }
1926
1927
1928 /**
1929  * High-level instruction translators.
1930  */
1931
1932 static void
1933 emit_tex( struct lp_build_tgsi_soa_context *bld,
1934           const struct tgsi_full_instruction *inst,
1935           enum lp_build_tex_modifier modifier,
1936           LLVMValueRef *texel)
1937 {
1938    unsigned unit;
1939    LLVMValueRef lod_bias, explicit_lod;
1940    LLVMValueRef oow = NULL;
1941    LLVMValueRef coords[5];
1942    LLVMValueRef offsets[3] = { NULL };
1943    struct lp_derivatives derivs;
1944    struct lp_derivatives *deriv_ptr = NULL;
1945    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
1946    unsigned num_derivs, num_offsets, i;
1947    unsigned shadow_coord = 0;
1948    unsigned layer_coord = 0;
1949
1950    if (!bld->sampler) {
1951       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
1952       for (i = 0; i < 4; i++) {
1953          texel[i] = bld->bld_base.base.undef;
1954       }
1955       return;
1956    }
1957
1958    switch (inst->Texture.Texture) {
1959    case TGSI_TEXTURE_1D_ARRAY:
1960       layer_coord = 1;
1961       /* fallthrough */
1962    case TGSI_TEXTURE_1D:
1963       num_offsets = 1;
1964       num_derivs = 1;
1965       break;
1966    case TGSI_TEXTURE_2D_ARRAY:
1967       layer_coord = 2;
1968       /* fallthrough */
1969    case TGSI_TEXTURE_2D:
1970    case TGSI_TEXTURE_RECT:
1971       num_offsets = 2;
1972       num_derivs = 2;
1973       break;
1974    case TGSI_TEXTURE_SHADOW1D_ARRAY:
1975       layer_coord = 1;
1976       /* fallthrough */
1977    case TGSI_TEXTURE_SHADOW1D:
1978       shadow_coord = 2;
1979       num_offsets = 1;
1980       num_derivs = 1;
1981       break;
1982    case TGSI_TEXTURE_SHADOW2D_ARRAY:
1983       layer_coord = 2;
1984       shadow_coord = 3;
1985       num_offsets = 2;
1986       num_derivs = 2;
1987       break;
1988    case TGSI_TEXTURE_SHADOW2D:
1989    case TGSI_TEXTURE_SHADOWRECT:
1990       shadow_coord = 2;
1991       num_offsets = 2;
1992       num_derivs = 2;
1993       break;
1994    case TGSI_TEXTURE_CUBE:
1995       num_offsets = 2;
1996       num_derivs = 3;
1997       break;
1998    case TGSI_TEXTURE_3D:
1999       num_offsets = 3;
2000       num_derivs = 3;
2001       break;
2002    case TGSI_TEXTURE_SHADOWCUBE:
2003       shadow_coord = 3;
2004       num_offsets = 2;
2005       num_derivs = 3;
2006       break;
2007    case TGSI_TEXTURE_CUBE_ARRAY:
2008    case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2009    case TGSI_TEXTURE_2D_MSAA:
2010    case TGSI_TEXTURE_2D_ARRAY_MSAA:
2011    default:
2012       assert(0);
2013       return;
2014    }
2015
2016    /* Note lod and especially projected are illegal in a LOT of cases */
2017    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2018        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2019       LLVMValueRef lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2020       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2021          lod_bias = lod;
2022          explicit_lod = NULL;
2023       }
2024       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2025          lod_bias = NULL;
2026          explicit_lod = lod;
2027       }
2028       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2029    }
2030    else {
2031       lod_bias = NULL;
2032       explicit_lod = NULL;
2033    }
2034
2035    if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED) {
2036       oow = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2037       oow = lp_build_rcp(&bld->bld_base.base, oow);
2038    }
2039
2040    for (i = 0; i < num_derivs; i++) {
2041       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2042       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2043          coords[i] = lp_build_mul(&bld->bld_base.base, coords[i], oow);
2044    }
2045    for (i = num_derivs; i < 5; i++) {
2046       coords[i] = bld->bld_base.base.undef;
2047    }
2048
2049    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2050    if (layer_coord) {
2051       coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2052       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2053          coords[2] = lp_build_mul(&bld->bld_base.base, coords[2], oow);
2054    }
2055    /* Shadow coord occupies always 5th slot. */
2056    if (shadow_coord) {
2057       coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 0, shadow_coord);
2058       if (modifier == LP_BLD_TEX_MODIFIER_PROJECTED)
2059          coords[4] = lp_build_mul(&bld->bld_base.base, coords[4], oow);
2060    }
2061
2062    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2063       unsigned dim;
2064       for (dim = 0; dim < num_derivs; ++dim) {
2065          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 1, dim);
2066          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 2, dim);
2067       }
2068       deriv_ptr = &derivs;
2069       unit = inst->Src[3].Register.Index;
2070       /*
2071        * could also check all src regs if constant but I doubt such
2072        * cases exist in practice.
2073        */
2074       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2075          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2076             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2077          }
2078          else {
2079             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2080          }
2081       }
2082       else {
2083          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2084       }
2085    } else {
2086       unit = inst->Src[1].Register.Index;
2087    }
2088
2089    /* some advanced gather instructions (txgo) would require 4 offsets */
2090    if (inst->Texture.NumOffsets == 1) {
2091       unsigned dim;
2092       for (dim = 0; dim < num_offsets; dim++) {
2093          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2094       }
2095    }
2096
2097    bld->sampler->emit_fetch_texel(bld->sampler,
2098                                   bld->bld_base.base.gallivm,
2099                                   bld->bld_base.base.type,
2100                                   FALSE,
2101                                   unit, unit,
2102                                   coords,
2103                                   offsets,
2104                                   deriv_ptr,
2105                                   lod_bias, explicit_lod, lod_property,
2106                                   texel);
2107 }
2108
2109 static void
2110 emit_sample(struct lp_build_tgsi_soa_context *bld,
2111             const struct tgsi_full_instruction *inst,
2112             enum lp_build_tex_modifier modifier,
2113             boolean compare,
2114             LLVMValueRef *texel)
2115 {
2116    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2117    unsigned texture_unit, sampler_unit;
2118    LLVMValueRef lod_bias, explicit_lod;
2119    LLVMValueRef coords[5];
2120    LLVMValueRef offsets[3] = { NULL };
2121    struct lp_derivatives derivs;
2122    struct lp_derivatives *deriv_ptr = NULL;
2123    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2124
2125    unsigned num_offsets, num_derivs, i;
2126    unsigned layer_coord = 0;
2127
2128    if (!bld->sampler) {
2129       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2130       for (i = 0; i < 4; i++) {
2131          texel[i] = bld->bld_base.base.undef;
2132       }
2133       return;
2134    }
2135
2136    /*
2137     * unlike old-style tex opcodes the texture/sampler indices
2138     * always come from src1 and src2 respectively.
2139     */
2140    texture_unit = inst->Src[1].Register.Index;
2141    sampler_unit = inst->Src[2].Register.Index;
2142
2143    /*
2144     * Note inst->Texture.Texture will contain the number of offsets,
2145     * however the target information is NOT there and comes from the
2146     * declared sampler views instead.
2147     */
2148    switch (bld->sv[texture_unit].Resource) {
2149    case TGSI_TEXTURE_1D:
2150       num_offsets = 1;
2151       num_derivs = 1;
2152       break;
2153    case TGSI_TEXTURE_1D_ARRAY:
2154       layer_coord = 1;
2155       num_offsets = 1;
2156       num_derivs = 1;
2157       break;
2158    case TGSI_TEXTURE_2D:
2159    case TGSI_TEXTURE_RECT:
2160       num_offsets = 2;
2161       num_derivs = 2;
2162       break;
2163    case TGSI_TEXTURE_2D_ARRAY:
2164       layer_coord = 2;
2165       num_offsets = 2;
2166       num_derivs = 2;
2167       break;
2168    case TGSI_TEXTURE_CUBE:
2169       num_offsets = 2;
2170       num_derivs = 3;
2171       break;
2172    case TGSI_TEXTURE_3D:
2173       num_offsets = 3;
2174       num_derivs = 3;
2175       break;
2176    case TGSI_TEXTURE_CUBE_ARRAY:
2177       layer_coord = 3;
2178       num_offsets = 2;
2179       num_derivs = 3;
2180       break;
2181    default:
2182       assert(0);
2183       return;
2184    }
2185
2186    if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS ||
2187        modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2188       LLVMValueRef lod = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2189       if (modifier == LP_BLD_TEX_MODIFIER_LOD_BIAS) {
2190          lod_bias = lod;
2191          explicit_lod = NULL;
2192       }
2193       else if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_LOD) {
2194          lod_bias = NULL;
2195          explicit_lod = lod;
2196       }
2197       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2198    }
2199    else if (modifier == LP_BLD_TEX_MODIFIER_LOD_ZERO) {
2200       lod_bias = NULL;
2201       /* XXX might be better to explicitly pass the level zero information */
2202       explicit_lod = lp_build_const_vec(gallivm, bld->bld_base.base.type, 0.0F);
2203    }
2204    else {
2205       lod_bias = NULL;
2206       explicit_lod = NULL;
2207    }
2208
2209    for (i = 0; i < num_derivs; i++) {
2210       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2211    }
2212    for (i = num_derivs; i < 5; i++) {
2213       coords[i] = bld->bld_base.base.undef;
2214    }
2215
2216    /* Layer coord always goes into 3rd slot, except for cube map arrays */
2217    if (layer_coord) {
2218       if (layer_coord == 3)
2219          coords[3] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2220       else
2221          coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2222    }
2223    /* Shadow coord occupies always 5th slot. */
2224    if (compare) {
2225       coords[4] = lp_build_emit_fetch(&bld->bld_base, inst, 3, 0);
2226    }
2227
2228    if (modifier == LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV) {
2229       unsigned dim;
2230       for (dim = 0; dim < num_derivs; ++dim) {
2231          derivs.ddx[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 3, dim);
2232          derivs.ddy[dim] = lp_build_emit_fetch(&bld->bld_base, inst, 4, dim);
2233       }
2234       deriv_ptr = &derivs;
2235       /*
2236        * could also check all src regs if constant but I doubt such
2237        * cases exist in practice.
2238        */
2239       if (bld->bld_base.info->processor == TGSI_PROCESSOR_FRAGMENT) {
2240          if (gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) {
2241             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2242          }
2243          else {
2244             lod_property = LP_SAMPLER_LOD_PER_QUAD;
2245          }
2246       }
2247       else {
2248          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
2249       }
2250    }
2251
2252    /* some advanced gather instructions (txgo) would require 4 offsets */
2253    if (inst->Texture.NumOffsets == 1) {
2254       unsigned dim;
2255       for (dim = 0; dim < num_offsets; dim++) {
2256          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2257       }
2258    }
2259
2260    bld->sampler->emit_fetch_texel(bld->sampler,
2261                                   bld->bld_base.base.gallivm,
2262                                   bld->bld_base.base.type,
2263                                   FALSE,
2264                                   texture_unit, sampler_unit,
2265                                   coords,
2266                                   offsets,
2267                                   deriv_ptr,
2268                                   lod_bias, explicit_lod, lod_property,
2269                                   texel);
2270
2271    if (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2272        inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2273        inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2274        inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA) {
2275       unsigned char swizzles[4];
2276       swizzles[0] = inst->Src[1].Register.SwizzleX;
2277       swizzles[1] = inst->Src[1].Register.SwizzleY;
2278       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2279       swizzles[3] = inst->Src[1].Register.SwizzleW;
2280
2281       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2282    }
2283 }
2284
2285 static void
2286 emit_fetch_texels( struct lp_build_tgsi_soa_context *bld,
2287                    const struct tgsi_full_instruction *inst,
2288                    LLVMValueRef *texel,
2289                    boolean is_samplei)
2290 {
2291    unsigned unit, target;
2292    LLVMValueRef coord_undef = LLVMGetUndef(bld->bld_base.base.int_vec_type);
2293    LLVMValueRef explicit_lod = NULL;
2294    LLVMValueRef coords[3];
2295    LLVMValueRef offsets[3] = { NULL };
2296    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
2297    unsigned dims, i;
2298    unsigned layer_coord = 0;
2299
2300    if (!bld->sampler) {
2301       _debug_printf("warning: found texture instruction but no sampler generator supplied\n");
2302       for (i = 0; i < 4; i++) {
2303          texel[i] = coord_undef;
2304       }
2305       return;
2306    }
2307
2308    unit = inst->Src[1].Register.Index;
2309
2310    if (is_samplei) {
2311       target = bld->sv[unit].Resource;
2312    }
2313    else {
2314       target = inst->Texture.Texture;
2315    }
2316
2317    switch (target) {
2318    case TGSI_TEXTURE_1D:
2319    case TGSI_TEXTURE_BUFFER:
2320       dims = 1;
2321       break;
2322    case TGSI_TEXTURE_1D_ARRAY:
2323       layer_coord = 1;
2324       dims = 1;
2325       break;
2326    case TGSI_TEXTURE_2D:
2327    case TGSI_TEXTURE_RECT:
2328       dims = 2;
2329       break;
2330    case TGSI_TEXTURE_2D_ARRAY:
2331       layer_coord = 2;
2332       dims = 2;
2333       break;
2334    case TGSI_TEXTURE_3D:
2335       dims = 3;
2336       break;
2337    default:
2338       assert(0);
2339       return;
2340    }
2341
2342    /* always have lod except for buffers ? */
2343    if (target != TGSI_TEXTURE_BUFFER) {
2344       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 3);
2345       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2346    }
2347
2348    for (i = 0; i < dims; i++) {
2349       coords[i] = lp_build_emit_fetch(&bld->bld_base, inst, 0, i);
2350    }
2351    for (i = dims; i < 3; i++) {
2352       coords[i] = coord_undef;
2353    }
2354    if (layer_coord)
2355       coords[2] = lp_build_emit_fetch(&bld->bld_base, inst, 0, layer_coord);
2356
2357    if (inst->Texture.NumOffsets == 1) {
2358       unsigned dim;
2359       for (dim = 0; dim < dims; dim++) {
2360          offsets[dim] = lp_build_emit_fetch_texoffset(&bld->bld_base, inst, 0, dim);
2361       }
2362    }
2363
2364    bld->sampler->emit_fetch_texel(bld->sampler,
2365                                   bld->bld_base.base.gallivm,
2366                                   bld->bld_base.base.type,
2367                                   TRUE,
2368                                   unit, unit,
2369                                   coords,
2370                                   offsets,
2371                                   NULL,
2372                                   NULL, explicit_lod, lod_property,
2373                                   texel);
2374
2375    if (is_samplei &&
2376        (inst->Src[1].Register.SwizzleX != PIPE_SWIZZLE_RED ||
2377         inst->Src[1].Register.SwizzleY != PIPE_SWIZZLE_GREEN ||
2378         inst->Src[1].Register.SwizzleZ != PIPE_SWIZZLE_BLUE ||
2379         inst->Src[1].Register.SwizzleW != PIPE_SWIZZLE_ALPHA)) {
2380       unsigned char swizzles[4];
2381       swizzles[0] = inst->Src[1].Register.SwizzleX;
2382       swizzles[1] = inst->Src[1].Register.SwizzleY;
2383       swizzles[2] = inst->Src[1].Register.SwizzleZ;
2384       swizzles[3] = inst->Src[1].Register.SwizzleW;
2385
2386       lp_build_swizzle_soa_inplace(&bld->bld_base.base, texel, swizzles);
2387    }
2388 }
2389
2390 static void
2391 emit_size_query( struct lp_build_tgsi_soa_context *bld,
2392                  const struct tgsi_full_instruction *inst,
2393                  LLVMValueRef *sizes_out,
2394                  boolean is_sviewinfo)
2395 {
2396    LLVMValueRef explicit_lod;
2397    enum lp_sampler_lod_property lod_property;
2398    unsigned has_lod;
2399    unsigned i;
2400    unsigned unit = inst->Src[1].Register.Index;
2401    unsigned target, pipe_target;
2402
2403    if (is_sviewinfo) {
2404       target = bld->sv[unit].Resource;
2405    }
2406    else {
2407       target = inst->Texture.Texture;
2408    }
2409    switch (target) {
2410    case TGSI_TEXTURE_BUFFER:
2411    case TGSI_TEXTURE_RECT:
2412    case TGSI_TEXTURE_SHADOWRECT:
2413       has_lod = 0;
2414       break;
2415    default:
2416       has_lod = 1;
2417       break;
2418    }
2419
2420    if (!bld->sampler) {
2421       _debug_printf("warning: found texture query instruction but no sampler generator supplied\n");
2422       for (i = 0; i < 4; i++)
2423          sizes_out[i] = bld->bld_base.int_bld.undef;
2424       return;
2425    }
2426
2427    if (has_lod) {
2428       explicit_lod = lp_build_emit_fetch(&bld->bld_base, inst, 0, 0);
2429       lod_property = lp_build_lod_property(&bld->bld_base, inst, 0);
2430    }
2431    else {
2432       explicit_lod = NULL;
2433       lod_property = LP_SAMPLER_LOD_SCALAR;
2434    }
2435
2436
2437    pipe_target = tgsi_to_pipe_tex_target(target);
2438
2439    bld->sampler->emit_size_query(bld->sampler,
2440                                  bld->bld_base.base.gallivm,
2441                                  bld->bld_base.int_bld.type,
2442                                  unit, pipe_target,
2443                                  is_sviewinfo,
2444                                  lod_property,
2445                                  explicit_lod,
2446                                  sizes_out);
2447 }
2448
2449 static boolean
2450 near_end_of_shader(struct lp_build_tgsi_soa_context *bld,
2451                    int pc)
2452 {
2453    int i;
2454
2455    for (i = 0; i < 5; i++) {
2456       unsigned opcode;
2457
2458       if (pc + i >= bld->bld_base.info->num_instructions)
2459          return TRUE;
2460
2461       opcode = bld->bld_base.instructions[pc + i].Instruction.Opcode;
2462
2463       if (opcode == TGSI_OPCODE_END)
2464          return TRUE;
2465
2466       if (opcode == TGSI_OPCODE_TEX ||
2467           opcode == TGSI_OPCODE_TXP ||
2468           opcode == TGSI_OPCODE_TXD ||
2469           opcode == TGSI_OPCODE_TXB ||
2470           opcode == TGSI_OPCODE_TXL ||
2471           opcode == TGSI_OPCODE_TXF ||
2472           opcode == TGSI_OPCODE_TXQ ||
2473           opcode == TGSI_OPCODE_CAL ||
2474           opcode == TGSI_OPCODE_CALLNZ ||
2475           opcode == TGSI_OPCODE_IF ||
2476           opcode == TGSI_OPCODE_UIF ||
2477           opcode == TGSI_OPCODE_BGNLOOP ||
2478           opcode == TGSI_OPCODE_SWITCH)
2479          return FALSE;
2480    }
2481
2482    return TRUE;
2483 }
2484
2485
2486
2487 /**
2488  * Kill fragment if any of the src register values are negative.
2489  */
2490 static void
2491 emit_kill_if(
2492    struct lp_build_tgsi_soa_context *bld,
2493    const struct tgsi_full_instruction *inst,
2494    int pc)
2495 {
2496    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2497    const struct tgsi_full_src_register *reg = &inst->Src[0];
2498    LLVMValueRef terms[TGSI_NUM_CHANNELS];
2499    LLVMValueRef mask;
2500    unsigned chan_index;
2501
2502    memset(&terms, 0, sizeof terms);
2503
2504    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2505       unsigned swizzle;
2506
2507       /* Unswizzle channel */
2508       swizzle = tgsi_util_get_full_src_register_swizzle( reg, chan_index );
2509
2510       /* Check if the component has not been already tested. */
2511       assert(swizzle < TGSI_NUM_CHANNELS);
2512       if( !terms[swizzle] )
2513          /* TODO: change the comparison operator instead of setting the sign */
2514          terms[swizzle] =  lp_build_emit_fetch(&bld->bld_base, inst, 0, chan_index );
2515    }
2516
2517    mask = NULL;
2518    TGSI_FOR_EACH_CHANNEL( chan_index ) {
2519       if(terms[chan_index]) {
2520          LLVMValueRef chan_mask;
2521
2522          /*
2523           * If term < 0 then mask = 0 else mask = ~0.
2524           */
2525          chan_mask = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GEQUAL, terms[chan_index], bld->bld_base.base.zero);
2526
2527          if(mask)
2528             mask = LLVMBuildAnd(builder, mask, chan_mask, "");
2529          else
2530             mask = chan_mask;
2531       }
2532    }
2533
2534    if (bld->exec_mask.has_mask) {
2535       LLVMValueRef invmask;
2536       invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2537       mask = LLVMBuildOr(builder, mask, invmask, "");
2538    }
2539
2540    lp_build_mask_update(bld->mask, mask);
2541    if (!near_end_of_shader(bld, pc))
2542       lp_build_mask_check(bld->mask);
2543 }
2544
2545
2546 /**
2547  * Unconditional fragment kill.
2548  * The only predication is the execution mask which will apply if
2549  * we're inside a loop or conditional.
2550  */
2551 static void
2552 emit_kill(struct lp_build_tgsi_soa_context *bld,
2553           int pc)
2554 {
2555    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
2556    LLVMValueRef mask;
2557
2558    /* For those channels which are "alive", disable fragment shader
2559     * execution.
2560     */
2561    if (bld->exec_mask.has_mask) {
2562       mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
2563    }
2564    else {
2565       LLVMValueRef zero = LLVMConstNull(bld->bld_base.base.int_vec_type);
2566       mask = zero;
2567    }
2568
2569    lp_build_mask_update(bld->mask, mask);
2570
2571    if (!near_end_of_shader(bld, pc))
2572       lp_build_mask_check(bld->mask);
2573 }
2574
2575
2576 /**
2577  * Emit code which will dump the value of all the temporary registers
2578  * to stdout.
2579  */
2580 static void
2581 emit_dump_file(struct lp_build_tgsi_soa_context *bld,
2582                unsigned file)
2583 {
2584    const struct tgsi_shader_info *info = bld->bld_base.info;
2585    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2586    LLVMBuilderRef builder = gallivm->builder;
2587    LLVMValueRef reg_ptr;
2588    int index;
2589    int max_index = info->file_max[file];
2590
2591    /*
2592     * Some register files, particularly constants, can be very large,
2593     * and dumping everything could make this unusably slow.
2594     */
2595    max_index = MIN2(max_index, 32);
2596
2597    for (index = 0; index <= max_index; index++) {
2598       LLVMValueRef res;
2599       unsigned mask;
2600       int chan;
2601
2602       if (index < 8 * sizeof(unsigned) &&
2603           (info->file_mask[file] & (1 << index)) == 0)  {
2604          /* This was not declared.*/
2605          continue;
2606       }
2607
2608       if (file == TGSI_FILE_INPUT) {
2609          mask = info->input_usage_mask[index];
2610       } else {
2611          mask = TGSI_WRITEMASK_XYZW;
2612       }
2613
2614       for (chan = 0; chan < 4; chan++) {
2615          if ((mask & (1 << chan)) == 0) {
2616             /* This channel is not used.*/
2617             continue;
2618          }
2619
2620          if (file == TGSI_FILE_CONSTANT) {
2621             struct tgsi_full_src_register reg;
2622             memset(&reg, 0, sizeof reg);
2623             reg.Register.File = file;
2624             reg.Register.Index = index;
2625             reg.Register.SwizzleX = 0;
2626             reg.Register.SwizzleY = 1;
2627             reg.Register.SwizzleZ = 2;
2628             reg.Register.SwizzleW = 3;
2629
2630             res = bld->bld_base.emit_fetch_funcs[file](&bld->bld_base, &reg, TGSI_TYPE_FLOAT, chan);
2631             if (!res) {
2632                continue;
2633             }
2634          } else if (file == TGSI_FILE_INPUT) {
2635             res = bld->inputs[index][chan];
2636             if (!res) {
2637                continue;
2638             }
2639          } else if (file == TGSI_FILE_TEMPORARY) {
2640             reg_ptr = lp_get_temp_ptr_soa(bld, index, chan);
2641             assert(reg_ptr);
2642             res = LLVMBuildLoad(builder, reg_ptr, "");
2643          } else if (file == TGSI_FILE_OUTPUT) {
2644             reg_ptr = lp_get_output_ptr(bld, index, chan);
2645             assert(reg_ptr);
2646             res = LLVMBuildLoad(builder, reg_ptr, "");
2647          } else {
2648             assert(0);
2649             continue;
2650          }
2651
2652          emit_dump_reg(gallivm, file, index, chan, res);
2653       }
2654    }
2655 }
2656
2657
2658
2659 void
2660 lp_emit_declaration_soa(
2661    struct lp_build_tgsi_context *bld_base,
2662    const struct tgsi_full_declaration *decl)
2663 {
2664    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2665    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2666    LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
2667    const unsigned first = decl->Range.First;
2668    const unsigned last = decl->Range.Last;
2669    unsigned idx, i;
2670
2671    for (idx = first; idx <= last; ++idx) {
2672       assert(last <= bld->bld_base.info->file_max[decl->Declaration.File]);
2673       switch (decl->Declaration.File) {
2674       case TGSI_FILE_TEMPORARY:
2675          if (!(bld->indirect_files & (1 << TGSI_FILE_TEMPORARY))) {
2676             assert(idx < LP_MAX_INLINED_TEMPS);
2677             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2678                bld->temps[idx][i] = lp_build_alloca(gallivm, vec_type, "temp");
2679          }
2680          break;
2681
2682       case TGSI_FILE_OUTPUT:
2683          if (!(bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {
2684             for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2685                bld->outputs[idx][i] = lp_build_alloca(gallivm,
2686                                                       vec_type, "output");
2687          }
2688          break;
2689
2690       case TGSI_FILE_ADDRESS:
2691          /* ADDR registers are only allocated with an integer LLVM IR type,
2692           * as they are guaranteed to always have integers.
2693           * XXX: Not sure if this exception is worthwhile (or the whole idea of
2694           * an ADDR register for that matter).
2695           */
2696          assert(idx < LP_MAX_TGSI_ADDRS);
2697          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2698             bld->addr[idx][i] = lp_build_alloca(gallivm, bld_base->base.int_vec_type, "addr");
2699          break;
2700
2701       case TGSI_FILE_PREDICATE:
2702          assert(idx < LP_MAX_TGSI_PREDS);
2703          for (i = 0; i < TGSI_NUM_CHANNELS; i++)
2704             bld->preds[idx][i] = lp_build_alloca(gallivm, vec_type,
2705                                                  "predicate");
2706          break;
2707
2708       case TGSI_FILE_SAMPLER_VIEW:
2709          /*
2710           * The target stored here MUST match whatever there actually
2711           * is in the set sampler views (what about return type?).
2712           */
2713          assert(idx < PIPE_MAX_SHADER_SAMPLER_VIEWS);
2714          bld->sv[idx] = decl->SamplerView;
2715          break;
2716
2717       default:
2718          /* don't need to declare other vars */
2719          break;
2720       }
2721    }
2722 }
2723
2724
2725 void lp_emit_immediate_soa(
2726    struct lp_build_tgsi_context *bld_base,
2727    const struct tgsi_full_immediate *imm)
2728 {
2729    struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2730    struct gallivm_state * gallivm = bld_base->base.gallivm;
2731
2732    /* simply copy the immediate values into the next immediates[] slot */
2733    unsigned i;
2734    const uint size = imm->Immediate.NrTokens - 1;
2735    assert(size <= 4);
2736    assert(bld->num_immediates < LP_MAX_TGSI_IMMEDIATES);
2737    switch (imm->Immediate.DataType) {
2738    case TGSI_IMM_FLOAT32:
2739       for( i = 0; i < size; ++i )
2740          bld->immediates[bld->num_immediates][i] =
2741             lp_build_const_vec(gallivm, bld_base->base.type, imm->u[i].Float);
2742
2743       break;
2744    case TGSI_IMM_UINT32:
2745       for( i = 0; i < size; ++i ) {
2746          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->uint_bld.type, imm->u[i].Uint);
2747          bld->immediates[bld->num_immediates][i] =
2748             LLVMConstBitCast(tmp, bld_base->base.vec_type);
2749       }
2750
2751       break;
2752    case TGSI_IMM_INT32:
2753       for( i = 0; i < size; ++i ) {
2754          LLVMValueRef tmp = lp_build_const_vec(gallivm, bld_base->int_bld.type, imm->u[i].Int);
2755          bld->immediates[bld->num_immediates][i] =
2756             LLVMConstBitCast(tmp, bld_base->base.vec_type);
2757       }
2758
2759       break;
2760    }
2761    for( i = size; i < 4; ++i )
2762       bld->immediates[bld->num_immediates][i] = bld_base->base.undef;
2763
2764    if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
2765       unsigned index = bld->num_immediates;
2766       struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
2767       LLVMBuilderRef builder = gallivm->builder;
2768       for (i = 0; i < 4; ++i ) {
2769          LLVMValueRef lindex = lp_build_const_int32(
2770             bld->bld_base.base.gallivm, index * 4 + i);
2771          LLVMValueRef imm_ptr = LLVMBuildGEP(builder,
2772                                              bld->imms_array, &lindex, 1, "");
2773          LLVMBuildStore(builder,
2774                         bld->immediates[index][i],
2775                         imm_ptr);
2776       }
2777    }
2778
2779    bld->num_immediates++;
2780 }
2781
2782 static void
2783 ddx_emit(
2784    const struct lp_build_tgsi_action * action,
2785    struct lp_build_tgsi_context * bld_base,
2786    struct lp_build_emit_data * emit_data)
2787 {
2788    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2789
2790    emit_fetch_deriv(bld, emit_data->args[0], NULL,
2791                     &emit_data->output[emit_data->chan], NULL);
2792 }
2793
2794 static void
2795 ddy_emit(
2796    const struct lp_build_tgsi_action * action,
2797    struct lp_build_tgsi_context * bld_base,
2798    struct lp_build_emit_data * emit_data)
2799 {
2800    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2801
2802    emit_fetch_deriv(bld, emit_data->args[0], NULL, NULL,
2803                     &emit_data->output[emit_data->chan]);
2804 }
2805
2806 static void
2807 kill_emit(
2808    const struct lp_build_tgsi_action * action,
2809    struct lp_build_tgsi_context * bld_base,
2810    struct lp_build_emit_data * emit_data)
2811 {
2812    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2813
2814    emit_kill(bld, bld_base->pc - 1);
2815 }
2816
2817 static void
2818 kill_if_emit(
2819    const struct lp_build_tgsi_action * action,
2820    struct lp_build_tgsi_context * bld_base,
2821    struct lp_build_emit_data * emit_data)
2822 {
2823    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2824
2825    emit_kill_if(bld, emit_data->inst, bld_base->pc - 1);
2826 }
2827
2828 static void
2829 tex_emit(
2830    const struct lp_build_tgsi_action * action,
2831    struct lp_build_tgsi_context * bld_base,
2832    struct lp_build_emit_data * emit_data)
2833 {
2834    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2835
2836    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE, emit_data->output);
2837 }
2838
2839 static void
2840 txb_emit(
2841    const struct lp_build_tgsi_action * action,
2842    struct lp_build_tgsi_context * bld_base,
2843    struct lp_build_emit_data * emit_data)
2844 {
2845    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2846
2847    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
2848             emit_data->output);
2849 }
2850
2851 static void
2852 txd_emit(
2853    const struct lp_build_tgsi_action * action,
2854    struct lp_build_tgsi_context * bld_base,
2855    struct lp_build_emit_data * emit_data)
2856 {
2857    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2858
2859    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
2860             emit_data->output);
2861 }
2862
2863 static void
2864 txl_emit(
2865    const struct lp_build_tgsi_action * action,
2866    struct lp_build_tgsi_context * bld_base,
2867    struct lp_build_emit_data * emit_data)
2868 {
2869    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2870
2871    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
2872             emit_data->output);
2873 }
2874
2875 static void
2876 txp_emit(
2877    const struct lp_build_tgsi_action * action,
2878    struct lp_build_tgsi_context * bld_base,
2879    struct lp_build_emit_data * emit_data)
2880 {
2881    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2882
2883    emit_tex(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_PROJECTED,
2884             emit_data->output);
2885 }
2886
2887 static void
2888 txq_emit(
2889    const struct lp_build_tgsi_action * action,
2890    struct lp_build_tgsi_context * bld_base,
2891    struct lp_build_emit_data * emit_data)
2892 {
2893    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2894
2895    emit_size_query(bld, emit_data->inst, emit_data->output, FALSE);
2896 }
2897
2898 static void
2899 txf_emit(
2900    const struct lp_build_tgsi_action * action,
2901    struct lp_build_tgsi_context * bld_base,
2902    struct lp_build_emit_data * emit_data)
2903 {
2904    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2905
2906    emit_fetch_texels(bld, emit_data->inst, emit_data->output, FALSE);
2907 }
2908
2909 static void
2910 sample_i_emit(
2911    const struct lp_build_tgsi_action * action,
2912    struct lp_build_tgsi_context * bld_base,
2913    struct lp_build_emit_data * emit_data)
2914 {
2915    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2916
2917    emit_fetch_texels(bld, emit_data->inst, emit_data->output, TRUE);
2918 }
2919
2920 static void
2921 sample_emit(
2922    const struct lp_build_tgsi_action * action,
2923    struct lp_build_tgsi_context * bld_base,
2924    struct lp_build_emit_data * emit_data)
2925 {
2926    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2927
2928    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
2929                FALSE, emit_data->output);
2930 }
2931
2932 static void
2933 sample_b_emit(
2934    const struct lp_build_tgsi_action * action,
2935    struct lp_build_tgsi_context * bld_base,
2936    struct lp_build_emit_data * emit_data)
2937 {
2938    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2939
2940    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_BIAS,
2941                FALSE, emit_data->output);
2942 }
2943
2944 static void
2945 sample_c_emit(
2946    const struct lp_build_tgsi_action * action,
2947    struct lp_build_tgsi_context * bld_base,
2948    struct lp_build_emit_data * emit_data)
2949 {
2950    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2951
2952    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_NONE,
2953                TRUE, emit_data->output);
2954 }
2955
2956 static void
2957 sample_c_lz_emit(
2958    const struct lp_build_tgsi_action * action,
2959    struct lp_build_tgsi_context * bld_base,
2960    struct lp_build_emit_data * emit_data)
2961 {
2962    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2963
2964    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_LOD_ZERO,
2965                TRUE, emit_data->output);
2966 }
2967
2968 static void
2969 sample_d_emit(
2970    const struct lp_build_tgsi_action * action,
2971    struct lp_build_tgsi_context * bld_base,
2972    struct lp_build_emit_data * emit_data)
2973 {
2974    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2975
2976    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV,
2977                FALSE, emit_data->output);
2978 }
2979
2980 static void
2981 sample_l_emit(
2982    const struct lp_build_tgsi_action * action,
2983    struct lp_build_tgsi_context * bld_base,
2984    struct lp_build_emit_data * emit_data)
2985 {
2986    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2987
2988    emit_sample(bld, emit_data->inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD,
2989                FALSE, emit_data->output);
2990 }
2991
2992 static void
2993 sviewinfo_emit(
2994    const struct lp_build_tgsi_action * action,
2995    struct lp_build_tgsi_context * bld_base,
2996    struct lp_build_emit_data * emit_data)
2997 {
2998    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
2999
3000    emit_size_query(bld, emit_data->inst, emit_data->output, TRUE);
3001 }
3002
3003 static LLVMValueRef
3004 mask_vec(struct lp_build_tgsi_context *bld_base)
3005 {
3006    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3007    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3008    struct lp_exec_mask *exec_mask = &bld->exec_mask;
3009
3010    if (!exec_mask->has_mask) {
3011       return lp_build_mask_value(bld->mask);
3012    }
3013    return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
3014                        exec_mask->exec_mask, "");
3015 }
3016
3017 static void
3018 increment_vec_ptr_by_mask(struct lp_build_tgsi_context * bld_base,
3019                           LLVMValueRef ptr,
3020                           LLVMValueRef mask)
3021 {
3022    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3023    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3024
3025    current_vec = LLVMBuildSub(builder, current_vec, mask, "");
3026
3027    LLVMBuildStore(builder, current_vec, ptr);
3028 }
3029
3030 static void
3031 clear_uint_vec_ptr_from_mask(struct lp_build_tgsi_context * bld_base,
3032                              LLVMValueRef ptr,
3033                              LLVMValueRef mask)
3034 {
3035    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3036    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
3037
3038    current_vec = lp_build_select(&bld_base->uint_bld,
3039                                  mask,
3040                                  bld_base->uint_bld.zero,
3041                                  current_vec);
3042
3043    LLVMBuildStore(builder, current_vec, ptr);
3044 }
3045
3046 static LLVMValueRef
3047 clamp_mask_to_max_output_vertices(struct lp_build_tgsi_soa_context * bld,
3048                                   LLVMValueRef current_mask_vec,
3049                                   LLVMValueRef total_emitted_vertices_vec)
3050 {
3051    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3052    struct lp_build_context *int_bld = &bld->bld_base.int_bld;
3053    LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
3054                                         total_emitted_vertices_vec,
3055                                         bld->max_output_vertices_vec);
3056
3057    return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
3058 }
3059
3060 static void
3061 emit_vertex(
3062    const struct lp_build_tgsi_action * action,
3063    struct lp_build_tgsi_context * bld_base,
3064    struct lp_build_emit_data * emit_data)
3065 {
3066    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3067    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3068
3069    if (bld->gs_iface->emit_vertex) {
3070       LLVMValueRef mask = mask_vec(bld_base);
3071       LLVMValueRef total_emitted_vertices_vec =
3072          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3073       mask = clamp_mask_to_max_output_vertices(bld, mask,
3074                                                total_emitted_vertices_vec);
3075       gather_outputs(bld);
3076       bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base,
3077                                  bld->outputs,
3078                                  total_emitted_vertices_vec);
3079       increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr,
3080                                 mask);
3081       increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr,
3082                                 mask);
3083 #if DUMP_GS_EMITS
3084       lp_build_print_value(bld->bld_base.base.gallivm,
3085                            " +++ emit vertex masked ones = ",
3086                            mask);
3087       lp_build_print_value(bld->bld_base.base.gallivm,
3088                            " +++ emit vertex emitted = ",
3089                            total_emitted_vertices_vec);
3090 #endif
3091    }
3092 }
3093
3094
3095 static void
3096 end_primitive_masked(struct lp_build_tgsi_context * bld_base,
3097                      LLVMValueRef mask)
3098 {
3099    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3100    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
3101
3102    if (bld->gs_iface->end_primitive) {
3103       struct lp_build_context *uint_bld = &bld_base->uint_bld;
3104       LLVMValueRef emitted_vertices_vec =
3105          LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr, "");
3106       LLVMValueRef emitted_prims_vec =
3107          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3108
3109       LLVMValueRef emitted_mask = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3110                                                emitted_vertices_vec,
3111                                                uint_bld->zero);
3112       /* We need to combine the current execution mask with the mask
3113          telling us which, if any, execution slots actually have
3114          unemitted primitives, this way we make sure that end_primitives
3115          executes only on the paths that have unflushed vertices */
3116       mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
3117
3118       bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base,
3119                                    emitted_vertices_vec,
3120                                    emitted_prims_vec);
3121
3122 #if DUMP_GS_EMITS
3123       lp_build_print_value(bld->bld_base.base.gallivm,
3124                            " +++ end prim masked ones = ",
3125                            mask);
3126       lp_build_print_value(bld->bld_base.base.gallivm,
3127                            " +++ end prim emitted verts1 = ",
3128                            emitted_vertices_vec);
3129       lp_build_print_value(bld->bld_base.base.gallivm,
3130                            " +++ end prim emitted prims1 = ",
3131                            LLVMBuildLoad(builder,
3132                                          bld->emitted_prims_vec_ptr, ""));
3133 #endif
3134       increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr,
3135                                 mask);
3136       clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr,
3137                                    mask);
3138 #if DUMP_GS_EMITS
3139       lp_build_print_value(bld->bld_base.base.gallivm,
3140                            " +++ end prim emitted verts2 = ",
3141                            LLVMBuildLoad(builder,
3142                                          bld->emitted_vertices_vec_ptr, ""));
3143 #endif
3144    }
3145
3146 }
3147
3148 static void
3149 end_primitive(
3150    const struct lp_build_tgsi_action * action,
3151    struct lp_build_tgsi_context * bld_base,
3152    struct lp_build_emit_data * emit_data)
3153 {
3154    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3155
3156    if (bld->gs_iface->end_primitive) {
3157       LLVMValueRef mask = mask_vec(bld_base);
3158       end_primitive_masked(bld_base, mask);
3159    }
3160 }
3161
3162 static void
3163 cal_emit(
3164    const struct lp_build_tgsi_action * action,
3165    struct lp_build_tgsi_context * bld_base,
3166    struct lp_build_emit_data * emit_data)
3167 {
3168    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3169
3170    lp_exec_mask_call(&bld->exec_mask, emit_data->inst->Label.Label,
3171                      &bld_base->pc);
3172 }
3173
3174 static void
3175 ret_emit(
3176    const struct lp_build_tgsi_action * action,
3177    struct lp_build_tgsi_context * bld_base,
3178    struct lp_build_emit_data * emit_data)
3179 {
3180    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3181
3182    lp_exec_mask_ret(&bld->exec_mask, &bld_base->pc);
3183 }
3184
3185 static void
3186 brk_emit(
3187    const struct lp_build_tgsi_action * action,
3188    struct lp_build_tgsi_context * bld_base,
3189    struct lp_build_emit_data * emit_data)
3190 {
3191    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3192
3193    lp_exec_break(&bld->exec_mask, bld_base);
3194 }
3195
3196 static void
3197 breakc_emit(
3198    const struct lp_build_tgsi_action * action,
3199    struct lp_build_tgsi_context * bld_base,
3200    struct lp_build_emit_data * emit_data)
3201 {
3202    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3203    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3204    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3205    LLVMValueRef unsigned_cond =
3206       LLVMBuildBitCast(builder, emit_data->args[0], uint_bld->vec_type, "");
3207    LLVMValueRef cond = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3208                                     unsigned_cond,
3209                                     uint_bld->zero);
3210
3211    lp_exec_break_condition(&bld->exec_mask, cond);
3212 }
3213
3214 static void
3215 if_emit(
3216    const struct lp_build_tgsi_action * action,
3217    struct lp_build_tgsi_context * bld_base,
3218    struct lp_build_emit_data * emit_data)
3219 {
3220    LLVMValueRef tmp;
3221    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3222
3223    tmp = lp_build_cmp(&bld_base->base, PIPE_FUNC_NOTEQUAL,
3224                       emit_data->args[0], bld->bld_base.base.zero);
3225    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3226 }
3227
3228 static void
3229 uif_emit(
3230    const struct lp_build_tgsi_action * action,
3231    struct lp_build_tgsi_context * bld_base,
3232    struct lp_build_emit_data * emit_data)
3233 {
3234    LLVMValueRef tmp;
3235    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3236    struct lp_build_context *uint_bld = &bld_base->uint_bld;
3237
3238    tmp = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
3239                       emit_data->args[0], uint_bld->zero);
3240    lp_exec_mask_cond_push(&bld->exec_mask, tmp);
3241 }
3242
3243 static void
3244 case_emit(
3245    const struct lp_build_tgsi_action * action,
3246    struct lp_build_tgsi_context * bld_base,
3247    struct lp_build_emit_data * emit_data)
3248 {
3249    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3250
3251    lp_exec_case(&bld->exec_mask, emit_data->args[0]);
3252 }
3253
3254 static void
3255 default_emit(
3256    const struct lp_build_tgsi_action * action,
3257    struct lp_build_tgsi_context * bld_base,
3258    struct lp_build_emit_data * emit_data)
3259 {
3260    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3261
3262    lp_exec_default(&bld->exec_mask, bld_base);
3263 }
3264
3265 static void
3266 switch_emit(
3267    const struct lp_build_tgsi_action * action,
3268    struct lp_build_tgsi_context * bld_base,
3269    struct lp_build_emit_data * emit_data)
3270 {
3271    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3272
3273    lp_exec_switch(&bld->exec_mask, emit_data->args[0]);
3274 }
3275
3276 static void
3277 endswitch_emit(
3278    const struct lp_build_tgsi_action * action,
3279    struct lp_build_tgsi_context * bld_base,
3280    struct lp_build_emit_data * emit_data)
3281 {
3282    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3283
3284    lp_exec_endswitch(&bld->exec_mask, bld_base);
3285 }
3286
3287 static void
3288 bgnloop_emit(
3289    const struct lp_build_tgsi_action * action,
3290    struct lp_build_tgsi_context * bld_base,
3291    struct lp_build_emit_data * emit_data)
3292 {
3293    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3294
3295    lp_exec_bgnloop(&bld->exec_mask);
3296 }
3297
3298 static void
3299 bgnsub_emit(
3300    const struct lp_build_tgsi_action * action,
3301    struct lp_build_tgsi_context * bld_base,
3302    struct lp_build_emit_data * emit_data)
3303 {
3304    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3305
3306    lp_exec_mask_bgnsub(&bld->exec_mask);
3307 }
3308
3309 static void
3310 else_emit(
3311    const struct lp_build_tgsi_action * action,
3312    struct lp_build_tgsi_context * bld_base,
3313    struct lp_build_emit_data * emit_data)
3314 {
3315    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3316
3317    lp_exec_mask_cond_invert(&bld->exec_mask);
3318 }
3319
3320 static void
3321 endif_emit(
3322    const struct lp_build_tgsi_action * action,
3323    struct lp_build_tgsi_context * bld_base,
3324    struct lp_build_emit_data * emit_data)
3325 {
3326    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3327
3328    lp_exec_mask_cond_pop(&bld->exec_mask);
3329 }
3330
3331 static void
3332 endloop_emit(
3333    const struct lp_build_tgsi_action * action,
3334    struct lp_build_tgsi_context * bld_base,
3335    struct lp_build_emit_data * emit_data)
3336 {
3337    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3338
3339    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
3340 }
3341
3342 static void
3343 endsub_emit(
3344    const struct lp_build_tgsi_action * action,
3345    struct lp_build_tgsi_context * bld_base,
3346    struct lp_build_emit_data * emit_data)
3347 {
3348    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3349
3350    lp_exec_mask_endsub(&bld->exec_mask, &bld_base->pc);
3351 }
3352
3353 static void
3354 cont_emit(
3355    const struct lp_build_tgsi_action * action,
3356    struct lp_build_tgsi_context * bld_base,
3357    struct lp_build_emit_data * emit_data)
3358 {
3359    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3360
3361    lp_exec_continue(&bld->exec_mask);
3362 }
3363
3364 /* XXX: Refactor and move it to lp_bld_tgsi_action.c
3365  *
3366  * XXX: What do the comments about xmm registers mean?  Maybe they are left over
3367  * from old code, but there is no garauntee that LLVM will use those registers
3368  * for this code.
3369  *
3370  * XXX: There should be no calls to lp_build_emit_fetch in this function.  This
3371  * should be handled by the emit_data->fetch_args function. */
3372 static void
3373 nrm_emit(
3374    const struct lp_build_tgsi_action * action,
3375    struct lp_build_tgsi_context * bld_base,
3376    struct lp_build_emit_data * emit_data)
3377 {
3378    LLVMValueRef tmp0, tmp1;
3379    LLVMValueRef tmp4 = NULL;
3380    LLVMValueRef tmp5 = NULL;
3381    LLVMValueRef tmp6 = NULL;
3382    LLVMValueRef tmp7 = NULL;
3383    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3384
3385    uint dims = (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_NRM) ? 3 : 4;
3386
3387   if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X) ||
3388       TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Y) ||
3389       TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Z) ||
3390       (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_W) && dims == 4)) {
3391
3392       /* NOTE: Cannot use xmm regs 2/3 here (see emit_rsqrt() above). */
3393
3394       /* xmm4 = src.x */
3395       /* xmm0 = src.x * src.x */
3396       tmp0 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_X);
3397       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X)) {
3398          tmp4 = tmp0;
3399       }
3400       tmp0 = lp_build_mul( &bld->bld_base.base, tmp0, tmp0);
3401
3402       /* xmm5 = src.y */
3403       /* xmm0 = xmm0 + src.y * src.y */
3404       tmp1 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_Y);
3405       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Y)) {
3406          tmp5 = tmp1;
3407       }
3408       tmp1 = lp_build_mul( &bld->bld_base.base, tmp1, tmp1);
3409       tmp0 = lp_build_add( &bld->bld_base.base, tmp0, tmp1);
3410
3411       /* xmm6 = src.z */
3412       /* xmm0 = xmm0 + src.z * src.z */
3413       tmp1 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_Z);
3414       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Z)) {
3415          tmp6 = tmp1;
3416       }
3417       tmp1 = lp_build_mul( &bld->bld_base.base, tmp1, tmp1);
3418       tmp0 = lp_build_add( &bld->bld_base.base, tmp0, tmp1);
3419
3420       if (dims == 4) {
3421          /* xmm7 = src.w */
3422          /* xmm0 = xmm0 + src.w * src.w */
3423          tmp1 = lp_build_emit_fetch(&bld->bld_base, emit_data->inst, 0, TGSI_CHAN_W);
3424          if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_W)) {
3425             tmp7 = tmp1;
3426          }
3427          tmp1 = lp_build_mul( &bld->bld_base.base, tmp1, tmp1);
3428          tmp0 = lp_build_add( &bld->bld_base.base, tmp0, tmp1);
3429       }
3430       /* xmm1 = 1 / sqrt(xmm0) */
3431       tmp1 = lp_build_rsqrt( &bld->bld_base.base, tmp0);
3432        /* dst.x = xmm1 * src.x */
3433       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X)) {
3434          emit_data->output[TGSI_CHAN_X] = lp_build_mul( &bld->bld_base.base, tmp4, tmp1);
3435       }
3436       /* dst.y = xmm1 * src.y */
3437       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Y)) {
3438          emit_data->output[TGSI_CHAN_Y] = lp_build_mul( &bld->bld_base.base, tmp5, tmp1);
3439       }
3440
3441       /* dst.z = xmm1 * src.z */
3442       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_Z)) {
3443          emit_data->output[TGSI_CHAN_Z] = lp_build_mul( &bld->bld_base.base, tmp6, tmp1);
3444       }
3445       /* dst.w = xmm1 * src.w */
3446       if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_X) && dims == 4) {
3447          emit_data->output[TGSI_CHAN_W] = lp_build_mul( &bld->bld_base.base, tmp7, tmp1);
3448       }
3449    }
3450
3451    /* dst.w = 1.0 */
3452    if (TGSI_IS_DST0_CHANNEL_ENABLED(emit_data->inst, TGSI_CHAN_W) && dims == 3) {
3453        emit_data->output[TGSI_CHAN_W] = bld->bld_base.base.one;
3454    }
3455 }
3456
3457 static void emit_prologue(struct lp_build_tgsi_context * bld_base)
3458 {
3459    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3460    struct gallivm_state * gallivm = bld_base->base.gallivm;
3461
3462    if (bld->indirect_files & (1 << TGSI_FILE_TEMPORARY)) {
3463       LLVMValueRef array_size =
3464          lp_build_const_int32(gallivm,
3465                          bld_base->info->file_max[TGSI_FILE_TEMPORARY] * 4 + 4);
3466       bld->temps_array = lp_build_array_alloca(gallivm,
3467                                               bld_base->base.vec_type, array_size,
3468                                               "temp_array");
3469    }
3470
3471    if (bld->indirect_files & (1 << TGSI_FILE_OUTPUT)) {
3472       LLVMValueRef array_size =
3473          lp_build_const_int32(gallivm,
3474                             bld_base->info->file_max[TGSI_FILE_OUTPUT] * 4 + 4);
3475       bld->outputs_array = lp_build_array_alloca(gallivm,
3476                                                 bld_base->base.vec_type, array_size,
3477                                                 "output_array");
3478    }
3479
3480    if (bld->indirect_files & (1 << TGSI_FILE_IMMEDIATE)) {
3481       LLVMValueRef array_size =
3482          lp_build_const_int32(gallivm,
3483                          bld_base->info->file_max[TGSI_FILE_IMMEDIATE] * 4 + 4);
3484       bld->imms_array = lp_build_array_alloca(gallivm,
3485                                               bld_base->base.vec_type, array_size,
3486                                               "imms_array");
3487    }
3488
3489    /* If we have indirect addressing in inputs we need to copy them into
3490     * our alloca array to be able to iterate over them */
3491    if (bld->indirect_files & (1 << TGSI_FILE_INPUT) && !bld->gs_iface) {
3492       unsigned index, chan;
3493       LLVMTypeRef vec_type = bld_base->base.vec_type;
3494       LLVMValueRef array_size = lp_build_const_int32(gallivm,
3495             bld_base->info->file_max[TGSI_FILE_INPUT]*4 + 4);
3496       bld->inputs_array = lp_build_array_alloca(gallivm,
3497                                                vec_type, array_size,
3498                                                "input_array");
3499
3500       assert(bld_base->info->num_inputs
3501                         <= bld_base->info->file_max[TGSI_FILE_INPUT] + 1);
3502
3503       for (index = 0; index < bld_base->info->num_inputs; ++index) {
3504          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3505             LLVMValueRef lindex =
3506                lp_build_const_int32(gallivm, index * 4 + chan);
3507             LLVMValueRef input_ptr =
3508                LLVMBuildGEP(gallivm->builder, bld->inputs_array,
3509                             &lindex, 1, "");
3510             LLVMValueRef value = bld->inputs[index][chan];
3511             if (value)
3512                LLVMBuildStore(gallivm->builder, value, input_ptr);
3513          }
3514       }
3515    }
3516
3517    if (bld->gs_iface) {
3518       struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;
3519       bld->emitted_prims_vec_ptr =
3520          lp_build_alloca(gallivm,
3521                          uint_bld->vec_type,
3522                          "emitted_prims_ptr");
3523       bld->emitted_vertices_vec_ptr =
3524          lp_build_alloca(gallivm,
3525                          uint_bld->vec_type,
3526                          "emitted_vertices_ptr");
3527       bld->total_emitted_vertices_vec_ptr =
3528          lp_build_alloca(gallivm,
3529                          uint_bld->vec_type,
3530                          "total_emitted_vertices_ptr");
3531
3532       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3533                      bld->emitted_prims_vec_ptr);
3534       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3535                      bld->emitted_vertices_vec_ptr);
3536       LLVMBuildStore(gallivm->builder, uint_bld->zero,
3537                      bld->total_emitted_vertices_vec_ptr);
3538    }
3539
3540    if (DEBUG_EXECUTION) {
3541       lp_build_printf(gallivm, "\n");
3542       emit_dump_file(bld, TGSI_FILE_CONSTANT);
3543       emit_dump_file(bld, TGSI_FILE_INPUT);
3544    }
3545 }
3546
3547 static void emit_epilogue(struct lp_build_tgsi_context * bld_base)
3548 {
3549    struct lp_build_tgsi_soa_context * bld = lp_soa_context(bld_base);
3550    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3551
3552    if (DEBUG_EXECUTION) {
3553       /* for debugging */
3554       if (0) {
3555          emit_dump_file(bld, TGSI_FILE_TEMPORARY);
3556       }
3557       emit_dump_file(bld, TGSI_FILE_OUTPUT);
3558       lp_build_printf(bld_base->base.gallivm, "\n");
3559    }
3560
3561    /* If we have indirect addressing in outputs we need to copy our alloca array
3562     * to the outputs slots specified by the caller */
3563    if (bld->gs_iface) {
3564       LLVMValueRef total_emitted_vertices_vec;
3565       LLVMValueRef emitted_prims_vec;
3566       /* implicit end_primitives, needed in case there are any unflushed
3567          vertices in the cache. Note must not call end_primitive here
3568          since the exec_mask is not valid at this point. */
3569       end_primitive_masked(bld_base, lp_build_mask_value(bld->mask));
3570
3571       total_emitted_vertices_vec =
3572          LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
3573       emitted_prims_vec =
3574          LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
3575
3576       bld->gs_iface->gs_epilogue(bld->gs_iface,
3577                                  &bld->bld_base,
3578                                  total_emitted_vertices_vec,
3579                                  emitted_prims_vec);
3580    } else {
3581       gather_outputs(bld);
3582    }
3583 }
3584
3585 void
3586 lp_build_tgsi_soa(struct gallivm_state *gallivm,
3587                   const struct tgsi_token *tokens,
3588                   struct lp_type type,
3589                   struct lp_build_mask_context *mask,
3590                   LLVMValueRef consts_ptr,
3591                   LLVMValueRef const_sizes_ptr,
3592                   const struct lp_bld_tgsi_system_values *system_values,
3593                   const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
3594                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
3595                   struct lp_build_sampler_soa *sampler,
3596                   const struct tgsi_shader_info *info,
3597                   const struct lp_build_tgsi_gs_iface *gs_iface)
3598 {
3599    struct lp_build_tgsi_soa_context bld;
3600
3601    struct lp_type res_type;
3602
3603    assert(type.length <= LP_MAX_VECTOR_LENGTH);
3604    memset(&res_type, 0, sizeof res_type);
3605    res_type.width = type.width;
3606    res_type.length = type.length;
3607    res_type.sign = 1;
3608
3609    /* Setup build context */
3610    memset(&bld, 0, sizeof bld);
3611    lp_build_context_init(&bld.bld_base.base, gallivm, type);
3612    lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
3613    lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
3614    lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
3615    bld.mask = mask;
3616    bld.inputs = inputs;
3617    bld.outputs = outputs;
3618    bld.consts_ptr = consts_ptr;
3619    bld.const_sizes_ptr = const_sizes_ptr;
3620    bld.sampler = sampler;
3621    bld.bld_base.info = info;
3622    bld.indirect_files = info->indirect_files;
3623
3624    /*
3625     * If the number of temporaries is rather large then we just
3626     * allocate them as an array right from the start and treat
3627     * like indirect temporaries.
3628     */
3629    if (info->file_max[TGSI_FILE_TEMPORARY] >= LP_MAX_INLINED_TEMPS) {
3630       bld.indirect_files |= (1 << TGSI_FILE_TEMPORARY);
3631    }
3632
3633    bld.bld_base.soa = TRUE;
3634    bld.bld_base.emit_debug = emit_debug;
3635    bld.bld_base.emit_fetch_funcs[TGSI_FILE_CONSTANT] = emit_fetch_constant;
3636    bld.bld_base.emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = emit_fetch_immediate;
3637    bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_input;
3638    bld.bld_base.emit_fetch_funcs[TGSI_FILE_TEMPORARY] = emit_fetch_temporary;
3639    bld.bld_base.emit_fetch_funcs[TGSI_FILE_SYSTEM_VALUE] = emit_fetch_system_value;
3640    bld.bld_base.emit_store = emit_store;
3641
3642    bld.bld_base.emit_declaration = lp_emit_declaration_soa;
3643    bld.bld_base.emit_immediate = lp_emit_immediate_soa;
3644
3645    bld.bld_base.emit_prologue = emit_prologue;
3646    bld.bld_base.emit_epilogue = emit_epilogue;
3647
3648    /* Set opcode actions */
3649    lp_set_default_actions_cpu(&bld.bld_base);
3650
3651    bld.bld_base.op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit;
3652    bld.bld_base.op_actions[TGSI_OPCODE_BGNSUB].emit = bgnsub_emit;
3653    bld.bld_base.op_actions[TGSI_OPCODE_BRK].emit = brk_emit;
3654    bld.bld_base.op_actions[TGSI_OPCODE_BREAKC].emit = breakc_emit;
3655    bld.bld_base.op_actions[TGSI_OPCODE_CAL].emit = cal_emit;
3656    bld.bld_base.op_actions[TGSI_OPCODE_CASE].emit = case_emit;
3657    bld.bld_base.op_actions[TGSI_OPCODE_CONT].emit = cont_emit;
3658    bld.bld_base.op_actions[TGSI_OPCODE_DDX].emit = ddx_emit;
3659    bld.bld_base.op_actions[TGSI_OPCODE_DDY].emit = ddy_emit;
3660    bld.bld_base.op_actions[TGSI_OPCODE_DEFAULT].emit = default_emit;
3661    bld.bld_base.op_actions[TGSI_OPCODE_ELSE].emit = else_emit;
3662    bld.bld_base.op_actions[TGSI_OPCODE_ENDIF].emit = endif_emit;
3663    bld.bld_base.op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit;
3664    bld.bld_base.op_actions[TGSI_OPCODE_ENDSUB].emit = endsub_emit;
3665    bld.bld_base.op_actions[TGSI_OPCODE_ENDSWITCH].emit = endswitch_emit;
3666    bld.bld_base.op_actions[TGSI_OPCODE_IF].emit = if_emit;
3667    bld.bld_base.op_actions[TGSI_OPCODE_UIF].emit = uif_emit;
3668    bld.bld_base.op_actions[TGSI_OPCODE_KILL_IF].emit = kill_if_emit;
3669    bld.bld_base.op_actions[TGSI_OPCODE_KILL].emit = kill_emit;
3670    bld.bld_base.op_actions[TGSI_OPCODE_NRM].emit = nrm_emit;
3671    bld.bld_base.op_actions[TGSI_OPCODE_NRM4].emit = nrm_emit;
3672    bld.bld_base.op_actions[TGSI_OPCODE_RET].emit = ret_emit;
3673    bld.bld_base.op_actions[TGSI_OPCODE_SWITCH].emit = switch_emit;
3674    bld.bld_base.op_actions[TGSI_OPCODE_TEX].emit = tex_emit;
3675    bld.bld_base.op_actions[TGSI_OPCODE_TXB].emit = txb_emit;
3676    bld.bld_base.op_actions[TGSI_OPCODE_TXD].emit = txd_emit;
3677    bld.bld_base.op_actions[TGSI_OPCODE_TXL].emit = txl_emit;
3678    bld.bld_base.op_actions[TGSI_OPCODE_TXP].emit = txp_emit;
3679    bld.bld_base.op_actions[TGSI_OPCODE_TXQ].emit = txq_emit;
3680    bld.bld_base.op_actions[TGSI_OPCODE_TXF].emit = txf_emit;
3681    /* DX10 sampling ops */
3682    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE].emit = sample_emit;
3683    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_B].emit = sample_b_emit;
3684    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C].emit = sample_c_emit;
3685    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_C_LZ].emit = sample_c_lz_emit;
3686    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_D].emit = sample_d_emit;
3687    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_I].emit = sample_i_emit;
3688    bld.bld_base.op_actions[TGSI_OPCODE_SAMPLE_L].emit = sample_l_emit;
3689    bld.bld_base.op_actions[TGSI_OPCODE_SVIEWINFO].emit = sviewinfo_emit;
3690
3691    if (gs_iface) {
3692       /* There's no specific value for this because it should always
3693        * be set, but apps using ext_geometry_shader4 quite often
3694        * were forgetting so we're using MAX_VERTEX_VARYING from
3695        * that spec even though we could debug_assert if it's not
3696        * set, but that's a lot uglier. */
3697       uint max_output_vertices = 32;
3698       uint i = 0;
3699       /* inputs are always indirect with gs */
3700       bld.indirect_files |= (1 << TGSI_FILE_INPUT);
3701       bld.gs_iface = gs_iface;
3702       bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_gs_input;
3703       bld.bld_base.op_actions[TGSI_OPCODE_EMIT].emit = emit_vertex;
3704       bld.bld_base.op_actions[TGSI_OPCODE_ENDPRIM].emit = end_primitive;
3705
3706       for (i = 0; i < info->num_properties; ++i) {
3707          if (info->properties[i].name ==
3708              TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES) {
3709             max_output_vertices = info->properties[i].data[0];
3710          }
3711       }
3712       bld.max_output_vertices_vec =
3713          lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
3714                                 max_output_vertices);
3715    }
3716
3717    lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
3718
3719    bld.system_values = *system_values;
3720
3721    lp_build_tgsi_llvm(&bld.bld_base, tokens);
3722
3723    if (0) {
3724       LLVMBasicBlockRef block = LLVMGetInsertBlock(gallivm->builder);
3725       LLVMValueRef function = LLVMGetBasicBlockParent(block);
3726       debug_printf("11111111111111111111111111111 \n");
3727       tgsi_dump(tokens, 0);
3728       lp_debug_dump_value(function);
3729       debug_printf("2222222222222222222222222222 \n");
3730    }
3731
3732    if (0) {
3733       LLVMModuleRef module = LLVMGetGlobalParent(
3734          LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
3735       LLVMDumpModule(module);
3736
3737    }
3738    lp_exec_mask_fini(&bld.exec_mask);
3739 }