src/gallium/auxiliary/gallivm/lp_bld_tgsi_action.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2011-2012 Advanced Micro Devices, Inc.
   4  * Copyright 2009 VMware, Inc.
   5  * Copyright 2007-2008 VMware, Inc.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the
  10  * "Software"), to deal in the Software without restriction, including
  11  * without limitation the rights to use, copy, modify, merge, publish,
  12  * distribute, sub license, and/or sell copies of the Software, and to
  13  * permit persons to whom the Software is furnished to do so, subject to
  14  * the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the
  17  * next paragraph) shall be included in all copies or substantial portions
  18  * of the Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  23  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  24  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  25  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  26  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27  *
  28  **************************************************************************/
  29
  30 /**
  31  * @file
  32  * TGSI to LLVM IR translation.
  33  *
  34  * @author Jose Fonseca <jfonseca@vmware.com>
  35  * @author Tom Stellard <thomas.stellard@amd.com>
  36  *
  37  * Based on tgsi_sse2.c code written by Michal Krol, Keith Whitwell,
  38  * Brian Paul, and others.
  39  */
  40
  41
  42 #include "lp_bld_tgsi_action.h"
  43
  44 #include "lp_bld_tgsi.h"
  45 #include "lp_bld_arit.h"
  46 #include "lp_bld_bitarit.h"
  47 #include "lp_bld_const.h"
  48 #include "lp_bld_conv.h"
  49 #include "lp_bld_gather.h"
  50 #include "lp_bld_logic.h"
  51 #include "lp_bld_pack.h"
  52
  53 #include "tgsi/tgsi_exec.h"
  54
  55 /* XXX: The CPU only defaults should be repaced by generic ones.  In most
  56  * cases, the CPU defaults are just wrappers around a function in
  57  * lp_build_arit.c and these functions should be inlined here and the CPU
  58  * generic code should be removed and placed elsewhere.
  59  */
  60
  61 /* Default actions */
  62
  63 /* Generic fetch_arg functions */
  64
  65 static void scalar_unary_fetch_args(
  66    struct lp_build_tgsi_context * bld_base,
  67    struct lp_build_emit_data * emit_data)
  68 {
  69    /* src0.x */
  70    emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
  71    emit_data->arg_count = 1;
  72    emit_data->dst_type = LLVMTypeOf(emit_data->args[0]);
  73 }
  74
  75 static void scalar_binary_fetch_args(
  76    struct lp_build_tgsi_context * bld_base,
  77    struct lp_build_emit_data * emit_data)
  78 {
  79    /* src0.x */
  80    emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
  81                                             0, TGSI_CHAN_X);
  82    /* src1.x */
  83    emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
  84                                             1, TGSI_CHAN_X);
  85    emit_data->arg_count = 2;
  86    emit_data->dst_type = LLVMTypeOf(emit_data->args[0]);
  87 }
  88
  89 /* TGSI_OPCODE_ADD */
  90 static void
  91 add_emit(
  92    const struct lp_build_tgsi_action * action,
  93    struct lp_build_tgsi_context * bld_base,
  94    struct lp_build_emit_data * emit_data)
  95 {
  96    emit_data->output[emit_data->chan] = LLVMBuildFAdd(
  97                                 bld_base->base.gallivm->builder,
  98                                 emit_data->args[0], emit_data->args[1], "");
  99 }
 100
 101 /* TGSI_OPCODE_ARR */
 102 static void
 103 arr_emit(
 104    const struct lp_build_tgsi_action * action,
 105    struct lp_build_tgsi_context * bld_base,
 106    struct lp_build_emit_data * emit_data)
 107 {
 108    LLVMValueRef tmp = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_ROUND, emit_data->args[0]);
 109    emit_data->output[emit_data->chan] = LLVMBuildFPToSI(bld_base->base.gallivm->builder, tmp,
 110                                                         bld_base->uint_bld.vec_type, "");
 111 }
 112
 113 /* TGSI_OPCODE_CLAMP */
 114 static void
 115 clamp_emit(
 116    const struct lp_build_tgsi_action * action,
 117    struct lp_build_tgsi_context * bld_base,
 118    struct lp_build_emit_data * emit_data)
 119 {
 120    LLVMValueRef tmp;
 121    tmp = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
 122                                    emit_data->args[0],
 123                                    emit_data->args[1]);
 124    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 125                                        TGSI_OPCODE_MIN, tmp, emit_data->args[2]);
 126 }
 127
 128 /* DP* Helper */
 129
 130 static void
 131 dp_fetch_args(
 132    struct lp_build_tgsi_context * bld_base,
 133    struct lp_build_emit_data * emit_data,
 134    unsigned dp_components)
 135 {
 136    unsigned chan, src;
 137    for (src = 0; src < 2; src++) {
 138       for (chan = 0; chan < dp_components; chan++) {
 139          emit_data->args[(src * dp_components) + chan] =
 140                      lp_build_emit_fetch(bld_base, emit_data->inst, src, chan);
 141       }
 142    }
 143    emit_data->dst_type = bld_base->base.elem_type;
 144 }
 145
 146 /* TGSI_OPCODE_DP2 */
 147 static void
 148 dp2_fetch_args(
 149    struct lp_build_tgsi_context * bld_base,
 150    struct lp_build_emit_data * emit_data)
 151 {
 152    dp_fetch_args(bld_base, emit_data, 2);
 153 }
 154
 155 static void
 156 dp2_emit(
 157    const struct lp_build_tgsi_action * action,
 158    struct lp_build_tgsi_context * bld_base,
 159    struct lp_build_emit_data * emit_data)
 160 {
 161    LLVMValueRef tmp0, tmp1;
 162    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 163                                     emit_data->args[0] /* src0.x */,
 164                                     emit_data->args[2] /* src1.x */);
 165    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 166                                     emit_data->args[1] /* src0.y */,
 167                                     emit_data->args[3] /* src1.y */);
 168    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 169                                                     TGSI_OPCODE_ADD, tmp0, tmp1);
 170 }
 171
 172 static struct lp_build_tgsi_action dp2_action = {
 173    dp2_fetch_args,       /* fetch_args */
 174    dp2_emit      /* emit */
 175 };
 176
 177 /* TGSI_OPCODE_DP2A */
 178 static void
 179 dp2a_fetch_args(
 180    struct lp_build_tgsi_context * bld_base,
 181    struct lp_build_emit_data * emit_data)
 182 {
 183    dp_fetch_args(bld_base, emit_data, 2);
 184    emit_data->args[5] = lp_build_emit_fetch(bld_base, emit_data->inst,
 185                                             2, TGSI_CHAN_X);
 186 }
 187
 188 static void
 189 dp2a_emit(
 190    const struct lp_build_tgsi_action * action,
 191    struct lp_build_tgsi_context * bld_base,
 192    struct lp_build_emit_data * emit_data)
 193 {
 194    LLVMValueRef tmp;
 195    tmp = lp_build_emit_llvm(bld_base, TGSI_OPCODE_DP2, emit_data);
 196    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_ADD,
 197                                     emit_data->args[5], tmp);
 198 }
 199
 200 static struct lp_build_tgsi_action dp2a_action = {
 201    dp2a_fetch_args,      /* fetch_args */
 202    dp2a_emit     /* emit */
 203 };
 204
 205 /* TGSI_OPCODE_DP3 */
 206 static void
 207 dp3_fetch_args(
 208    struct lp_build_tgsi_context * bld_base,
 209    struct lp_build_emit_data * emit_data)
 210 {
 211    dp_fetch_args(bld_base, emit_data, 3);
 212 }
 213
 214 static void
 215 dp3_emit(
 216    const struct lp_build_tgsi_action * action,
 217    struct lp_build_tgsi_context * bld_base,
 218    struct lp_build_emit_data * emit_data)
 219 {
 220    LLVMValueRef tmp0, tmp1;
 221    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 222                                     emit_data->args[0] /* src0.x */,
 223                                     emit_data->args[3] /* src1.x */);
 224    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 225                                     emit_data->args[1] /* src0.y */,
 226                                     emit_data->args[4] /* src1.y */);
 227    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_ADD, tmp1, tmp0);
 228    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 229                                     emit_data->args[2] /* src0.z */,
 230                                     emit_data->args[5] /* src1.z */);
 231    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 232                                                     TGSI_OPCODE_ADD, tmp0, tmp1);
 233 }
 234
 235 static struct lp_build_tgsi_action dp3_action = {
 236    dp3_fetch_args,       /* fetch_args */
 237    dp3_emit      /* emit */
 238 };
 239
 240 /* TGSI_OPCODDE_DP4 */
 241
 242 static void
 243 dp4_fetch_args(
 244    struct lp_build_tgsi_context * bld_base,
 245    struct lp_build_emit_data * emit_data)
 246 {
 247    dp_fetch_args(bld_base, emit_data, 4);
 248 }
 249
 250 static void
 251 dp4_emit(
 252    const struct lp_build_tgsi_action * action,
 253    struct lp_build_tgsi_context * bld_base,
 254    struct lp_build_emit_data * emit_data)
 255 {
 256    LLVMValueRef tmp0, tmp1;
 257    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 258                                     emit_data->args[0] /* src0.x */,
 259                                     emit_data->args[4] /* src1.x */);
 260    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 261                                     emit_data->args[1] /* src0.y */,
 262                                     emit_data->args[5] /* src1.y */);
 263    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_ADD, tmp0, tmp1);
 264    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 265                                     emit_data->args[2] /* src0.z */,
 266                                     emit_data->args[6] /* src1.z */);
 267    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_ADD, tmp0, tmp1);
 268    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 269                                     emit_data->args[3] /* src0.w */,
 270                                     emit_data->args[7] /* src1.w */);
 271    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 272                                                     TGSI_OPCODE_ADD, tmp0, tmp1);
 273 }
 274
 275 static struct lp_build_tgsi_action dp4_action = {
 276    dp4_fetch_args,       /* fetch_args */
 277    dp4_emit      /* emit */
 278 };
 279
 280 /* TGSI_OPCODE_DPH */
 281 static void
 282 dph_fetch_args(
 283    struct lp_build_tgsi_context * bld_base,
 284    struct lp_build_emit_data * emit_data)
 285 {
 286    dp_fetch_args(bld_base, emit_data, 4);
 287    /* src0.w */
 288    emit_data->args[3] = bld_base->base.one;
 289 }
 290
 291 const struct lp_build_tgsi_action dph_action = {
 292    dph_fetch_args,       /* fetch_args */
 293    dp4_emit      /* emit */
 294 };
 295
 296 /* TGSI_OPCODE_DST */
 297 static void
 298 dst_fetch_args(
 299    struct lp_build_tgsi_context * bld_base,
 300    struct lp_build_emit_data * emit_data)
 301 {
 302    /* src0.y */
 303    emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
 304                                             0, TGSI_CHAN_Y);
 305    /* src0.z */
 306    emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
 307                                             0, TGSI_CHAN_Z);
 308    /* src1.y */
 309    emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst,
 310                                             1, TGSI_CHAN_Y);
 311    /* src1.w */
 312    emit_data->args[3] = lp_build_emit_fetch(bld_base, emit_data->inst,
 313                                             1, TGSI_CHAN_W);
 314 }
 315
 316 static void
 317 dst_emit(
 318    const struct lp_build_tgsi_action * action,
 319    struct lp_build_tgsi_context * bld_base,
 320    struct lp_build_emit_data * emit_data)
 321 {
 322    /* dst.x */
 323    emit_data->output[TGSI_CHAN_X] = bld_base->base.one;
 324
 325    /* dst.y */
 326    emit_data->output[TGSI_CHAN_Y] = lp_build_emit_llvm_binary(bld_base,
 327                                           TGSI_OPCODE_MUL,
 328                                           emit_data->args[0] /* src0.y */,
 329                                           emit_data->args[2] /* src1.y */);
 330    /* dst.z */
 331    emit_data->output[TGSI_CHAN_Z] = emit_data->args[1]; /* src0.z */
 332
 333    /* dst.w */
 334    emit_data->output[TGSI_CHAN_W] = emit_data->args[3]; /* src1.w */
 335 }
 336
 337 static struct lp_build_tgsi_action dst_action = {
 338    dst_fetch_args,       /* fetch_args */
 339    dst_emit      /* emit */
 340 };
 341
 342 /* TGSI_OPCODE_END */
 343 static void
 344 end_emit(
 345    const struct lp_build_tgsi_action * action,
 346    struct lp_build_tgsi_context * bld_base,
 347    struct lp_build_emit_data * emit_data)
 348 {
 349    bld_base->pc = -1;
 350 }
 351
 352 /* TGSI_OPCODE_EXP */
 353
 354 static void
 355 exp_emit(
 356    const struct lp_build_tgsi_action * action,
 357    struct lp_build_tgsi_context * bld_base,
 358    struct lp_build_emit_data * emit_data)
 359 {
 360    LLVMValueRef floor_x;
 361
 362    /* floor( src0.x ) */
 363    floor_x = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_FLR,
 364                                       emit_data->args[0]);
 365
 366    /* 2 ^ floor( src0.x ) */
 367    emit_data->output[TGSI_CHAN_X] = lp_build_emit_llvm_unary(bld_base,
 368                                        TGSI_OPCODE_EX2, floor_x);
 369
 370    /* src0.x - floor( src0.x ) */
 371    emit_data->output[TGSI_CHAN_Y] =
 372       lp_build_sub(&bld_base->base, emit_data->args[0] /* src0.x */, floor_x);
 373
 374    /* 2 ^ src0.x */
 375    emit_data->output[TGSI_CHAN_Z] = lp_build_emit_llvm_unary(bld_base,
 376                              TGSI_OPCODE_EX2, emit_data->args[0] /* src0.x */);
 377
 378    emit_data->output[TGSI_CHAN_W] = bld_base->base.one;
 379 }
 380
 381 const struct lp_build_tgsi_action exp_action = {
 382    scalar_unary_fetch_args,      /* fetch_args */
 383    exp_emit      /* emit */
 384 };
 385
 386 /* TGSI_OPCODE_FRC */
 387
 388 static void
 389 frc_emit(
 390    const struct lp_build_tgsi_action * action,
 391    struct lp_build_tgsi_context * bld_base,
 392    struct lp_build_emit_data * emit_data)
 393 {
 394    LLVMValueRef tmp;
 395    tmp = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_FLR,
 396                                   emit_data->args[0]);
 397    emit_data->output[emit_data->chan] =
 398       lp_build_sub(&bld_base->base, emit_data->args[0], tmp);
 399 }
 400
 401 /* TGSI_OPCODE_KILL_IF */
 402
 403 static void
 404 kil_fetch_args(
 405    struct lp_build_tgsi_context * bld_base,
 406    struct lp_build_emit_data * emit_data)
 407 {
 408    /* src0.x */
 409    emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
 410                                             0, TGSI_CHAN_X);
 411    /* src0.y */
 412    emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
 413                                             0, TGSI_CHAN_Y);
 414    /* src0.z */
 415    emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst,
 416                                             0, TGSI_CHAN_Z);
 417    /* src0.w */
 418    emit_data->args[3] = lp_build_emit_fetch(bld_base, emit_data->inst,
 419                                             0, TGSI_CHAN_W);
 420    emit_data->arg_count = 4;
 421    emit_data->dst_type = LLVMVoidTypeInContext(bld_base->base.gallivm->context);
 422 }
 423
 424 /* TGSI_OPCODE_KILL */
 425
 426 static void
 427 kilp_fetch_args(
 428    struct lp_build_tgsi_context * bld_base,
 429    struct lp_build_emit_data * emit_data)
 430 {
 431    emit_data->dst_type = LLVMVoidTypeInContext(bld_base->base.gallivm->context);
 432 }
 433
 434 /* TGSI_OPCODE_LIT */
 435
 436 static void
 437 lit_fetch_args(
 438    struct lp_build_tgsi_context * bld_base,
 439    struct lp_build_emit_data * emit_data)
 440 {
 441    /* src0.x */
 442    emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
 443    /* src0.y */
 444    emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_Y);
 445    /* src0.w */
 446    emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
 447    emit_data->arg_count = 3;
 448 }
 449
 450 static void
 451 lit_emit(
 452    const struct lp_build_tgsi_action * action,
 453    struct lp_build_tgsi_context * bld_base,
 454    struct lp_build_emit_data * emit_data)
 455 {
 456    LLVMValueRef tmp0, tmp1, tmp2;
 457
 458    /* dst.x */
 459    emit_data->output[TGSI_CHAN_X] = bld_base->base.one;
 460
 461    /* dst. y */
 462    emit_data->output[TGSI_CHAN_Y] = lp_build_emit_llvm_binary(bld_base,
 463                                                TGSI_OPCODE_MAX,
 464                                                emit_data->args[0] /* src0.x */,
 465                                                bld_base->base.zero);
 466
 467    /* dst.z */
 468    /* XMM[1] = SrcReg[0].yyyy */
 469    tmp1 = emit_data->args[1];
 470    /* XMM[1] = max(XMM[1], 0) */
 471    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
 472                                     tmp1, bld_base->base.zero);
 473    /* XMM[2] = SrcReg[0].wwww */
 474    tmp2 = emit_data->args[2];
 475    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_POW,
 476                                     tmp1, tmp2);
 477    tmp0 = emit_data->args[0];
 478    emit_data->output[TGSI_CHAN_Z] = lp_build_emit_llvm_ternary(bld_base,
 479                                              TGSI_OPCODE_CMP,
 480                                              tmp0, bld_base->base.zero, tmp1);
 481    /* dst.w */
 482    emit_data->output[TGSI_CHAN_W] = bld_base->base.one;
 483 }
 484
 485 static struct lp_build_tgsi_action lit_action = {
 486    lit_fetch_args,       /* fetch_args */
 487    lit_emit      /* emit */
 488 };
 489
 490 /* TGSI_OPCODE_LOG */
 491
 492 static void
 493 log_emit(
 494    const struct lp_build_tgsi_action * action,
 495    struct lp_build_tgsi_context * bld_base,
 496    struct lp_build_emit_data * emit_data)
 497 {
 498
 499    LLVMValueRef abs_x, log_abs_x, flr_log_abs_x, ex2_flr_log_abs_x;
 500
 501    /* abs( src0.x) */
 502    abs_x = lp_build_abs(&bld_base->base, emit_data->args[0] /* src0.x */);
 503
 504    /* log( abs( src0.x ) ) */
 505    log_abs_x = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_LG2,
 506                                         abs_x);
 507
 508    /* floor( log( abs( src0.x ) ) ) */
 509    flr_log_abs_x = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_FLR,
 510                                             log_abs_x);
 511    /* dst.x */
 512    emit_data->output[TGSI_CHAN_X] = flr_log_abs_x;
 513
 514    /* dst.y */
 515    ex2_flr_log_abs_x = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_EX2,
 516                                                 flr_log_abs_x);
 517
 518    /* abs( src0.x ) / 2^( floor( lg2( abs( src0.x ) ) ) ) */
 519    emit_data->output[TGSI_CHAN_Y] = lp_build_emit_llvm_binary(bld_base,
 520                                     TGSI_OPCODE_DIV, abs_x, ex2_flr_log_abs_x);
 521
 522    /* dst.x */
 523    emit_data->output[TGSI_CHAN_Z] = log_abs_x;
 524
 525    /* dst.w */
 526    emit_data->output[TGSI_CHAN_W] = bld_base->base.one;
 527 }
 528
 529 static struct lp_build_tgsi_action log_action = {
 530    scalar_unary_fetch_args,      /* fetch_args */
 531    log_emit      /* emit */
 532 };
 533
 534 /* TGSI_OPCODE_PK2H */
 535
 536 static void
 537 pk2h_fetch_args(
 538    struct lp_build_tgsi_context * bld_base,
 539    struct lp_build_emit_data * emit_data)
 540 {
 541    /* src0.x */
 542    emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
 543                                             0, TGSI_CHAN_X);
 544    /* src0.y */
 545    emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
 546                                             0, TGSI_CHAN_Y);
 547 }
 548
 549 static void
 550 pk2h_emit(
 551    const struct lp_build_tgsi_action *action,
 552    struct lp_build_tgsi_context *bld_base,
 553    struct lp_build_emit_data *emit_data)
 554 {
 555    struct gallivm_state *gallivm = bld_base->base.gallivm;
 556    struct lp_type f16i_t;
 557    LLVMValueRef lo, hi, res;
 558
 559    f16i_t = lp_type_uint_vec(16, bld_base->base.type.length * 32);
 560    lo = lp_build_float_to_half(gallivm, emit_data->args[0]);
 561    hi = lp_build_float_to_half(gallivm, emit_data->args[1]);
 562    /* maybe some interleave doubling vector width would be useful... */
 563    lo = lp_build_pad_vector(gallivm, lo, bld_base->base.type.length * 2);
 564    hi = lp_build_pad_vector(gallivm, hi, bld_base->base.type.length * 2);
 565    res = lp_build_interleave2(gallivm, f16i_t, lo, hi, 0);
 566
 567    emit_data->output[emit_data->chan] = res;
 568 }
 569
 570 static struct lp_build_tgsi_action pk2h_action = {
 571    pk2h_fetch_args, /* fetch_args */
 572    pk2h_emit        /* emit */
 573 };
 574
 575 /* TGSI_OPCODE_UP2H */
 576
 577 static void
 578 up2h_emit(
 579    const struct lp_build_tgsi_action *action,
 580    struct lp_build_tgsi_context *bld_base,
 581    struct lp_build_emit_data *emit_data)
 582 {
 583    struct gallivm_state *gallivm = bld_base->base.gallivm;
 584    LLVMBuilderRef builder = gallivm->builder;
 585    LLVMContextRef context = gallivm->context;
 586    LLVMValueRef lo, hi, res[2], arg;
 587    unsigned nr = bld_base->base.type.length;
 588    LLVMTypeRef i16t = LLVMVectorType(LLVMInt16TypeInContext(context), nr * 2);
 589
 590    arg = LLVMBuildBitCast(builder, emit_data->args[0], i16t, "");
 591    lo = lp_build_uninterleave1(gallivm, nr * 2, arg, 0);
 592    hi = lp_build_uninterleave1(gallivm, nr * 2, arg, 1);
 593    res[0] = lp_build_half_to_float(gallivm, lo);
 594    res[1] = lp_build_half_to_float(gallivm, hi);
 595
 596    emit_data->output[0] = emit_data->output[2] = res[0];
 597    emit_data->output[1] = emit_data->output[3] = res[1];
 598 }
 599
 600 static struct lp_build_tgsi_action up2h_action = {
 601    scalar_unary_fetch_args, /* fetch_args */
 602    up2h_emit                /* emit */
 603 };
 604
 605 /* TGSI_OPCODE_LRP */
 606
 607 static void
 608 lrp_emit(
 609    const struct lp_build_tgsi_action * action,
 610    struct lp_build_tgsi_context * bld_base,
 611    struct lp_build_emit_data * emit_data)
 612 {
 613    struct lp_build_context *bld = &bld_base->base;
 614    LLVMValueRef inv, a, b;
 615
 616    /* This uses the correct version: (1 - t)*a + t*b
 617     *
 618     * An alternative version is "a + t*(b-a)". The problem is this version
 619     * doesn't return "b" for t = 1, because "a + (b-a)" isn't equal to "b"
 620     * because of the floating-point rounding.
 621     */
 622    inv = lp_build_sub(bld, bld_base->base.one, emit_data->args[0]);
 623    a = lp_build_mul(bld, emit_data->args[1], emit_data->args[0]);
 624    b = lp_build_mul(bld, emit_data->args[2], inv);
 625    emit_data->output[emit_data->chan] = lp_build_add(bld, a, b);
 626 }
 627
 628 /* TGSI_OPCODE_MAD */
 629
 630 static void
 631 mad_emit(
 632    const struct lp_build_tgsi_action * action,
 633    struct lp_build_tgsi_context * bld_base,
 634    struct lp_build_emit_data * emit_data)
 635 {
 636    LLVMValueRef tmp;
 637    tmp = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL,
 638                                    emit_data->args[0],
 639                                    emit_data->args[1]);
 640    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 641                                        TGSI_OPCODE_ADD, tmp, emit_data->args[2]);
 642 }
 643
 644 /* TGSI_OPCODE_MOV */
 645
 646 static void
 647 mov_emit(
 648    const struct lp_build_tgsi_action * action,
 649    struct lp_build_tgsi_context * bld_base,
 650    struct lp_build_emit_data * emit_data)
 651 {
 652    emit_data->output[emit_data->chan] = emit_data->args[0];
 653 }
 654
 655 /* TGSI_OPCODE_MUL */
 656 static void
 657 mul_emit(
 658    const struct lp_build_tgsi_action * action,
 659    struct lp_build_tgsi_context * bld_base,
 660    struct lp_build_emit_data * emit_data)
 661 {
 662    emit_data->output[emit_data->chan] = LLVMBuildFMul(
 663                                    bld_base->base.gallivm->builder,
 664                                    emit_data->args[0], emit_data->args[1], "");
 665 }
 666
 667 /*.TGSI_OPCODE_DIV.*/
 668 static void fdiv_emit(
 669    const struct lp_build_tgsi_action * action,
 670    struct lp_build_tgsi_context * bld_base,
 671    struct lp_build_emit_data * emit_data)
 672 {
 673    emit_data->output[emit_data->chan] = LLVMBuildFDiv(
 674                                    bld_base->base.gallivm->builder,
 675                                    emit_data->args[0], emit_data->args[1], "");
 676 }
 677
 678 /*.TGSI_OPCODE_RCP.*/
 679 static void rcp_emit(
 680    const struct lp_build_tgsi_action * action,
 681    struct lp_build_tgsi_context * bld_base,
 682    struct lp_build_emit_data * emit_data)
 683 {
 684    LLVMValueRef one;
 685    one = lp_build_const_float(bld_base->base.gallivm, 1.0f);
 686    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 687                                    TGSI_OPCODE_DIV, one, emit_data->args[0]);
 688 }
 689
 690 /* TGSI_OPCODE_POW */
 691
 692 static void
 693 pow_emit(
 694    const struct lp_build_tgsi_action * action,
 695    struct lp_build_tgsi_context * bld_base,
 696    struct lp_build_emit_data * emit_data)
 697 {
 698    emit_data->output[emit_data->chan] = lp_build_pow(&bld_base->base,
 699                                    emit_data->args[0], emit_data->args[1]);
 700 }
 701
 702 static struct lp_build_tgsi_action pow_action = {
 703    scalar_binary_fetch_args,     /* fetch_args */
 704    pow_emit      /* emit */
 705 };
 706
 707 /* TGSI_OPCODE_RSQ */
 708
 709 static void
 710 rsq_emit(
 711    const struct lp_build_tgsi_action * action,
 712    struct lp_build_tgsi_context * bld_base,
 713    struct lp_build_emit_data * emit_data)
 714 {
 715    if (bld_base->rsq_action.emit) {
 716       bld_base->rsq_action.emit(&bld_base->rsq_action, bld_base, emit_data);
 717    } else {
 718       emit_data->output[emit_data->chan] = bld_base->base.undef;
 719    }
 720 }
 721
 722 const struct lp_build_tgsi_action rsq_action = {
 723    scalar_unary_fetch_args,      /* fetch_args */
 724    rsq_emit      /* emit */
 725
 726 };
 727
 728 /* TGSI_OPCODE_SQRT */
 729
 730 static void
 731 sqrt_emit(
 732    const struct lp_build_tgsi_action * action,
 733    struct lp_build_tgsi_context * bld_base,
 734    struct lp_build_emit_data * emit_data)
 735 {
 736    if (bld_base->sqrt_action.emit) {
 737       bld_base->sqrt_action.emit(&bld_base->sqrt_action, bld_base, emit_data);
 738    } else {
 739       emit_data->output[emit_data->chan] = bld_base->base.undef;
 740    }
 741 }
 742
 743 const struct lp_build_tgsi_action sqrt_action = {
 744    scalar_unary_fetch_args,      /* fetch_args */
 745    sqrt_emit     /* emit */
 746 };
 747
 748 /* TGSI_OPCODE_SCS */
 749 static void
 750 scs_emit(
 751    const struct lp_build_tgsi_action * action,
 752    struct lp_build_tgsi_context * bld_base,
 753    struct lp_build_emit_data * emit_data)
 754 {
 755    /* dst.x */
 756    emit_data->output[TGSI_CHAN_X] = lp_build_emit_llvm_unary(bld_base,
 757                                            TGSI_OPCODE_COS, emit_data->args[0]);
 758    /* dst.y */
 759    emit_data->output[TGSI_CHAN_Y] = lp_build_emit_llvm_unary(bld_base,
 760                                            TGSI_OPCODE_SIN, emit_data->args[0]);
 761    /* dst.z */
 762    emit_data->output[TGSI_CHAN_Z] = bld_base->base.zero;
 763
 764    /* dst.w */
 765    emit_data->output[TGSI_CHAN_W] = bld_base->base.one;
 766 }
 767
 768 const struct lp_build_tgsi_action scs_action = {
 769    scalar_unary_fetch_args,      /* fetch_args */
 770    scs_emit      /* emit */
 771 };
 772
 773 /* TGSI_OPCODE_F2U */
 774 static void
 775 f2u_emit(
 776    const struct lp_build_tgsi_action * action,
 777    struct lp_build_tgsi_context * bld_base,
 778    struct lp_build_emit_data * emit_data)
 779 {
 780    emit_data->output[emit_data->chan] =
 781       LLVMBuildFPToUI(bld_base->base.gallivm->builder,
 782                       emit_data->args[0],
 783                       bld_base->base.int_vec_type, "");
 784 }
 785
 786 /* TGSI_OPCODE_U2F */
 787 static void
 788 u2f_emit(
 789    const struct lp_build_tgsi_action * action,
 790    struct lp_build_tgsi_context * bld_base,
 791    struct lp_build_emit_data * emit_data)
 792 {
 793    emit_data->output[emit_data->chan] =
 794       LLVMBuildUIToFP(bld_base->base.gallivm->builder,
 795                       emit_data->args[0],
 796                       bld_base->base.vec_type, "");
 797 }
 798
 799 static void
 800 umad_emit(
 801    const struct lp_build_tgsi_action * action,
 802    struct lp_build_tgsi_context * bld_base,
 803    struct lp_build_emit_data * emit_data)
 804 {
 805    LLVMValueRef tmp;
 806    tmp = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_UMUL,
 807                                    emit_data->args[0],
 808                                    emit_data->args[1]);
 809    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
 810                                        TGSI_OPCODE_UADD, tmp, emit_data->args[2]);
 811 }
 812
 813 /* TGSI_OPCODE_UMUL */
 814 static void
 815 umul_emit(
 816    const struct lp_build_tgsi_action * action,
 817    struct lp_build_tgsi_context * bld_base,
 818    struct lp_build_emit_data * emit_data)
 819 {
 820    emit_data->output[emit_data->chan] = lp_build_mul(&bld_base->uint_bld,
 821                                    emit_data->args[0], emit_data->args[1]);
 822 }
 823
 824 /* TGSI_OPCODE_IMUL_HI */
 825 static void
 826 imul_hi_emit(
 827    const struct lp_build_tgsi_action * action,
 828    struct lp_build_tgsi_context * bld_base,
 829    struct lp_build_emit_data * emit_data)
 830 {
 831    struct lp_build_context *int_bld = &bld_base->int_bld;
 832    LLVMValueRef hi_bits;
 833
 834    assert(int_bld->type.width == 32);
 835
 836    /* low result bits are tossed away */
 837    lp_build_mul_32_lohi(int_bld, emit_data->args[0],
 838                         emit_data->args[1], &hi_bits);
 839    emit_data->output[emit_data->chan] = hi_bits;
 840 }
 841
 842 static void
 843 imul_hi_emit_cpu(
 844    const struct lp_build_tgsi_action * action,
 845    struct lp_build_tgsi_context * bld_base,
 846    struct lp_build_emit_data * emit_data)
 847 {
 848    struct lp_build_context *int_bld = &bld_base->int_bld;
 849    LLVMValueRef hi_bits;
 850
 851    assert(int_bld->type.width == 32);
 852
 853    /* low result bits are tossed away */
 854    lp_build_mul_32_lohi_cpu(int_bld, emit_data->args[0],
 855                             emit_data->args[1], &hi_bits);
 856    emit_data->output[emit_data->chan] = hi_bits;
 857 }
 858
 859 /* TGSI_OPCODE_UMUL_HI */
 860 static void
 861 umul_hi_emit(
 862    const struct lp_build_tgsi_action * action,
 863    struct lp_build_tgsi_context * bld_base,
 864    struct lp_build_emit_data * emit_data)
 865 {
 866    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 867    LLVMValueRef hi_bits;
 868
 869    assert(uint_bld->type.width == 32);
 870
 871    /* low result bits are tossed away */
 872    lp_build_mul_32_lohi(uint_bld, emit_data->args[0],
 873                         emit_data->args[1], &hi_bits);
 874    emit_data->output[emit_data->chan] = hi_bits;
 875 }
 876
 877 static void
 878 umul_hi_emit_cpu(
 879    const struct lp_build_tgsi_action * action,
 880    struct lp_build_tgsi_context * bld_base,
 881    struct lp_build_emit_data * emit_data)
 882 {
 883    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 884    LLVMValueRef hi_bits;
 885
 886    assert(uint_bld->type.width == 32);
 887
 888    /* low result bits are tossed away */
 889    lp_build_mul_32_lohi_cpu(uint_bld, emit_data->args[0],
 890                             emit_data->args[1], &hi_bits);
 891    emit_data->output[emit_data->chan] = hi_bits;
 892 }
 893
 894 /* TGSI_OPCODE_MAX */
 895 static void fmax_emit(
 896    const struct lp_build_tgsi_action * action,
 897    struct lp_build_tgsi_context * bld_base,
 898    struct lp_build_emit_data * emit_data)
 899 {
 900    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 901    emit_data->output[emit_data->chan] = LLVMBuildSelect(builder,
 902                                    LLVMBuildFCmp(builder, LLVMRealUGE,
 903                                    emit_data->args[0], emit_data->args[1], ""),
 904                                    emit_data->args[0], emit_data->args[1], "");
 905 }
 906
 907 /* TGSI_OPCODE_MIN */
 908 static void fmin_emit(
 909    const struct lp_build_tgsi_action * action,
 910    struct lp_build_tgsi_context * bld_base,
 911    struct lp_build_emit_data * emit_data)
 912 {
 913    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 914    emit_data->output[emit_data->chan] = LLVMBuildSelect(builder,
 915                                    LLVMBuildFCmp(builder, LLVMRealUGE,
 916                                    emit_data->args[0], emit_data->args[1], ""),
 917                                    emit_data->args[1], emit_data->args[0], "");
 918 }
 919
 920 /* TGSI_OPCODE_XPD */
 921
 922 static void
 923 xpd_fetch_args(
 924    struct lp_build_tgsi_context * bld_base,
 925    struct lp_build_emit_data * emit_data)
 926 {
 927    dp_fetch_args(bld_base, emit_data, 3);
 928 }
 929
 930 /**
 931  * (a * b) - (c * d)
 932  */
 933 static LLVMValueRef
 934 xpd_helper(
 935   struct lp_build_tgsi_context * bld_base,
 936   LLVMValueRef a,
 937   LLVMValueRef b,
 938   LLVMValueRef c,
 939   LLVMValueRef d)
 940 {
 941    LLVMValueRef tmp0, tmp1;
 942
 943    tmp0 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL, a, b);
 944    tmp1 = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MUL, c, d);
 945
 946    return lp_build_sub(&bld_base->base, tmp0, tmp1);
 947 }
 948
 949 static void
 950 xpd_emit(
 951    const struct lp_build_tgsi_action * action,
 952    struct lp_build_tgsi_context * bld_base,
 953    struct lp_build_emit_data * emit_data)
 954 {
 955    emit_data->output[TGSI_CHAN_X] = xpd_helper(bld_base,
 956               emit_data->args[1] /* src0.y */, emit_data->args[5] /* src1.z */,
 957               emit_data->args[4] /* src1.y */, emit_data->args[2] /* src0.z */);
 958
 959    emit_data->output[TGSI_CHAN_Y] = xpd_helper(bld_base,
 960               emit_data->args[2] /* src0.z */, emit_data->args[3] /* src1.x */,
 961               emit_data->args[5] /* src1.z */, emit_data->args[0] /* src0.x */);
 962
 963    emit_data->output[TGSI_CHAN_Z] = xpd_helper(bld_base,
 964               emit_data->args[0] /* src0.x */, emit_data->args[4] /* src1.y */,
 965               emit_data->args[3] /* src1.x */, emit_data->args[1] /* src0.y */);
 966
 967    emit_data->output[TGSI_CHAN_W] = bld_base->base.one;
 968 }
 969
 970 const struct lp_build_tgsi_action xpd_action = {
 971    xpd_fetch_args,       /* fetch_args */
 972    xpd_emit      /* emit */
 973 };
 974
 975 /* TGSI_OPCODE_D2F */
 976 static void
 977 d2f_emit(
 978    const struct lp_build_tgsi_action * action,
 979    struct lp_build_tgsi_context * bld_base,
 980    struct lp_build_emit_data * emit_data)
 981 {
 982    emit_data->output[emit_data->chan] =
 983       LLVMBuildFPTrunc(bld_base->base.gallivm->builder,
 984                       emit_data->args[0],
 985                        bld_base->base.vec_type, "");
 986 }
 987
 988 /* TGSI_OPCODE_D2I */
 989 static void
 990 d2i_emit(
 991    const struct lp_build_tgsi_action * action,
 992    struct lp_build_tgsi_context * bld_base,
 993    struct lp_build_emit_data * emit_data)
 994 {
 995    emit_data->output[emit_data->chan] =
 996       LLVMBuildFPToSI(bld_base->base.gallivm->builder,
 997                       emit_data->args[0],
 998                       bld_base->base.int_vec_type, "");
 999 }
1000
1001 /* TGSI_OPCODE_D2U */
1002 static void
1003 d2u_emit(
1004    const struct lp_build_tgsi_action * action,
1005    struct lp_build_tgsi_context * bld_base,
1006    struct lp_build_emit_data * emit_data)
1007 {
1008    emit_data->output[emit_data->chan] =
1009       LLVMBuildFPToUI(bld_base->base.gallivm->builder,
1010                       emit_data->args[0],
1011                       bld_base->base.int_vec_type, "");
1012 }
1013
1014 /* TGSI_OPCODE_F2D */
1015 static void
1016 f2d_emit(
1017    const struct lp_build_tgsi_action * action,
1018    struct lp_build_tgsi_context * bld_base,
1019    struct lp_build_emit_data * emit_data)
1020 {
1021    emit_data->output[emit_data->chan] =
1022       LLVMBuildFPExt(bld_base->base.gallivm->builder,
1023                       emit_data->args[0],
1024                       bld_base->dbl_bld.vec_type, "");
1025 }
1026
1027 /* TGSI_OPCODE_U2D */
1028 static void
1029 u2d_emit(
1030    const struct lp_build_tgsi_action * action,
1031    struct lp_build_tgsi_context * bld_base,
1032    struct lp_build_emit_data * emit_data)
1033 {
1034    emit_data->output[emit_data->chan] =
1035       LLVMBuildUIToFP(bld_base->base.gallivm->builder,
1036                       emit_data->args[0],
1037                       bld_base->dbl_bld.vec_type, "");
1038 }
1039
1040 /* TGSI_OPCODE_I2D */
1041 static void
1042 i2d_emit(
1043    const struct lp_build_tgsi_action * action,
1044    struct lp_build_tgsi_context * bld_base,
1045    struct lp_build_emit_data * emit_data)
1046 {
1047    emit_data->output[emit_data->chan] =
1048       LLVMBuildSIToFP(bld_base->base.gallivm->builder,
1049                       emit_data->args[0],
1050                       bld_base->dbl_bld.vec_type, "");
1051 }
1052
1053 /* TGSI_OPCODE_DMAD */
1054 static void
1055 dmad_emit(
1056    const struct lp_build_tgsi_action * action,
1057    struct lp_build_tgsi_context * bld_base,
1058    struct lp_build_emit_data * emit_data)
1059 {
1060    LLVMValueRef tmp;
1061    tmp = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_DMUL,
1062                                    emit_data->args[0],
1063                                    emit_data->args[1]);
1064    emit_data->output[emit_data->chan] = lp_build_emit_llvm_binary(bld_base,
1065                                        TGSI_OPCODE_DADD, tmp, emit_data->args[2]);
1066 }
1067
1068 /*.TGSI_OPCODE_DRCP.*/
1069 static void drcp_emit(
1070    const struct lp_build_tgsi_action * action,
1071    struct lp_build_tgsi_context * bld_base,
1072    struct lp_build_emit_data * emit_data)
1073 {
1074    LLVMValueRef one;
1075    one = lp_build_const_vec(bld_base->dbl_bld.gallivm, bld_base->dbl_bld.type, 1.0f);
1076    emit_data->output[emit_data->chan] = LLVMBuildFDiv(
1077       bld_base->base.gallivm->builder,
1078       one, emit_data->args[0], "");
1079 }
1080
1081 /* TGSI_OPCODE_DFRAC */
1082 static void dfrac_emit(
1083    const struct lp_build_tgsi_action * action,
1084    struct lp_build_tgsi_context * bld_base,
1085    struct lp_build_emit_data * emit_data)
1086 {
1087    LLVMValueRef tmp;
1088    tmp = lp_build_floor(&bld_base->dbl_bld,
1089                         emit_data->args[0]);
1090    emit_data->output[emit_data->chan] =  LLVMBuildFSub(bld_base->base.gallivm->builder,
1091                                                        emit_data->args[0], tmp, "");
1092 }
1093
1094 static void
1095 u64mul_emit(
1096    const struct lp_build_tgsi_action * action,
1097    struct lp_build_tgsi_context * bld_base,
1098    struct lp_build_emit_data * emit_data)
1099 {
1100    emit_data->output[emit_data->chan] = lp_build_mul(&bld_base->uint64_bld,
1101                                    emit_data->args[0], emit_data->args[1]);
1102 }
1103
1104 static void
1105 u64mod_emit_cpu(
1106    const struct lp_build_tgsi_action * action,
1107    struct lp_build_tgsi_context * bld_base,
1108    struct lp_build_emit_data * emit_data)
1109 {
1110    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1111    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint64_bld,
1112                                         PIPE_FUNC_EQUAL, emit_data->args[1],
1113                                         bld_base->uint64_bld.zero);
1114    /* We want to make sure that we never divide/mod by zero to not
1115     * generate sigfpe. We don't want to crash just because the
1116     * shader is doing something weird. */
1117    LLVMValueRef divisor = LLVMBuildOr(builder,
1118                                       div_mask,
1119                                       emit_data->args[1], "");
1120    LLVMValueRef result = lp_build_mod(&bld_base->uint64_bld,
1121                                       emit_data->args[0], divisor);
1122    /* umod by zero doesn't have a guaranteed return value chose -1 for now. */
1123    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
1124                                                     div_mask,
1125                                                     result, "");
1126 }
1127
1128 static void
1129 i64mod_emit_cpu(
1130    const struct lp_build_tgsi_action * action,
1131    struct lp_build_tgsi_context * bld_base,
1132    struct lp_build_emit_data * emit_data)
1133 {
1134    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1135    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint64_bld,
1136                                         PIPE_FUNC_EQUAL, emit_data->args[1],
1137                                         bld_base->uint64_bld.zero);
1138    /* We want to make sure that we never divide/mod by zero to not
1139     * generate sigfpe. We don't want to crash just because the
1140     * shader is doing something weird. */
1141    LLVMValueRef divisor = LLVMBuildOr(builder,
1142                                       div_mask,
1143                                       emit_data->args[1], "");
1144    LLVMValueRef result = lp_build_mod(&bld_base->int64_bld,
1145                                       emit_data->args[0], divisor);
1146    /* umod by zero doesn't have a guaranteed return value chose -1 for now. */
1147    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
1148                                                     div_mask,
1149                                                     result, "");
1150 }
1151
1152 static void
1153 u64div_emit_cpu(
1154    const struct lp_build_tgsi_action * action,
1155    struct lp_build_tgsi_context * bld_base,
1156    struct lp_build_emit_data * emit_data)
1157 {
1158
1159    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1160    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint64_bld,
1161                                         PIPE_FUNC_EQUAL, emit_data->args[1],
1162                                         bld_base->uint64_bld.zero);
1163    /* We want to make sure that we never divide/mod by zero to not
1164     * generate sigfpe. We don't want to crash just because the
1165     * shader is doing something weird. */
1166    LLVMValueRef divisor = LLVMBuildOr(builder,
1167                                       div_mask,
1168                                       emit_data->args[1], "");
1169    LLVMValueRef result = LLVMBuildUDiv(builder,
1170                                        emit_data->args[0], divisor, "");
1171    /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10 */
1172    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
1173                                                     div_mask,
1174                                                     result, "");
1175 }
1176
1177 static void
1178 i64div_emit_cpu(
1179    const struct lp_build_tgsi_action * action,
1180    struct lp_build_tgsi_context * bld_base,
1181    struct lp_build_emit_data * emit_data)
1182 {
1183
1184    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1185    LLVMValueRef div_mask = lp_build_cmp(&bld_base->int64_bld,
1186                                         PIPE_FUNC_EQUAL, emit_data->args[1],
1187                                         bld_base->int64_bld.zero);
1188    /* We want to make sure that we never divide/mod by zero to not
1189     * generate sigfpe. We don't want to crash just because the
1190     * shader is doing something weird. */
1191    LLVMValueRef divisor = LLVMBuildOr(builder,
1192                                       div_mask,
1193                                       emit_data->args[1], "");
1194    LLVMValueRef result = LLVMBuildSDiv(builder,
1195                                        emit_data->args[0], divisor, "");
1196    /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10 */
1197    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
1198                                                     div_mask,
1199                                                     result, "");
1200 }
1201
1202 static void
1203 f2u64_emit(
1204    const struct lp_build_tgsi_action * action,
1205    struct lp_build_tgsi_context * bld_base,
1206    struct lp_build_emit_data * emit_data)
1207 {
1208    emit_data->output[emit_data->chan] =
1209       LLVMBuildFPToUI(bld_base->base.gallivm->builder,
1210                       emit_data->args[0],
1211                       bld_base->uint64_bld.vec_type, "");
1212 }
1213
1214 static void
1215 f2i64_emit(
1216    const struct lp_build_tgsi_action * action,
1217    struct lp_build_tgsi_context * bld_base,
1218    struct lp_build_emit_data * emit_data)
1219 {
1220    emit_data->output[emit_data->chan] =
1221       LLVMBuildFPToSI(bld_base->base.gallivm->builder,
1222                       emit_data->args[0],
1223                       bld_base->int64_bld.vec_type, "");
1224 }
1225
1226 static void
1227 u2i64_emit(
1228    const struct lp_build_tgsi_action * action,
1229    struct lp_build_tgsi_context * bld_base,
1230    struct lp_build_emit_data * emit_data)
1231 {
1232    emit_data->output[emit_data->chan] =
1233       LLVMBuildZExt(bld_base->base.gallivm->builder,
1234                       emit_data->args[0],
1235                       bld_base->uint64_bld.vec_type, "");
1236 }
1237
1238 static void
1239 i2i64_emit(
1240    const struct lp_build_tgsi_action * action,
1241    struct lp_build_tgsi_context * bld_base,
1242    struct lp_build_emit_data * emit_data)
1243 {
1244    emit_data->output[emit_data->chan] =
1245       LLVMBuildSExt(bld_base->base.gallivm->builder,
1246                       emit_data->args[0],
1247                       bld_base->int64_bld.vec_type, "");
1248 }
1249
1250 static void
1251 i642f_emit(
1252    const struct lp_build_tgsi_action * action,
1253    struct lp_build_tgsi_context * bld_base,
1254    struct lp_build_emit_data * emit_data)
1255 {
1256    emit_data->output[emit_data->chan] =
1257       LLVMBuildSIToFP(bld_base->base.gallivm->builder,
1258                       emit_data->args[0],
1259                       bld_base->base.vec_type, "");
1260 }
1261
1262 static void
1263 u642f_emit(
1264    const struct lp_build_tgsi_action * action,
1265    struct lp_build_tgsi_context * bld_base,
1266    struct lp_build_emit_data * emit_data)
1267 {
1268    emit_data->output[emit_data->chan] =
1269       LLVMBuildUIToFP(bld_base->base.gallivm->builder,
1270                       emit_data->args[0],
1271                       bld_base->base.vec_type, "");
1272 }
1273
1274 static void
1275 i642d_emit(
1276    const struct lp_build_tgsi_action * action,
1277    struct lp_build_tgsi_context * bld_base,
1278    struct lp_build_emit_data * emit_data)
1279 {
1280    emit_data->output[emit_data->chan] =
1281       LLVMBuildSIToFP(bld_base->base.gallivm->builder,
1282                       emit_data->args[0],
1283                       bld_base->dbl_bld.vec_type, "");
1284 }
1285
1286 static void
1287 u642d_emit(
1288    const struct lp_build_tgsi_action * action,
1289    struct lp_build_tgsi_context * bld_base,
1290    struct lp_build_emit_data * emit_data)
1291 {
1292    emit_data->output[emit_data->chan] =
1293       LLVMBuildUIToFP(bld_base->base.gallivm->builder,
1294                       emit_data->args[0],
1295                       bld_base->dbl_bld.vec_type, "");
1296 }
1297
1298 void
1299 lp_set_default_actions(struct lp_build_tgsi_context * bld_base)
1300 {
1301    bld_base->op_actions[TGSI_OPCODE_DP2] = dp2_action;
1302    bld_base->op_actions[TGSI_OPCODE_DP3] = dp3_action;
1303    bld_base->op_actions[TGSI_OPCODE_DP4] = dp4_action;
1304    bld_base->op_actions[TGSI_OPCODE_DP2A] = dp2a_action;
1305    bld_base->op_actions[TGSI_OPCODE_DPH] = dph_action;
1306    bld_base->op_actions[TGSI_OPCODE_DST] = dst_action;
1307    bld_base->op_actions[TGSI_OPCODE_EXP] = exp_action;
1308    bld_base->op_actions[TGSI_OPCODE_LIT] = lit_action;
1309    bld_base->op_actions[TGSI_OPCODE_LOG] = log_action;
1310    bld_base->op_actions[TGSI_OPCODE_PK2H] = pk2h_action;
1311    bld_base->op_actions[TGSI_OPCODE_RSQ] = rsq_action;
1312    bld_base->op_actions[TGSI_OPCODE_SQRT] = sqrt_action;
1313    bld_base->op_actions[TGSI_OPCODE_POW] = pow_action;
1314    bld_base->op_actions[TGSI_OPCODE_SCS] = scs_action;
1315    bld_base->op_actions[TGSI_OPCODE_UP2H] = up2h_action;
1316    bld_base->op_actions[TGSI_OPCODE_XPD] = xpd_action;
1317
1318    bld_base->op_actions[TGSI_OPCODE_BREAKC].fetch_args = scalar_unary_fetch_args;
1319    bld_base->op_actions[TGSI_OPCODE_SWITCH].fetch_args = scalar_unary_fetch_args;
1320    bld_base->op_actions[TGSI_OPCODE_CASE].fetch_args = scalar_unary_fetch_args;
1321    bld_base->op_actions[TGSI_OPCODE_COS].fetch_args = scalar_unary_fetch_args;
1322    bld_base->op_actions[TGSI_OPCODE_EX2].fetch_args = scalar_unary_fetch_args;
1323    bld_base->op_actions[TGSI_OPCODE_IF].fetch_args = scalar_unary_fetch_args;
1324    bld_base->op_actions[TGSI_OPCODE_UIF].fetch_args = scalar_unary_fetch_args;
1325    bld_base->op_actions[TGSI_OPCODE_KILL_IF].fetch_args = kil_fetch_args;
1326    bld_base->op_actions[TGSI_OPCODE_KILL].fetch_args = kilp_fetch_args;
1327    bld_base->op_actions[TGSI_OPCODE_RCP].fetch_args = scalar_unary_fetch_args;
1328    bld_base->op_actions[TGSI_OPCODE_SIN].fetch_args = scalar_unary_fetch_args;
1329    bld_base->op_actions[TGSI_OPCODE_LG2].fetch_args = scalar_unary_fetch_args;
1330
1331    bld_base->op_actions[TGSI_OPCODE_ADD].emit = add_emit;
1332    bld_base->op_actions[TGSI_OPCODE_ARR].emit = arr_emit;
1333    bld_base->op_actions[TGSI_OPCODE_CLAMP].emit = clamp_emit;
1334    bld_base->op_actions[TGSI_OPCODE_END].emit = end_emit;
1335    bld_base->op_actions[TGSI_OPCODE_FRC].emit = frc_emit;
1336    bld_base->op_actions[TGSI_OPCODE_LRP].emit = lrp_emit;
1337    bld_base->op_actions[TGSI_OPCODE_MAD].emit = mad_emit;
1338    bld_base->op_actions[TGSI_OPCODE_MOV].emit = mov_emit;
1339    bld_base->op_actions[TGSI_OPCODE_MUL].emit = mul_emit;
1340    bld_base->op_actions[TGSI_OPCODE_DIV].emit = fdiv_emit;
1341    bld_base->op_actions[TGSI_OPCODE_RCP].emit = rcp_emit;
1342
1343    bld_base->op_actions[TGSI_OPCODE_UARL].emit = mov_emit;
1344    bld_base->op_actions[TGSI_OPCODE_F2U].emit = f2u_emit;
1345    bld_base->op_actions[TGSI_OPCODE_U2F].emit = u2f_emit;
1346    bld_base->op_actions[TGSI_OPCODE_UMAD].emit = umad_emit;
1347    bld_base->op_actions[TGSI_OPCODE_UMUL].emit = umul_emit;
1348    bld_base->op_actions[TGSI_OPCODE_IMUL_HI].emit = imul_hi_emit;
1349    bld_base->op_actions[TGSI_OPCODE_UMUL_HI].emit = umul_hi_emit;
1350
1351    bld_base->op_actions[TGSI_OPCODE_MAX].emit = fmax_emit;
1352    bld_base->op_actions[TGSI_OPCODE_MIN].emit = fmin_emit;
1353
1354    bld_base->op_actions[TGSI_OPCODE_DADD].emit = add_emit;
1355    bld_base->op_actions[TGSI_OPCODE_DMAX].emit = fmax_emit;
1356    bld_base->op_actions[TGSI_OPCODE_DMIN].emit = fmin_emit;
1357    bld_base->op_actions[TGSI_OPCODE_DMUL].emit = mul_emit;
1358
1359    bld_base->op_actions[TGSI_OPCODE_D2F].emit = d2f_emit;
1360    bld_base->op_actions[TGSI_OPCODE_D2I].emit = d2i_emit;
1361    bld_base->op_actions[TGSI_OPCODE_D2U].emit = d2u_emit;
1362
1363    bld_base->op_actions[TGSI_OPCODE_F2D].emit = f2d_emit;
1364    bld_base->op_actions[TGSI_OPCODE_I2D].emit = i2d_emit;
1365    bld_base->op_actions[TGSI_OPCODE_U2D].emit = u2d_emit;
1366
1367    bld_base->op_actions[TGSI_OPCODE_DMAD].emit = dmad_emit;
1368
1369    bld_base->op_actions[TGSI_OPCODE_DRCP].emit = drcp_emit;
1370    bld_base->op_actions[TGSI_OPCODE_DFRAC].emit = dfrac_emit;
1371
1372    bld_base->op_actions[TGSI_OPCODE_U64MUL].emit = u64mul_emit;
1373
1374    bld_base->op_actions[TGSI_OPCODE_F2I64].emit = f2i64_emit;
1375    bld_base->op_actions[TGSI_OPCODE_F2U64].emit = f2u64_emit;
1376
1377    bld_base->op_actions[TGSI_OPCODE_D2I64].emit = f2i64_emit;
1378    bld_base->op_actions[TGSI_OPCODE_D2U64].emit = f2u64_emit;
1379
1380    bld_base->op_actions[TGSI_OPCODE_I2I64].emit = i2i64_emit;
1381    bld_base->op_actions[TGSI_OPCODE_U2I64].emit = u2i64_emit;
1382
1383    bld_base->op_actions[TGSI_OPCODE_I642F].emit = i642f_emit;
1384    bld_base->op_actions[TGSI_OPCODE_U642F].emit = u642f_emit;
1385
1386    bld_base->op_actions[TGSI_OPCODE_I642F].emit = i642f_emit;
1387    bld_base->op_actions[TGSI_OPCODE_U642F].emit = u642f_emit;
1388
1389    bld_base->op_actions[TGSI_OPCODE_I642D].emit = i642d_emit;
1390    bld_base->op_actions[TGSI_OPCODE_U642D].emit = u642d_emit;
1391
1392 }
1393
1394 /* CPU Only default actions */
1395
1396 /* These actions are CPU only, because they could potentially output SSE
1397  * intrinsics.
1398  */
1399
1400 /* TGSI_OPCODE_ADD (CPU Only) */
1401 static void
1402 add_emit_cpu(
1403    const struct lp_build_tgsi_action * action,
1404    struct lp_build_tgsi_context * bld_base,
1405    struct lp_build_emit_data * emit_data)
1406 {
1407    emit_data->output[emit_data->chan] = lp_build_add(&bld_base->base,
1408                                    emit_data->args[0], emit_data->args[1]);
1409 }
1410
1411 /* TGSI_OPCODE_AND (CPU Only) */
1412 static void
1413 and_emit_cpu(
1414    const struct lp_build_tgsi_action * action,
1415    struct lp_build_tgsi_context * bld_base,
1416    struct lp_build_emit_data * emit_data)
1417 {
1418    emit_data->output[emit_data->chan] = lp_build_and(&bld_base->uint_bld,
1419                                    emit_data->args[0], emit_data->args[1]);
1420 }
1421
1422 /* TGSI_OPCODE_ARL (CPU Only) */
1423 static void
1424 arl_emit_cpu(
1425    const struct lp_build_tgsi_action * action,
1426    struct lp_build_tgsi_context * bld_base,
1427    struct lp_build_emit_data * emit_data)
1428 {
1429    LLVMValueRef tmp;
1430    tmp = lp_build_floor(&bld_base->base,
1431                         emit_data->args[0]);
1432    emit_data->output[emit_data->chan] = LLVMBuildFPToSI(bld_base->base.gallivm->builder, tmp,
1433                                                         bld_base->uint_bld.vec_type, "");
1434 }
1435
1436 /* TGSI_OPCODE_ARR (CPU Only) */
1437 static void
1438 arr_emit_cpu(
1439    const struct lp_build_tgsi_action * action,
1440    struct lp_build_tgsi_context * bld_base,
1441    struct lp_build_emit_data * emit_data)
1442 {
1443    emit_data->output[emit_data->chan] = lp_build_iround(&bld_base->base, emit_data->args[0]);
1444 }
1445
1446 /* TGSI_OPCODE_CEIL (CPU Only) */
1447 static void
1448 ceil_emit_cpu(
1449    const struct lp_build_tgsi_action * action,
1450    struct lp_build_tgsi_context * bld_base,
1451    struct lp_build_emit_data * emit_data)
1452 {
1453    emit_data->output[emit_data->chan] = lp_build_ceil(&bld_base->base,
1454                                                       emit_data->args[0]);
1455 }
1456
1457 /* TGSI_OPCODE_CMP (CPU Only) */
1458 static void
1459 cmp_emit_cpu(
1460    const struct lp_build_tgsi_action * action,
1461    struct lp_build_tgsi_context * bld_base,
1462    struct lp_build_emit_data * emit_data)
1463 {
1464    LLVMValueRef cond = lp_build_cmp(&bld_base->base, PIPE_FUNC_LESS,
1465                                    emit_data->args[0], bld_base->base.zero);
1466    emit_data->output[emit_data->chan] = lp_build_select(&bld_base->base,
1467                                 cond, emit_data->args[1], emit_data->args[2]);
1468 }
1469
1470 /* TGSI_OPCODE_UCMP (CPU Only) */
1471 static void
1472 ucmp_emit_cpu(
1473    const struct lp_build_tgsi_action * action,
1474    struct lp_build_tgsi_context * bld_base,
1475    struct lp_build_emit_data * emit_data)
1476 {
1477    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1478    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1479    LLVMValueRef unsigned_cond =
1480       LLVMBuildBitCast(builder, emit_data->args[0], uint_bld->vec_type, "");
1481    LLVMValueRef cond = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL,
1482                                     unsigned_cond,
1483                                     uint_bld->zero);
1484    emit_data->output[emit_data->chan] =
1485       lp_build_select(&bld_base->base,
1486                       cond, emit_data->args[1], emit_data->args[2]);
1487 }
1488
1489 /* TGSI_OPCODE_COS (CPU Only) */
1490 static void
1491 cos_emit_cpu(
1492    const struct lp_build_tgsi_action * action,
1493    struct lp_build_tgsi_context * bld_base,
1494    struct lp_build_emit_data * emit_data)
1495 {
1496    emit_data->output[emit_data->chan] = lp_build_cos(&bld_base->base,
1497                                                        emit_data->args[0]);
1498 }
1499
1500 /* TGSI_OPCODE_DIV (CPU Only) */
1501 static void
1502 div_emit_cpu(
1503    const struct lp_build_tgsi_action * action,
1504    struct lp_build_tgsi_context * bld_base,
1505    struct lp_build_emit_data * emit_data)
1506 {
1507    emit_data->output[emit_data->chan] = lp_build_div(&bld_base->base,
1508                                    emit_data->args[0], emit_data->args[1]);
1509 }
1510
1511 /* TGSI_OPCODE_EX2 (CPU Only) */
1512 static void
1513 ex2_emit_cpu(
1514    const struct lp_build_tgsi_action * action,
1515    struct lp_build_tgsi_context * bld_base,
1516    struct lp_build_emit_data * emit_data)
1517 {
1518    emit_data->output[emit_data->chan] = lp_build_exp2(&bld_base->base,
1519                                                         emit_data->args[0]);
1520 }
1521
1522 /* TGSI_OPCODE_F2I (CPU Only) */
1523 static void
1524 f2i_emit_cpu(
1525    const struct lp_build_tgsi_action * action,
1526    struct lp_build_tgsi_context * bld_base,
1527    struct lp_build_emit_data * emit_data)
1528 {
1529    emit_data->output[emit_data->chan] = lp_build_itrunc(&bld_base->base,
1530                                                         emit_data->args[0]);
1531 }
1532
1533 /* TGSI_OPCODE_FSET Helper (CPU Only) */
1534 static void
1535 fset_emit_cpu(
1536    const struct lp_build_tgsi_action * action,
1537    struct lp_build_tgsi_context * bld_base,
1538    struct lp_build_emit_data * emit_data,
1539    unsigned pipe_func)
1540 {
1541    LLVMValueRef cond;
1542
1543    if (pipe_func != PIPE_FUNC_NOTEQUAL) {
1544       cond = lp_build_cmp_ordered(&bld_base->base, pipe_func,
1545                                   emit_data->args[0], emit_data->args[1]);
1546    }
1547    else {
1548       cond = lp_build_cmp(&bld_base->base, pipe_func,
1549                           emit_data->args[0], emit_data->args[1]);
1550
1551    }
1552    emit_data->output[emit_data->chan] = cond;
1553 }
1554
1555
1556 /* TGSI_OPCODE_FSEQ (CPU Only) */
1557 static void
1558 fseq_emit_cpu(
1559    const struct lp_build_tgsi_action * action,
1560    struct lp_build_tgsi_context * bld_base,
1561    struct lp_build_emit_data * emit_data)
1562 {
1563    fset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_EQUAL);
1564 }
1565
1566 /* TGSI_OPCODE_ISGE (CPU Only) */
1567 static void
1568 fsge_emit_cpu(
1569    const struct lp_build_tgsi_action * action,
1570    struct lp_build_tgsi_context * bld_base,
1571    struct lp_build_emit_data * emit_data)
1572 {
1573    fset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
1574 }
1575
1576 /* TGSI_OPCODE_ISLT (CPU Only) */
1577 static void
1578 fslt_emit_cpu(
1579    const struct lp_build_tgsi_action * action,
1580    struct lp_build_tgsi_context * bld_base,
1581    struct lp_build_emit_data * emit_data)
1582 {
1583    fset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
1584 }
1585
1586 /* TGSI_OPCODE_USNE (CPU Only) */
1587
1588 static void
1589 fsne_emit_cpu(
1590    const struct lp_build_tgsi_action * action,
1591    struct lp_build_tgsi_context * bld_base,
1592    struct lp_build_emit_data * emit_data)
1593 {
1594    fset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_NOTEQUAL);
1595 }
1596
1597 /* TGSI_OPCODE_FLR (CPU Only) */
1598
1599 static void
1600 flr_emit_cpu(
1601    const struct lp_build_tgsi_action * action,
1602    struct lp_build_tgsi_context * bld_base,
1603    struct lp_build_emit_data * emit_data)
1604 {
1605    emit_data->output[emit_data->chan] = lp_build_floor(&bld_base->base,
1606                                                          emit_data->args[0]);
1607 }
1608
1609 /* TGSI_OPCODE_I2F (CPU Only) */
1610 static void
1611 i2f_emit_cpu(
1612    const struct lp_build_tgsi_action * action,
1613    struct lp_build_tgsi_context * bld_base,
1614    struct lp_build_emit_data * emit_data)
1615 {
1616    emit_data->output[emit_data->chan] = lp_build_int_to_float(&bld_base->base,
1617                                                               emit_data->args[0]);
1618 }
1619
1620 /* TGSI_OPCODE_IABS (CPU Only) */
1621 static void
1622 iabs_emit_cpu(
1623    const struct lp_build_tgsi_action * action,
1624    struct lp_build_tgsi_context * bld_base,
1625    struct lp_build_emit_data * emit_data)
1626 {
1627    emit_data->output[emit_data->chan] = lp_build_abs(&bld_base->int_bld,
1628                                                        emit_data->args[0]);
1629 }
1630
1631 /* TGSI_OPCODE_IDIV (CPU Only) */
1632 static void
1633 idiv_emit_cpu(
1634    const struct lp_build_tgsi_action * action,
1635    struct lp_build_tgsi_context * bld_base,
1636    struct lp_build_emit_data * emit_data)
1637 {
1638    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1639    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint_bld,
1640                                         PIPE_FUNC_EQUAL, emit_data->args[1],
1641                                         bld_base->uint_bld.zero);
1642    /* We want to make sure that we never divide/mod by zero to not
1643     * generate sigfpe. We don't want to crash just because the
1644     * shader is doing something weird. */
1645    LLVMValueRef divisor = LLVMBuildOr(builder,
1646                                       div_mask,
1647                                       emit_data->args[1], "");
1648    LLVMValueRef result = lp_build_div(&bld_base->int_bld,
1649                                       emit_data->args[0], divisor);
1650    LLVMValueRef not_div_mask = LLVMBuildNot(builder,
1651                                             div_mask,"");
1652    /* idiv by zero doesn't have a guaranteed return value chose 0 for now. */
1653    emit_data->output[emit_data->chan] = LLVMBuildAnd(builder,
1654                                                      not_div_mask,
1655                                                      result, "");
1656 }
1657
1658 /* TGSI_OPCODE_INEG (CPU Only) */
1659 static void
1660 ineg_emit_cpu(
1661    const struct lp_build_tgsi_action * action,
1662    struct lp_build_tgsi_context * bld_base,
1663    struct lp_build_emit_data * emit_data)
1664 {
1665    emit_data->output[emit_data->chan] = lp_build_sub(&bld_base->int_bld,
1666                                                      bld_base->int_bld.zero,
1667                                                      emit_data->args[0]);
1668 }
1669
1670 /* TGSI_OPCODE_ISET Helper (CPU Only) */
1671 static void
1672 iset_emit_cpu(
1673    const struct lp_build_tgsi_action * action,
1674    struct lp_build_tgsi_context * bld_base,
1675    struct lp_build_emit_data * emit_data,
1676    unsigned pipe_func)
1677 {
1678    LLVMValueRef cond = lp_build_cmp(&bld_base->int_bld, pipe_func,
1679                                     emit_data->args[0], emit_data->args[1]);
1680    emit_data->output[emit_data->chan] = cond;
1681 }
1682
1683 /* TGSI_OPCODE_IMAX (CPU Only) */
1684 static void
1685 imax_emit_cpu(
1686    const struct lp_build_tgsi_action * action,
1687    struct lp_build_tgsi_context * bld_base,
1688    struct lp_build_emit_data * emit_data)
1689 {
1690    emit_data->output[emit_data->chan] = lp_build_max(&bld_base->int_bld,
1691                                    emit_data->args[0], emit_data->args[1]);
1692 }
1693
1694 /* TGSI_OPCODE_IMIN (CPU Only) */
1695 static void
1696 imin_emit_cpu(
1697    const struct lp_build_tgsi_action * action,
1698    struct lp_build_tgsi_context * bld_base,
1699    struct lp_build_emit_data * emit_data)
1700 {
1701    emit_data->output[emit_data->chan] = lp_build_min(&bld_base->int_bld,
1702                                    emit_data->args[0], emit_data->args[1]);
1703 }
1704
1705 /* TGSI_OPCODE_ISGE (CPU Only) */
1706 static void
1707 isge_emit_cpu(
1708    const struct lp_build_tgsi_action * action,
1709    struct lp_build_tgsi_context * bld_base,
1710    struct lp_build_emit_data * emit_data)
1711 {
1712    iset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
1713 }
1714
1715 /* TGSI_OPCODE_ISHR (CPU Only) */
1716 static void
1717 ishr_emit_cpu(
1718    const struct lp_build_tgsi_action * action,
1719    struct lp_build_tgsi_context * bld_base,
1720    struct lp_build_emit_data * emit_data)
1721 {
1722    struct lp_build_context *int_bld = &bld_base->int_bld;
1723    LLVMValueRef mask = lp_build_const_vec(int_bld->gallivm, int_bld->type,
1724                                           int_bld->type.width - 1);
1725    LLVMValueRef masked_count = lp_build_and(int_bld, emit_data->args[1], mask);
1726    emit_data->output[emit_data->chan] = lp_build_shr(int_bld, emit_data->args[0],
1727                                                      masked_count);
1728 }
1729
1730 /* TGSI_OPCODE_ISLT (CPU Only) */
1731 static void
1732 islt_emit_cpu(
1733    const struct lp_build_tgsi_action * action,
1734    struct lp_build_tgsi_context * bld_base,
1735    struct lp_build_emit_data * emit_data)
1736 {
1737    iset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
1738 }
1739
1740
1741 /* TGSI_OPCODE_ISSG (CPU Only) */
1742 static void
1743 issg_emit_cpu(
1744    const struct lp_build_tgsi_action * action,
1745    struct lp_build_tgsi_context * bld_base,
1746    struct lp_build_emit_data * emit_data)
1747 {
1748    emit_data->output[emit_data->chan] = lp_build_sgn(&bld_base->int_bld,
1749                                                        emit_data->args[0]);
1750 }
1751
1752 /* TGSI_OPCODE_LG2 (CPU Only) */
1753 static void
1754 lg2_emit_cpu(
1755    const struct lp_build_tgsi_action * action,
1756    struct lp_build_tgsi_context * bld_base,
1757    struct lp_build_emit_data * emit_data)
1758 {
1759    emit_data->output[emit_data->chan] = lp_build_log2_safe(&bld_base->base,
1760                                                            emit_data->args[0]);
1761 }
1762
1763 /* TGSI_OPCODE_LOG (CPU Only) */
1764 static void
1765 log_emit_cpu(
1766    const struct lp_build_tgsi_action * action,
1767    struct lp_build_tgsi_context * bld_base,
1768    struct lp_build_emit_data * emit_data)
1769 {
1770    LLVMValueRef p_floor_log2;
1771    LLVMValueRef p_exp;
1772    LLVMValueRef p_log2;
1773    LLVMValueRef src0 = emit_data->args[0];
1774
1775    lp_build_log2_approx(&bld_base->base, src0,
1776                         &p_exp, &p_floor_log2, &p_log2, FALSE);
1777
1778    emit_data->output[TGSI_CHAN_X] = p_floor_log2;
1779
1780    emit_data->output[TGSI_CHAN_Y] = lp_build_emit_llvm_binary(bld_base,
1781                                              TGSI_OPCODE_DIV,
1782                                              src0, p_exp);
1783    emit_data->output[TGSI_CHAN_Z] = p_log2;
1784
1785    emit_data->output[TGSI_CHAN_W] = bld_base->base.one;
1786
1787 }
1788
1789 /* TGSI_OPCODE_MAD (CPU Only) */
1790
1791 static void
1792 mad_emit_cpu(
1793    const struct lp_build_tgsi_action * action,
1794    struct lp_build_tgsi_context * bld_base,
1795    struct lp_build_emit_data * emit_data)
1796 {
1797    emit_data->output[emit_data->chan] =
1798       lp_build_mad(&bld_base->base,
1799                    emit_data->args[0], emit_data->args[1], emit_data->args[2]);
1800 }
1801
1802 /* TGSI_OPCODE_MAX (CPU Only) */
1803
1804 static void
1805 max_emit_cpu(
1806    const struct lp_build_tgsi_action * action,
1807    struct lp_build_tgsi_context * bld_base,
1808    struct lp_build_emit_data * emit_data)
1809 {
1810    emit_data->output[emit_data->chan] =
1811       lp_build_max_ext(&bld_base->base,
1812                        emit_data->args[0], emit_data->args[1],
1813                        GALLIVM_NAN_RETURN_OTHER);
1814 }
1815
1816 /* TGSI_OPCODE_MIN (CPU Only) */
1817 static void
1818 min_emit_cpu(
1819    const struct lp_build_tgsi_action * action,
1820    struct lp_build_tgsi_context * bld_base,
1821    struct lp_build_emit_data * emit_data)
1822 {
1823    emit_data->output[emit_data->chan] =
1824       lp_build_min_ext(&bld_base->base,
1825                        emit_data->args[0], emit_data->args[1],
1826                        GALLIVM_NAN_RETURN_OTHER);
1827 }
1828
1829 /* TGSI_OPCODE_MOD (CPU Only) */
1830 static void
1831 mod_emit_cpu(
1832    const struct lp_build_tgsi_action * action,
1833    struct lp_build_tgsi_context * bld_base,
1834    struct lp_build_emit_data * emit_data)
1835 {
1836    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1837    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint_bld,
1838                                         PIPE_FUNC_EQUAL, emit_data->args[1],
1839                                         bld_base->uint_bld.zero);
1840    /* We want to make sure that we never divide/mod by zero to not
1841     * generate sigfpe. We don't want to crash just because the
1842     * shader is doing something weird. */
1843    LLVMValueRef divisor = LLVMBuildOr(builder,
1844                                       div_mask,
1845                                       emit_data->args[1], "");
1846    LLVMValueRef result = lp_build_mod(&bld_base->int_bld,
1847                                       emit_data->args[0], divisor);
1848    /* umod by zero doesn't have a guaranteed return value chose -1 for now. */
1849    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
1850                                                     div_mask,
1851                                                     result, "");
1852 }
1853
1854 /* TGSI_OPCODE_NOT */
1855 static void
1856 not_emit_cpu(
1857    const struct lp_build_tgsi_action * action,
1858    struct lp_build_tgsi_context * bld_base,
1859    struct lp_build_emit_data * emit_data)
1860 {
1861    emit_data->output[emit_data->chan] = lp_build_not(&bld_base->uint_bld,
1862                                                      emit_data->args[0]);
1863 }
1864
1865 /* TGSI_OPCODE_OR (CPU Only) */
1866 static void
1867 or_emit_cpu(
1868    const struct lp_build_tgsi_action * action,
1869    struct lp_build_tgsi_context * bld_base,
1870    struct lp_build_emit_data * emit_data)
1871 {
1872    emit_data->output[emit_data->chan] = lp_build_or(&bld_base->uint_bld,
1873                                    emit_data->args[0], emit_data->args[1]);
1874 }
1875
1876 /* TGSI_OPCODE_POW (CPU Only) */
1877 static void
1878 pow_emit_cpu(
1879    const struct lp_build_tgsi_action * action,
1880    struct lp_build_tgsi_context * bld_base,
1881    struct lp_build_emit_data * emit_data)
1882 {
1883    emit_data->output[emit_data->chan] = lp_build_pow(&bld_base->base,
1884                                    emit_data->args[0], emit_data->args[1]);
1885 }
1886
1887
1888 /* TGSI_OPCODE_RCP (CPU Only) */
1889
1890 static void
1891 rcp_emit_cpu(
1892    const struct lp_build_tgsi_action * action,
1893    struct lp_build_tgsi_context * bld_base,
1894    struct lp_build_emit_data * emit_data)
1895 {
1896    emit_data->output[emit_data->chan] = lp_build_rcp(&bld_base->base,
1897                                                        emit_data->args[0]);
1898 }
1899
1900 /* Reciprical squareroot (CPU Only) */
1901 static void
1902 recip_sqrt_emit_cpu(
1903    const struct lp_build_tgsi_action * action,
1904    struct lp_build_tgsi_context * bld_base,
1905    struct lp_build_emit_data * emit_data)
1906 {
1907    emit_data->output[emit_data->chan] = lp_build_rsqrt(&bld_base->base,
1908                                                          emit_data->args[0]);
1909 }
1910
1911 static void
1912 sqrt_emit_cpu(
1913    const struct lp_build_tgsi_action * action,
1914    struct lp_build_tgsi_context * bld_base,
1915    struct lp_build_emit_data * emit_data)
1916 {
1917    emit_data->output[emit_data->chan] = lp_build_sqrt(&bld_base->base,
1918                                                       emit_data->args[0]);
1919 }
1920
1921
1922 /* TGSI_OPCODE_ROUND (CPU Only) */
1923 static void
1924 round_emit_cpu(
1925    const struct lp_build_tgsi_action * action,
1926    struct lp_build_tgsi_context * bld_base,
1927    struct lp_build_emit_data * emit_data)
1928 {
1929    emit_data->output[emit_data->chan] = lp_build_round(&bld_base->base,
1930                                                          emit_data->args[0]);
1931 }
1932
1933 /* TGSI_OPCODE_SET Helper (CPU Only) */
1934
1935 static void
1936 set_emit_cpu(
1937    const struct lp_build_tgsi_action * action,
1938    struct lp_build_tgsi_context * bld_base,
1939    struct lp_build_emit_data * emit_data,
1940    unsigned pipe_func)
1941 {
1942    LLVMValueRef cond;
1943
1944    if (pipe_func != PIPE_FUNC_NOTEQUAL) {
1945       cond = lp_build_cmp_ordered(&bld_base->base, pipe_func,
1946                                   emit_data->args[0], emit_data->args[1]);
1947    }
1948    else {
1949       cond = lp_build_cmp(&bld_base->base, pipe_func,
1950                           emit_data->args[0], emit_data->args[1]);
1951
1952    }
1953    emit_data->output[emit_data->chan] = lp_build_select(&bld_base->base,
1954                                           cond,
1955                                           bld_base->base.one,
1956                                           bld_base->base.zero);
1957 }
1958
1959 /* TGSI_OPCODE_SEQ (CPU Only) */
1960
1961 static void
1962 seq_emit_cpu(
1963    const struct lp_build_tgsi_action * action,
1964    struct lp_build_tgsi_context * bld_base,
1965    struct lp_build_emit_data * emit_data)
1966 {
1967    set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_EQUAL);
1968 }
1969
1970 /* TGSI_OPCODE_SGE (CPU Only) */
1971 static void
1972 sge_emit_cpu(
1973    const struct lp_build_tgsi_action * action,
1974    struct lp_build_tgsi_context * bld_base,
1975    struct lp_build_emit_data * emit_data)
1976 {
1977    set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
1978 }
1979
1980 /* TGSI_OPCODE_SGT (CPU Only)*/
1981
1982 static void
1983 sgt_emit_cpu(
1984    const struct lp_build_tgsi_action * action,
1985    struct lp_build_tgsi_context * bld_base,
1986    struct lp_build_emit_data * emit_data)
1987 {
1988    set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GREATER);
1989 }
1990
1991 /* TGSI_OPCODE_SHL (CPU Only) */
1992 static void
1993 shl_emit_cpu(
1994    const struct lp_build_tgsi_action * action,
1995    struct lp_build_tgsi_context * bld_base,
1996    struct lp_build_emit_data * emit_data)
1997 {
1998    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1999    LLVMValueRef mask = lp_build_const_vec(uint_bld->gallivm, uint_bld->type,
2000                                           uint_bld->type.width - 1);
2001    LLVMValueRef masked_count = lp_build_and(uint_bld, emit_data->args[1], mask);
2002    emit_data->output[emit_data->chan] = lp_build_shl(uint_bld, emit_data->args[0],
2003                                                      masked_count);
2004 }
2005
2006 /* TGSI_OPCODE_SIN (CPU Only) */
2007 static void
2008 sin_emit_cpu(
2009    const struct lp_build_tgsi_action * action,
2010    struct lp_build_tgsi_context * bld_base,
2011    struct lp_build_emit_data * emit_data)
2012 {
2013    emit_data->output[emit_data->chan] = lp_build_sin(&bld_base->base,
2014                                                        emit_data->args[0]);
2015 }
2016
2017 /* TGSI_OPCODE_SLE (CPU Only) */
2018 static void
2019 sle_emit_cpu(
2020    const struct lp_build_tgsi_action * action,
2021    struct lp_build_tgsi_context * bld_base,
2022    struct lp_build_emit_data * emit_data)
2023 {
2024    set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LEQUAL);
2025 }
2026
2027 /* TGSI_OPCODE_SLT (CPU Only) */
2028 static void
2029 slt_emit_cpu(
2030    const struct lp_build_tgsi_action * action,
2031    struct lp_build_tgsi_context * bld_base,
2032    struct lp_build_emit_data * emit_data)
2033 {
2034    set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
2035 }
2036
2037 /* TGSI_OPCODE_SNE (CPU Only) */
2038
2039 static void
2040 sne_emit_cpu(
2041    const struct lp_build_tgsi_action * action,
2042    struct lp_build_tgsi_context * bld_base,
2043    struct lp_build_emit_data * emit_data)
2044 {
2045    set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_NOTEQUAL);
2046 }
2047
2048 /* TGSI_OPCODE_SSG (CPU Only) */
2049
2050 static void
2051 ssg_emit_cpu(
2052    const struct lp_build_tgsi_action * action,
2053    struct lp_build_tgsi_context * bld_base,
2054    struct lp_build_emit_data * emit_data)
2055 {
2056    emit_data->output[emit_data->chan] = lp_build_sgn(&bld_base->base,
2057                                                        emit_data->args[0]);
2058 }
2059
2060 /* TGSI_OPCODE_TRUNC (CPU Only) */
2061
2062 static void
2063 trunc_emit_cpu(
2064    const struct lp_build_tgsi_action * action,
2065    struct lp_build_tgsi_context * bld_base,
2066    struct lp_build_emit_data * emit_data)
2067 {
2068    emit_data->output[emit_data->chan] = lp_build_trunc(&bld_base->base,
2069                                                          emit_data->args[0]);
2070 }
2071
2072 /* TGSI_OPCODE_UADD (CPU Only) */
2073 static void
2074 uadd_emit_cpu(
2075    const struct lp_build_tgsi_action * action,
2076    struct lp_build_tgsi_context * bld_base,
2077    struct lp_build_emit_data * emit_data)
2078 {
2079    emit_data->output[emit_data->chan] = lp_build_add(&bld_base->uint_bld,
2080                                    emit_data->args[0], emit_data->args[1]);
2081 }
2082
2083 /* TGSI_OPCODE_UDIV (CPU Only) */
2084 static void
2085 udiv_emit_cpu(
2086    const struct lp_build_tgsi_action * action,
2087    struct lp_build_tgsi_context * bld_base,
2088    struct lp_build_emit_data * emit_data)
2089 {
2090
2091    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2092    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint_bld,
2093                                         PIPE_FUNC_EQUAL, emit_data->args[1],
2094                                         bld_base->uint_bld.zero);
2095    /* We want to make sure that we never divide/mod by zero to not
2096     * generate sigfpe. We don't want to crash just because the
2097     * shader is doing something weird. */
2098    LLVMValueRef divisor = LLVMBuildOr(builder,
2099                                       div_mask,
2100                                       emit_data->args[1], "");
2101    LLVMValueRef result = lp_build_div(&bld_base->uint_bld,
2102                                       emit_data->args[0], divisor);
2103    /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10 */
2104    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
2105                                                     div_mask,
2106                                                     result, "");
2107 }
2108
2109 /* TGSI_OPCODE_UMAX (CPU Only) */
2110 static void
2111 umax_emit_cpu(
2112    const struct lp_build_tgsi_action * action,
2113    struct lp_build_tgsi_context * bld_base,
2114    struct lp_build_emit_data * emit_data)
2115 {
2116    emit_data->output[emit_data->chan] = lp_build_max(&bld_base->uint_bld,
2117                                    emit_data->args[0], emit_data->args[1]);
2118 }
2119
2120 /* TGSI_OPCODE_UMIN (CPU Only) */
2121 static void
2122 umin_emit_cpu(
2123    const struct lp_build_tgsi_action * action,
2124    struct lp_build_tgsi_context * bld_base,
2125    struct lp_build_emit_data * emit_data)
2126 {
2127    emit_data->output[emit_data->chan] = lp_build_min(&bld_base->uint_bld,
2128                                    emit_data->args[0], emit_data->args[1]);
2129 }
2130
2131 /* TGSI_OPCODE_UMOD (CPU Only) */
2132 static void
2133 umod_emit_cpu(
2134    const struct lp_build_tgsi_action * action,
2135    struct lp_build_tgsi_context * bld_base,
2136    struct lp_build_emit_data * emit_data)
2137 {
2138    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2139    LLVMValueRef div_mask = lp_build_cmp(&bld_base->uint_bld,
2140                                         PIPE_FUNC_EQUAL, emit_data->args[1],
2141                                         bld_base->uint_bld.zero);
2142    /* We want to make sure that we never divide/mod by zero to not
2143     * generate sigfpe. We don't want to crash just because the
2144     * shader is doing something weird. */
2145    LLVMValueRef divisor = LLVMBuildOr(builder,
2146                                       div_mask,
2147                                       emit_data->args[1], "");
2148    LLVMValueRef result = lp_build_mod(&bld_base->uint_bld,
2149                                       emit_data->args[0], divisor);
2150    /* umod by zero is guaranteed to return 0xffffffff */
2151    emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
2152                                                     div_mask,
2153                                                     result, "");
2154 }
2155
2156 /* TGSI_OPCODE_USET Helper (CPU Only) */
2157 static void
2158 uset_emit_cpu(
2159    const struct lp_build_tgsi_action * action,
2160    struct lp_build_tgsi_context * bld_base,
2161    struct lp_build_emit_data * emit_data,
2162    unsigned pipe_func)
2163 {
2164    LLVMValueRef cond = lp_build_cmp(&bld_base->uint_bld, pipe_func,
2165                                     emit_data->args[0], emit_data->args[1]);
2166    emit_data->output[emit_data->chan] = cond;
2167 }
2168
2169
2170 /* TGSI_OPCODE_USEQ (CPU Only) */
2171 static void
2172 useq_emit_cpu(
2173    const struct lp_build_tgsi_action * action,
2174    struct lp_build_tgsi_context * bld_base,
2175    struct lp_build_emit_data * emit_data)
2176 {
2177    uset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_EQUAL);
2178 }
2179
2180 /* TGSI_OPCODE_ISGE (CPU Only) */
2181 static void
2182 usge_emit_cpu(
2183    const struct lp_build_tgsi_action * action,
2184    struct lp_build_tgsi_context * bld_base,
2185    struct lp_build_emit_data * emit_data)
2186 {
2187    uset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
2188 }
2189
2190 /* TGSI_OPCODE_USHR (CPU Only) */
2191 static void
2192 ushr_emit_cpu(
2193    const struct lp_build_tgsi_action * action,
2194    struct lp_build_tgsi_context * bld_base,
2195    struct lp_build_emit_data * emit_data)
2196 {
2197    struct lp_build_context *uint_bld = &bld_base->uint_bld;
2198    LLVMValueRef mask = lp_build_const_vec(uint_bld->gallivm, uint_bld->type,
2199                                           uint_bld->type.width - 1);
2200    LLVMValueRef masked_count = lp_build_and(uint_bld, emit_data->args[1], mask);
2201    emit_data->output[emit_data->chan] = lp_build_shr(uint_bld, emit_data->args[0],
2202                                                      masked_count);
2203 }
2204
2205 /* TGSI_OPCODE_ISLT (CPU Only) */
2206 static void
2207 uslt_emit_cpu(
2208    const struct lp_build_tgsi_action * action,
2209    struct lp_build_tgsi_context * bld_base,
2210    struct lp_build_emit_data * emit_data)
2211 {
2212    uset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
2213 }
2214
2215 /* TGSI_OPCODE_USNE (CPU Only) */
2216
2217 static void
2218 usne_emit_cpu(
2219    const struct lp_build_tgsi_action * action,
2220    struct lp_build_tgsi_context * bld_base,
2221    struct lp_build_emit_data * emit_data)
2222 {
2223    uset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_NOTEQUAL);
2224 }
2225
2226 /* TGSI_OPCODE_XOR */
2227 static void
2228 xor_emit_cpu(
2229    const struct lp_build_tgsi_action * action,
2230    struct lp_build_tgsi_context * bld_base,
2231    struct lp_build_emit_data * emit_data)
2232 {
2233    emit_data->output[emit_data->chan] = lp_build_xor(&bld_base->uint_bld,
2234                                                      emit_data->args[0],
2235                                                      emit_data->args[1]);
2236 }
2237
2238 /* TGSI_OPCODE_DABS (CPU Only) */
2239 static void
2240 dabs_emit_cpu(
2241    const struct lp_build_tgsi_action * action,
2242    struct lp_build_tgsi_context * bld_base,
2243    struct lp_build_emit_data * emit_data)
2244 {
2245    emit_data->output[emit_data->chan] = lp_build_abs(&bld_base->dbl_bld,
2246                                                        emit_data->args[0]);
2247 }
2248
2249 /* TGSI_OPCODE_DNEG (CPU Only) */
2250 static void
2251 dneg_emit_cpu(
2252    const struct lp_build_tgsi_action * action,
2253    struct lp_build_tgsi_context * bld_base,
2254    struct lp_build_emit_data * emit_data)
2255 {
2256    emit_data->output[emit_data->chan] = lp_build_sub(&bld_base->dbl_bld,
2257                                                      bld_base->dbl_bld.zero,
2258                                                      emit_data->args[0]);
2259 }
2260
2261 /* TGSI_OPCODE_DSET Helper (CPU Only) */
2262 static void
2263 dset_emit_cpu(
2264    const struct lp_build_tgsi_action * action,
2265    struct lp_build_tgsi_context * bld_base,
2266    struct lp_build_emit_data * emit_data,
2267    unsigned pipe_func)
2268 {
2269    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2270    LLVMValueRef cond = lp_build_cmp(&bld_base->dbl_bld, pipe_func,
2271                                     emit_data->args[0], emit_data->args[1]);
2272    /* arguments were 64 bit but store as 32 bit */
2273    cond = LLVMBuildTrunc(builder, cond, bld_base->int_bld.int_vec_type, "");
2274    emit_data->output[emit_data->chan] = cond;
2275 }
2276
2277 /* TGSI_OPCODE_DSEQ (CPU Only) */
2278 static void
2279 dseq_emit_cpu(
2280    const struct lp_build_tgsi_action * action,
2281    struct lp_build_tgsi_context * bld_base,
2282    struct lp_build_emit_data * emit_data)
2283 {
2284    dset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_EQUAL);
2285 }
2286
2287 /* TGSI_OPCODE_DSGE (CPU Only) */
2288 static void
2289 dsge_emit_cpu(
2290    const struct lp_build_tgsi_action * action,
2291    struct lp_build_tgsi_context * bld_base,
2292    struct lp_build_emit_data * emit_data)
2293 {
2294    dset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
2295 }
2296
2297 /* TGSI_OPCODE_DSLT (CPU Only) */
2298 static void
2299 dslt_emit_cpu(
2300    const struct lp_build_tgsi_action * action,
2301    struct lp_build_tgsi_context * bld_base,
2302    struct lp_build_emit_data * emit_data)
2303 {
2304    dset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
2305 }
2306
2307 /* TGSI_OPCODE_DSNE (CPU Only) */
2308 static void
2309 dsne_emit_cpu(
2310    const struct lp_build_tgsi_action * action,
2311    struct lp_build_tgsi_context * bld_base,
2312    struct lp_build_emit_data * emit_data)
2313 {
2314    dset_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_NOTEQUAL);
2315 }
2316
2317 /* Double Reciprocal squareroot (CPU Only) */
2318 static void
2319 drecip_sqrt_emit_cpu(
2320    const struct lp_build_tgsi_action * action,
2321    struct lp_build_tgsi_context * bld_base,
2322    struct lp_build_emit_data * emit_data)
2323 {
2324    emit_data->output[emit_data->chan] = lp_build_rsqrt(&bld_base->dbl_bld,
2325                                                          emit_data->args[0]);
2326 }
2327
2328 /* Double Squareroot (CPU Only) */
2329 static void
2330 dsqrt_emit_cpu(
2331    const struct lp_build_tgsi_action * action,
2332    struct lp_build_tgsi_context * bld_base,
2333    struct lp_build_emit_data * emit_data)
2334 {
2335    emit_data->output[emit_data->chan] = lp_build_sqrt(&bld_base->dbl_bld,
2336                                                       emit_data->args[0]);
2337 }
2338
2339 static void
2340 i64abs_emit_cpu(
2341    const struct lp_build_tgsi_action * action,
2342    struct lp_build_tgsi_context * bld_base,
2343    struct lp_build_emit_data * emit_data)
2344 {
2345    emit_data->output[emit_data->chan] = lp_build_abs(&bld_base->int64_bld,
2346                                                        emit_data->args[0]);
2347 }
2348
2349 static void
2350 i64ssg_emit_cpu(
2351    const struct lp_build_tgsi_action * action,
2352    struct lp_build_tgsi_context * bld_base,
2353    struct lp_build_emit_data * emit_data)
2354 {
2355    emit_data->output[emit_data->chan] = lp_build_sgn(&bld_base->int64_bld,
2356                                                        emit_data->args[0]);
2357 }
2358
2359 static void
2360 i64neg_emit_cpu(
2361    const struct lp_build_tgsi_action * action,
2362    struct lp_build_tgsi_context * bld_base,
2363    struct lp_build_emit_data * emit_data)
2364 {
2365    emit_data->output[emit_data->chan] = lp_build_sub(&bld_base->int64_bld,
2366                                                      bld_base->int64_bld.zero,
2367                                                      emit_data->args[0]);
2368 }
2369
2370 static void
2371 u64set_emit_cpu(
2372    const struct lp_build_tgsi_action * action,
2373    struct lp_build_tgsi_context * bld_base,
2374    struct lp_build_emit_data * emit_data,
2375    unsigned pipe_func)
2376 {
2377    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2378    LLVMValueRef cond = lp_build_cmp(&bld_base->uint64_bld, pipe_func,
2379                                     emit_data->args[0], emit_data->args[1]);
2380    /* arguments were 64 bit but store as 32 bit */
2381    cond = LLVMBuildTrunc(builder, cond, bld_base->int_bld.int_vec_type, "");
2382    emit_data->output[emit_data->chan] = cond;
2383 }
2384
2385 static void
2386 u64seq_emit_cpu(
2387    const struct lp_build_tgsi_action * action,
2388    struct lp_build_tgsi_context * bld_base,
2389    struct lp_build_emit_data * emit_data)
2390 {
2391    u64set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_EQUAL);
2392 }
2393
2394 static void
2395 u64sne_emit_cpu(
2396    const struct lp_build_tgsi_action * action,
2397    struct lp_build_tgsi_context * bld_base,
2398    struct lp_build_emit_data * emit_data)
2399 {
2400    u64set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_NOTEQUAL);
2401 }
2402
2403 static void
2404 u64slt_emit_cpu(
2405    const struct lp_build_tgsi_action * action,
2406    struct lp_build_tgsi_context * bld_base,
2407    struct lp_build_emit_data * emit_data)
2408 {
2409    u64set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
2410 }
2411
2412 static void
2413 u64sge_emit_cpu(
2414    const struct lp_build_tgsi_action * action,
2415    struct lp_build_tgsi_context * bld_base,
2416    struct lp_build_emit_data * emit_data)
2417 {
2418    u64set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
2419 }
2420
2421 static void
2422 i64set_emit_cpu(
2423    const struct lp_build_tgsi_action * action,
2424    struct lp_build_tgsi_context * bld_base,
2425    struct lp_build_emit_data * emit_data,
2426    unsigned pipe_func)
2427 {
2428    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2429    LLVMValueRef cond = lp_build_cmp(&bld_base->int64_bld, pipe_func,
2430                                     emit_data->args[0], emit_data->args[1]);
2431    /* arguments were 64 bit but store as 32 bit */
2432    cond = LLVMBuildTrunc(builder, cond, bld_base->int_bld.int_vec_type, "");
2433    emit_data->output[emit_data->chan] = cond;
2434 }
2435
2436 static void
2437 i64slt_emit_cpu(
2438    const struct lp_build_tgsi_action * action,
2439    struct lp_build_tgsi_context * bld_base,
2440    struct lp_build_emit_data * emit_data)
2441 {
2442    i64set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_LESS);
2443 }
2444
2445 static void
2446 i64sge_emit_cpu(
2447    const struct lp_build_tgsi_action * action,
2448    struct lp_build_tgsi_context * bld_base,
2449    struct lp_build_emit_data * emit_data)
2450 {
2451    i64set_emit_cpu(action, bld_base, emit_data, PIPE_FUNC_GEQUAL);
2452 }
2453
2454 static void
2455 u64max_emit_cpu(
2456    const struct lp_build_tgsi_action * action,
2457    struct lp_build_tgsi_context * bld_base,
2458    struct lp_build_emit_data * emit_data)
2459 {
2460    emit_data->output[emit_data->chan] = lp_build_max(&bld_base->uint64_bld,
2461                                    emit_data->args[0], emit_data->args[1]);
2462 }
2463
2464 static void
2465 u64min_emit_cpu(
2466    const struct lp_build_tgsi_action * action,
2467    struct lp_build_tgsi_context * bld_base,
2468    struct lp_build_emit_data * emit_data)
2469 {
2470    emit_data->output[emit_data->chan] = lp_build_min(&bld_base->uint64_bld,
2471                                    emit_data->args[0], emit_data->args[1]);
2472 }
2473
2474 static void
2475 i64max_emit_cpu(
2476    const struct lp_build_tgsi_action * action,
2477    struct lp_build_tgsi_context * bld_base,
2478    struct lp_build_emit_data * emit_data)
2479 {
2480    emit_data->output[emit_data->chan] = lp_build_max(&bld_base->int64_bld,
2481                                    emit_data->args[0], emit_data->args[1]);
2482 }
2483
2484 static void
2485 i64min_emit_cpu(
2486    const struct lp_build_tgsi_action * action,
2487    struct lp_build_tgsi_context * bld_base,
2488    struct lp_build_emit_data * emit_data)
2489 {
2490    emit_data->output[emit_data->chan] = lp_build_min(&bld_base->int64_bld,
2491                                    emit_data->args[0], emit_data->args[1]);
2492 }
2493
2494 static void
2495 u64add_emit_cpu(
2496    const struct lp_build_tgsi_action * action,
2497    struct lp_build_tgsi_context * bld_base,
2498    struct lp_build_emit_data * emit_data)
2499 {
2500    emit_data->output[emit_data->chan] = lp_build_add(&bld_base->uint64_bld,
2501                                    emit_data->args[0], emit_data->args[1]);
2502 }
2503
2504 static void
2505 u64shl_emit_cpu(
2506    const struct lp_build_tgsi_action * action,
2507    struct lp_build_tgsi_context * bld_base,
2508    struct lp_build_emit_data * emit_data)
2509 {
2510    struct lp_build_context *uint_bld = &bld_base->uint64_bld;
2511    LLVMValueRef mask = lp_build_const_vec(uint_bld->gallivm, uint_bld->type,
2512                                           uint_bld->type.width - 1);
2513    LLVMValueRef masked_count = lp_build_and(uint_bld, emit_data->args[1], mask);
2514    emit_data->output[emit_data->chan] = lp_build_shl(uint_bld, emit_data->args[0],
2515                                                      masked_count);
2516 }
2517
2518 static void
2519 i64shr_emit_cpu(
2520    const struct lp_build_tgsi_action * action,
2521    struct lp_build_tgsi_context * bld_base,
2522    struct lp_build_emit_data * emit_data)
2523 {
2524    struct lp_build_context *int_bld = &bld_base->int64_bld;
2525    LLVMValueRef mask = lp_build_const_vec(int_bld->gallivm, int_bld->type,
2526                                           int_bld->type.width - 1);
2527    LLVMValueRef masked_count = lp_build_and(int_bld, emit_data->args[1], mask);
2528    emit_data->output[emit_data->chan] = lp_build_shr(int_bld, emit_data->args[0],
2529                                                      masked_count);
2530 }
2531
2532 static void
2533 u64shr_emit_cpu(
2534    const struct lp_build_tgsi_action * action,
2535    struct lp_build_tgsi_context * bld_base,
2536    struct lp_build_emit_data * emit_data)
2537 {
2538    struct lp_build_context *uint_bld = &bld_base->uint64_bld;
2539    LLVMValueRef mask = lp_build_const_vec(uint_bld->gallivm, uint_bld->type,
2540                                           uint_bld->type.width - 1);
2541    LLVMValueRef masked_count = lp_build_and(uint_bld, emit_data->args[1], mask);
2542    emit_data->output[emit_data->chan] = lp_build_shr(uint_bld, emit_data->args[0],
2543                                                      masked_count);
2544 }
2545
2546 void
2547 lp_set_default_actions_cpu(
2548    struct lp_build_tgsi_context * bld_base)
2549 {
2550    lp_set_default_actions(bld_base);
2551    bld_base->op_actions[TGSI_OPCODE_ADD].emit = add_emit_cpu;
2552    bld_base->op_actions[TGSI_OPCODE_AND].emit = and_emit_cpu;
2553    bld_base->op_actions[TGSI_OPCODE_ARL].emit = arl_emit_cpu;
2554    bld_base->op_actions[TGSI_OPCODE_ARR].emit = arr_emit_cpu;
2555    bld_base->op_actions[TGSI_OPCODE_CEIL].emit = ceil_emit_cpu;
2556    bld_base->op_actions[TGSI_OPCODE_COS].emit = cos_emit_cpu;
2557    bld_base->op_actions[TGSI_OPCODE_CMP].emit = cmp_emit_cpu;
2558    bld_base->op_actions[TGSI_OPCODE_DIV].emit = div_emit_cpu;
2559    bld_base->op_actions[TGSI_OPCODE_EX2].emit = ex2_emit_cpu;
2560    bld_base->op_actions[TGSI_OPCODE_F2I].emit = f2i_emit_cpu;
2561    bld_base->op_actions[TGSI_OPCODE_FLR].emit = flr_emit_cpu;
2562    bld_base->op_actions[TGSI_OPCODE_FSEQ].emit = fseq_emit_cpu;
2563    bld_base->op_actions[TGSI_OPCODE_FSGE].emit = fsge_emit_cpu;
2564    bld_base->op_actions[TGSI_OPCODE_FSLT].emit = fslt_emit_cpu;
2565    bld_base->op_actions[TGSI_OPCODE_FSNE].emit = fsne_emit_cpu;
2566
2567    bld_base->op_actions[TGSI_OPCODE_I2F].emit = i2f_emit_cpu;
2568    bld_base->op_actions[TGSI_OPCODE_IABS].emit = iabs_emit_cpu;
2569    bld_base->op_actions[TGSI_OPCODE_IDIV].emit = idiv_emit_cpu;
2570    bld_base->op_actions[TGSI_OPCODE_INEG].emit = ineg_emit_cpu;
2571    bld_base->op_actions[TGSI_OPCODE_IMAX].emit = imax_emit_cpu;
2572    bld_base->op_actions[TGSI_OPCODE_IMIN].emit = imin_emit_cpu;
2573    bld_base->op_actions[TGSI_OPCODE_ISGE].emit = isge_emit_cpu;
2574    bld_base->op_actions[TGSI_OPCODE_ISHR].emit = ishr_emit_cpu;
2575    bld_base->op_actions[TGSI_OPCODE_ISLT].emit = islt_emit_cpu;
2576    bld_base->op_actions[TGSI_OPCODE_ISSG].emit = issg_emit_cpu;
2577    bld_base->op_actions[TGSI_OPCODE_IMUL_HI].emit = imul_hi_emit_cpu;
2578    bld_base->op_actions[TGSI_OPCODE_UMUL_HI].emit = umul_hi_emit_cpu;
2579
2580    bld_base->op_actions[TGSI_OPCODE_LG2].emit = lg2_emit_cpu;
2581    bld_base->op_actions[TGSI_OPCODE_LOG].emit = log_emit_cpu;
2582    bld_base->op_actions[TGSI_OPCODE_MAD].emit = mad_emit_cpu;
2583    bld_base->op_actions[TGSI_OPCODE_MAX].emit = max_emit_cpu;
2584    bld_base->op_actions[TGSI_OPCODE_MIN].emit = min_emit_cpu;
2585    bld_base->op_actions[TGSI_OPCODE_MOD].emit = mod_emit_cpu;
2586    bld_base->op_actions[TGSI_OPCODE_NOT].emit = not_emit_cpu;
2587    bld_base->op_actions[TGSI_OPCODE_OR].emit = or_emit_cpu;
2588    bld_base->op_actions[TGSI_OPCODE_POW].emit = pow_emit_cpu;
2589    bld_base->op_actions[TGSI_OPCODE_RCP].emit = rcp_emit_cpu;
2590    bld_base->op_actions[TGSI_OPCODE_ROUND].emit = round_emit_cpu;
2591    bld_base->op_actions[TGSI_OPCODE_SEQ].emit = seq_emit_cpu;
2592    bld_base->op_actions[TGSI_OPCODE_SGE].emit = sge_emit_cpu;
2593    bld_base->op_actions[TGSI_OPCODE_SGT].emit = sgt_emit_cpu;
2594    bld_base->op_actions[TGSI_OPCODE_SIN].emit = sin_emit_cpu;
2595    bld_base->op_actions[TGSI_OPCODE_SHL].emit = shl_emit_cpu;
2596    bld_base->op_actions[TGSI_OPCODE_SLE].emit = sle_emit_cpu;
2597    bld_base->op_actions[TGSI_OPCODE_SLT].emit = slt_emit_cpu;
2598    bld_base->op_actions[TGSI_OPCODE_SNE].emit = sne_emit_cpu;
2599    bld_base->op_actions[TGSI_OPCODE_SSG].emit = ssg_emit_cpu;
2600    bld_base->op_actions[TGSI_OPCODE_TRUNC].emit = trunc_emit_cpu;
2601
2602    bld_base->rsq_action.emit = recip_sqrt_emit_cpu;
2603    bld_base->sqrt_action.emit = sqrt_emit_cpu;
2604
2605    bld_base->op_actions[TGSI_OPCODE_UADD].emit = uadd_emit_cpu;
2606    bld_base->op_actions[TGSI_OPCODE_UCMP].emit = ucmp_emit_cpu;
2607    bld_base->op_actions[TGSI_OPCODE_UDIV].emit = udiv_emit_cpu;
2608    bld_base->op_actions[TGSI_OPCODE_UMAX].emit = umax_emit_cpu;
2609    bld_base->op_actions[TGSI_OPCODE_UMIN].emit = umin_emit_cpu;
2610    bld_base->op_actions[TGSI_OPCODE_UMOD].emit = umod_emit_cpu;
2611    bld_base->op_actions[TGSI_OPCODE_USEQ].emit = useq_emit_cpu;
2612    bld_base->op_actions[TGSI_OPCODE_USGE].emit = usge_emit_cpu;
2613    bld_base->op_actions[TGSI_OPCODE_USHR].emit = ushr_emit_cpu;
2614    bld_base->op_actions[TGSI_OPCODE_USLT].emit = uslt_emit_cpu;
2615    bld_base->op_actions[TGSI_OPCODE_USNE].emit = usne_emit_cpu;
2616
2617    bld_base->op_actions[TGSI_OPCODE_XOR].emit = xor_emit_cpu;
2618
2619    bld_base->op_actions[TGSI_OPCODE_DABS].emit = dabs_emit_cpu;
2620    bld_base->op_actions[TGSI_OPCODE_DNEG].emit = dneg_emit_cpu;
2621    bld_base->op_actions[TGSI_OPCODE_DSEQ].emit = dseq_emit_cpu;
2622    bld_base->op_actions[TGSI_OPCODE_DSGE].emit = dsge_emit_cpu;
2623    bld_base->op_actions[TGSI_OPCODE_DSLT].emit = dslt_emit_cpu;
2624    bld_base->op_actions[TGSI_OPCODE_DSNE].emit = dsne_emit_cpu;
2625
2626    bld_base->op_actions[TGSI_OPCODE_DRSQ].emit = drecip_sqrt_emit_cpu;
2627    bld_base->op_actions[TGSI_OPCODE_DSQRT].emit = dsqrt_emit_cpu;
2628
2629    bld_base->op_actions[TGSI_OPCODE_I64ABS].emit = i64abs_emit_cpu;
2630    bld_base->op_actions[TGSI_OPCODE_I64SSG].emit = i64ssg_emit_cpu;
2631    bld_base->op_actions[TGSI_OPCODE_I64NEG].emit = i64neg_emit_cpu;
2632
2633    bld_base->op_actions[TGSI_OPCODE_U64SEQ].emit = u64seq_emit_cpu;
2634    bld_base->op_actions[TGSI_OPCODE_U64SNE].emit = u64sne_emit_cpu;
2635    bld_base->op_actions[TGSI_OPCODE_U64SLT].emit = u64slt_emit_cpu;
2636    bld_base->op_actions[TGSI_OPCODE_U64SGE].emit = u64sge_emit_cpu;
2637    bld_base->op_actions[TGSI_OPCODE_I64SLT].emit = i64slt_emit_cpu;
2638    bld_base->op_actions[TGSI_OPCODE_I64SGE].emit = i64sge_emit_cpu;
2639
2640    bld_base->op_actions[TGSI_OPCODE_U64MIN].emit = u64min_emit_cpu;
2641    bld_base->op_actions[TGSI_OPCODE_U64MAX].emit = u64max_emit_cpu;
2642    bld_base->op_actions[TGSI_OPCODE_I64MIN].emit = i64min_emit_cpu;
2643    bld_base->op_actions[TGSI_OPCODE_I64MAX].emit = i64max_emit_cpu;
2644
2645    bld_base->op_actions[TGSI_OPCODE_U64ADD].emit = u64add_emit_cpu;
2646    bld_base->op_actions[TGSI_OPCODE_U64MOD].emit = u64mod_emit_cpu;
2647    bld_base->op_actions[TGSI_OPCODE_I64MOD].emit = i64mod_emit_cpu;
2648    bld_base->op_actions[TGSI_OPCODE_U64DIV].emit = u64div_emit_cpu;
2649    bld_base->op_actions[TGSI_OPCODE_I64DIV].emit = i64div_emit_cpu;
2650
2651    bld_base->op_actions[TGSI_OPCODE_U64SHL].emit = u64shl_emit_cpu;
2652    bld_base->op_actions[TGSI_OPCODE_I64SHR].emit = i64shr_emit_cpu;
2653    bld_base->op_actions[TGSI_OPCODE_U64SHR].emit = u64shr_emit_cpu;
2654 }