src/gallium/drivers/radeon/radeon_setup_tgsi_llvm.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors: Tom Stellard <thomas.stellard@amd.com>
  24  *
  25  */
  26 #include "radeon_llvm.h"
  27
  28 #include "gallivm/lp_bld_const.h"
  29 #include "gallivm/lp_bld_gather.h"
  30 #include "gallivm/lp_bld_flow.h"
  31 #include "gallivm/lp_bld_init.h"
  32 #include "gallivm/lp_bld_intr.h"
  33 #include "gallivm/lp_bld_swizzle.h"
  34 #include "tgsi/tgsi_info.h"
  35 #include "tgsi/tgsi_parse.h"
  36 #include "util/u_math.h"
  37 #include "util/u_memory.h"
  38 #include "util/u_debug.h"
  39
  40 #include <llvm-c/Core.h>
  41 #include <llvm-c/Transforms/Scalar.h>
  42
  43 static struct radeon_llvm_loop * get_current_loop(struct radeon_llvm_context * ctx)
  44 {
  45         return ctx->loop_depth > 0 ? ctx->loop + (ctx->loop_depth - 1) : NULL;
  46 }
  47
  48 static struct radeon_llvm_branch * get_current_branch(
  49         struct radeon_llvm_context * ctx)
  50 {
  51         return ctx->branch_depth > 0 ?
  52                         ctx->branch + (ctx->branch_depth - 1) : NULL;
  53 }
  54
  55 unsigned radeon_llvm_reg_index_soa(unsigned index, unsigned chan)
  56 {
  57  return (index * 4) + chan;
  58 }
  59
  60 static LLVMValueRef emit_swizzle(
  61         struct lp_build_tgsi_context * bld_base,
  62         LLVMValueRef value,
  63         unsigned swizzle_x,
  64         unsigned swizzle_y,
  65         unsigned swizzle_z,
  66         unsigned swizzle_w)
  67 {
  68         LLVMValueRef swizzles[4];
  69         LLVMTypeRef i32t =
  70                 LLVMInt32TypeInContext(bld_base->base.gallivm->context);
  71
  72         swizzles[0] = LLVMConstInt(i32t, swizzle_x, 0);
  73         swizzles[1] = LLVMConstInt(i32t, swizzle_y, 0);
  74         swizzles[2] = LLVMConstInt(i32t, swizzle_z, 0);
  75         swizzles[3] = LLVMConstInt(i32t, swizzle_w, 0);
  76
  77         return LLVMBuildShuffleVector(bld_base->base.gallivm->builder,
  78                 value,
  79                 LLVMGetUndef(LLVMTypeOf(value)),
  80                 LLVMConstVector(swizzles, 4), "");
  81 }
  82
  83 static LLVMValueRef
  84 emit_array_index(
  85         struct lp_build_tgsi_soa_context *bld,
  86         const struct tgsi_full_src_register *reg,
  87         unsigned swizzle)
  88 {
  89         struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
  90
  91         LLVMValueRef addr = LLVMBuildLoad(gallivm->builder,
  92         bld->addr[reg->Indirect.Index][swizzle], "");
  93         LLVMValueRef offset = lp_build_const_int32(gallivm, reg->Register.Index);
  94         LLVMValueRef hw_index = LLVMBuildAdd(gallivm->builder, addr, offset, "");
  95         LLVMValueRef soa_index = LLVMBuildMul(gallivm->builder, hw_index,
  96         lp_build_const_int32(gallivm, 4), "");
  97         LLVMValueRef array_index = LLVMBuildAdd(gallivm->builder, soa_index,
  98         lp_build_const_int32(gallivm, swizzle), "");
  99
 100         return array_index;
 101 }
 102
 103 static LLVMValueRef
 104 emit_fetch_immediate(
 105         struct lp_build_tgsi_context *bld_base,
 106         const struct tgsi_full_src_register *reg,
 107         enum tgsi_opcode_type type,
 108         unsigned swizzle)
 109 {
 110         LLVMTypeRef ctype;
 111         LLVMContextRef ctx = bld_base->base.gallivm->context;
 112
 113         switch (type) {
 114         case TGSI_TYPE_UNSIGNED:
 115         case TGSI_TYPE_SIGNED:
 116                 ctype = LLVMInt32TypeInContext(ctx);
 117                 break;
 118         case TGSI_TYPE_UNTYPED:
 119         case TGSI_TYPE_FLOAT:
 120                 ctype = LLVMFloatTypeInContext(ctx);
 121                 break;
 122         default:
 123                 ctype = 0;
 124                 break;
 125         }
 126
 127         struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
 128         return LLVMConstBitCast(bld->immediates[reg->Register.Index][swizzle], ctype);
 129 }
 130
 131 static LLVMValueRef
 132 emit_fetch_input(
 133         struct lp_build_tgsi_context *bld_base,
 134         const struct tgsi_full_src_register *reg,
 135         enum tgsi_opcode_type type,
 136         unsigned swizzle)
 137 {
 138         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 139         if (swizzle == ~0) {
 140                 LLVMValueRef values[TGSI_NUM_CHANNELS] = {};
 141                 unsigned chan;
 142                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 143                         values[chan] = ctx->inputs[radeon_llvm_reg_index_soa(
 144                                                 reg->Register.Index, chan)];
 145                 }
 146                 return lp_build_gather_values(bld_base->base.gallivm, values,
 147                                                 TGSI_NUM_CHANNELS);
 148         } else {
 149                 return bitcast(bld_base, type, ctx->inputs[radeon_llvm_reg_index_soa(reg->Register.Index, swizzle)]);
 150         }
 151 }
 152
 153 static LLVMValueRef
 154 emit_fetch_temporary(
 155         struct lp_build_tgsi_context *bld_base,
 156         const struct tgsi_full_src_register *reg,
 157         enum tgsi_opcode_type type,
 158         unsigned swizzle)
 159 {
 160         struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
 161         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 162         if (swizzle == ~0) {
 163                 LLVMValueRef values[TGSI_NUM_CHANNELS] = {};
 164                 unsigned chan;
 165                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 166                         values[chan] = emit_fetch_temporary(bld_base, reg, type, chan);
 167                 }
 168                 return lp_build_gather_values(bld_base->base.gallivm, values,
 169                                                 TGSI_NUM_CHANNELS);
 170         }
 171
 172         if (reg->Register.Indirect) {
 173                 LLVMValueRef array_index = emit_array_index(bld, reg, swizzle);
 174                 LLVMValueRef ptr = LLVMBuildGEP(builder, bld->temps_array, &array_index,
 175                                                 1, "");
 176                 return LLVMBuildLoad(builder, ptr, "");
 177         } else {
 178                 LLVMValueRef temp_ptr;
 179                 temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle);
 180                 return bitcast(bld_base,type,LLVMBuildLoad(builder, temp_ptr, ""));
 181         }
 182 }
 183
 184 static LLVMValueRef
 185 emit_fetch_output(
 186         struct lp_build_tgsi_context *bld_base,
 187         const struct tgsi_full_src_register *reg,
 188         enum tgsi_opcode_type type,
 189         unsigned swizzle)
 190 {
 191         struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
 192         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 193          if (reg->Register.Indirect) {
 194                 LLVMValueRef array_index = emit_array_index(bld, reg, swizzle);
 195                 LLVMValueRef ptr = LLVMBuildGEP(builder, bld->outputs_array, &array_index,
 196                                                 1, "");
 197                 return LLVMBuildLoad(builder, ptr, "");
 198         } else {
 199                 LLVMValueRef temp_ptr;
 200                 temp_ptr = lp_get_output_ptr(bld, reg->Register.Index, swizzle);
 201                 return LLVMBuildLoad(builder, temp_ptr, "");
 202          }
 203 }
 204
 205 static void emit_declaration(
 206         struct lp_build_tgsi_context * bld_base,
 207         const struct tgsi_full_declaration *decl)
 208 {
 209         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 210         switch(decl->Declaration.File) {
 211         case TGSI_FILE_ADDRESS:
 212         {
 213                  unsigned idx;
 214                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 215                         unsigned chan;
 216                         for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 217                                  ctx->soa.addr[idx][chan] = lp_build_alloca(
 218                                         &ctx->gallivm,
 219                                         ctx->soa.bld_base.uint_bld.elem_type, "");
 220                         }
 221                 }
 222                 break;
 223         }
 224
 225         case TGSI_FILE_TEMPORARY:
 226                 lp_emit_declaration_soa(bld_base, decl);
 227                 break;
 228
 229         case TGSI_FILE_INPUT:
 230         {
 231                 unsigned idx;
 232                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 233                         ctx->load_input(ctx, idx, decl);
 234                 }
 235         }
 236         break;
 237
 238         case TGSI_FILE_SYSTEM_VALUE:
 239         {
 240                 unsigned idx;
 241                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 242                         ctx->load_system_value(ctx, idx, decl);
 243                 }
 244         }
 245         break;
 246
 247         case TGSI_FILE_OUTPUT:
 248         {
 249                 unsigned idx;
 250                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 251                         unsigned chan;
 252                         assert(idx < RADEON_LLVM_MAX_OUTPUTS);
 253                         for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 254                                 ctx->soa.outputs[idx][chan] = lp_build_alloca(&ctx->gallivm,
 255                                         ctx->soa.bld_base.base.elem_type, "");
 256                         }
 257                 }
 258
 259                 ctx->output_reg_count = MAX2(ctx->output_reg_count,
 260                                                          decl->Range.Last + 1);
 261                 break;
 262         }
 263
 264         default:
 265                 break;
 266         }
 267 }
 268
 269 static void
 270 emit_store(
 271         struct lp_build_tgsi_context * bld_base,
 272         const struct tgsi_full_instruction * inst,
 273         const struct tgsi_opcode_info * info,
 274         LLVMValueRef dst[4])
 275 {
 276         struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
 277         struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
 278         struct lp_build_context base = bld->bld_base.base;
 279         const struct tgsi_full_dst_register *reg = &inst->Dst[0];
 280         LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 281         LLVMValueRef temp_ptr;
 282         unsigned chan, chan_index;
 283         boolean is_vec_store = FALSE;
 284         if (dst[0]) {
 285                 LLVMTypeKind k = LLVMGetTypeKind(LLVMTypeOf(dst[0]));
 286                 is_vec_store = (k == LLVMVectorTypeKind);
 287         }
 288
 289         if (is_vec_store) {
 290                 LLVMValueRef values[4] = {};
 291                 TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan) {
 292                         LLVMValueRef index = lp_build_const_int32(gallivm, chan);
 293                         values[chan]  = LLVMBuildExtractElement(gallivm->builder,
 294                                                         dst[0], index, "");
 295                 }
 296                 bld_base->emit_store(bld_base, inst, info, values);
 297                 return;
 298         }
 299
 300         TGSI_FOR_EACH_DST0_ENABLED_CHANNEL( inst, chan_index ) {
 301                 LLVMValueRef value = dst[chan_index];
 302
 303                 if (inst->Instruction.Saturate != TGSI_SAT_NONE) {
 304                         struct lp_build_emit_data clamp_emit_data;
 305
 306                         memset(&clamp_emit_data, 0, sizeof(clamp_emit_data));
 307                         clamp_emit_data.arg_count = 3;
 308                         clamp_emit_data.args[0] = value;
 309                         clamp_emit_data.args[2] = base.one;
 310
 311                         switch(inst->Instruction.Saturate) {
 312                         case TGSI_SAT_ZERO_ONE:
 313                                 clamp_emit_data.args[1] = base.zero;
 314                                 break;
 315                         case TGSI_SAT_MINUS_PLUS_ONE:
 316                                 clamp_emit_data.args[1] = LLVMConstReal(
 317                                                 base.elem_type, -1.0f);
 318                                 break;
 319                         default:
 320                                 assert(0);
 321                         }
 322                         value = lp_build_emit_llvm(bld_base, TGSI_OPCODE_CLAMP,
 323                                                 &clamp_emit_data);
 324                 }
 325
 326                 switch(reg->Register.File) {
 327                 case TGSI_FILE_OUTPUT:
 328                         temp_ptr = bld->outputs[reg->Register.Index][chan_index];
 329                         break;
 330
 331                 case TGSI_FILE_TEMPORARY:
 332                         temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, chan_index);
 333                         break;
 334
 335                 default:
 336                         return;
 337                 }
 338
 339                 value = bitcast(bld_base, TGSI_TYPE_FLOAT, value);
 340
 341                 LLVMBuildStore(builder, value, temp_ptr);
 342         }
 343 }
 344
 345 static void bgnloop_emit(
 346         const struct lp_build_tgsi_action * action,
 347         struct lp_build_tgsi_context * bld_base,
 348         struct lp_build_emit_data * emit_data)
 349 {
 350         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 351         struct gallivm_state * gallivm = bld_base->base.gallivm;
 352         LLVMBasicBlockRef loop_block;
 353         LLVMBasicBlockRef endloop_block;
 354         endloop_block = LLVMAppendBasicBlockInContext(gallivm->context,
 355                                                 ctx->main_fn, "ENDLOOP");
 356         loop_block = LLVMInsertBasicBlockInContext(gallivm->context,
 357                                                 endloop_block, "LOOP");
 358         LLVMBuildBr(gallivm->builder, loop_block);
 359         LLVMPositionBuilderAtEnd(gallivm->builder, loop_block);
 360         ctx->loop_depth++;
 361         ctx->loop[ctx->loop_depth - 1].loop_block = loop_block;
 362         ctx->loop[ctx->loop_depth - 1].endloop_block = endloop_block;
 363 }
 364
 365 static void brk_emit(
 366         const struct lp_build_tgsi_action * action,
 367         struct lp_build_tgsi_context * bld_base,
 368         struct lp_build_emit_data * emit_data)
 369 {
 370         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 371         struct gallivm_state * gallivm = bld_base->base.gallivm;
 372         struct radeon_llvm_loop * current_loop = get_current_loop(ctx);
 373
 374         LLVMBuildBr(gallivm->builder, current_loop->endloop_block);
 375 }
 376
 377 static void cont_emit(
 378         const struct lp_build_tgsi_action * action,
 379         struct lp_build_tgsi_context * bld_base,
 380         struct lp_build_emit_data * emit_data)
 381 {
 382         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 383         struct gallivm_state * gallivm = bld_base->base.gallivm;
 384         struct radeon_llvm_loop * current_loop = get_current_loop(ctx);
 385
 386         LLVMBuildBr(gallivm->builder, current_loop->loop_block);
 387 }
 388
 389 static void else_emit(
 390         const struct lp_build_tgsi_action * action,
 391         struct lp_build_tgsi_context * bld_base,
 392         struct lp_build_emit_data * emit_data)
 393 {
 394         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 395         struct gallivm_state * gallivm = bld_base->base.gallivm;
 396         struct radeon_llvm_branch * current_branch = get_current_branch(ctx);
 397         LLVMBasicBlockRef current_block = LLVMGetInsertBlock(gallivm->builder);
 398
 399         /* We need to add a terminator to the current block if the previous
 400          * instruction was an ENDIF.Example:
 401          * IF
 402          *   [code]
 403          *   IF
 404          *     [code]
 405          *   ELSE
 406          *    [code]
 407          *   ENDIF <--
 408          * ELSE<--
 409          *   [code]
 410          * ENDIF
 411          */
 412
 413         if (current_block != current_branch->if_block) {
 414                 LLVMBuildBr(gallivm->builder, current_branch->endif_block);
 415         }
 416         if (!LLVMGetBasicBlockTerminator(current_branch->if_block)) {
 417                 LLVMBuildBr(gallivm->builder, current_branch->endif_block);
 418         }
 419         current_branch->has_else = 1;
 420         LLVMPositionBuilderAtEnd(gallivm->builder, current_branch->else_block);
 421 }
 422
 423 static void endif_emit(
 424         const struct lp_build_tgsi_action * action,
 425         struct lp_build_tgsi_context * bld_base,
 426         struct lp_build_emit_data * emit_data)
 427 {
 428         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 429         struct gallivm_state * gallivm = bld_base->base.gallivm;
 430         struct radeon_llvm_branch * current_branch = get_current_branch(ctx);
 431         LLVMBasicBlockRef current_block = LLVMGetInsertBlock(gallivm->builder);
 432
 433         /* If we have consecutive ENDIF instructions, then the first ENDIF
 434          * will not have a terminator, so we need to add one. */
 435         if (current_block != current_branch->if_block
 436                         && current_block != current_branch->else_block
 437                         && !LLVMGetBasicBlockTerminator(current_block)) {
 438
 439                  LLVMBuildBr(gallivm->builder, current_branch->endif_block);
 440         }
 441         if (!LLVMGetBasicBlockTerminator(current_branch->else_block)) {
 442                 LLVMPositionBuilderAtEnd(gallivm->builder, current_branch->else_block);
 443                 LLVMBuildBr(gallivm->builder, current_branch->endif_block);
 444         }
 445
 446         if (!LLVMGetBasicBlockTerminator(current_branch->if_block)) {
 447                 LLVMPositionBuilderAtEnd(gallivm->builder, current_branch->if_block);
 448                 LLVMBuildBr(gallivm->builder, current_branch->endif_block);
 449         }
 450
 451         LLVMPositionBuilderAtEnd(gallivm->builder, current_branch->endif_block);
 452         ctx->branch_depth--;
 453 }
 454
 455 static void endloop_emit(
 456         const struct lp_build_tgsi_action * action,
 457         struct lp_build_tgsi_context * bld_base,
 458         struct lp_build_emit_data * emit_data)
 459 {
 460         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 461         struct gallivm_state * gallivm = bld_base->base.gallivm;
 462         struct radeon_llvm_loop * current_loop = get_current_loop(ctx);
 463
 464         if (!LLVMGetBasicBlockTerminator(LLVMGetInsertBlock(gallivm->builder))) {
 465                  LLVMBuildBr(gallivm->builder, current_loop->loop_block);
 466         }
 467
 468         LLVMPositionBuilderAtEnd(gallivm->builder, current_loop->endloop_block);
 469         ctx->loop_depth--;
 470 }
 471
 472 static void if_emit(
 473         const struct lp_build_tgsi_action * action,
 474         struct lp_build_tgsi_context * bld_base,
 475         struct lp_build_emit_data * emit_data)
 476 {
 477         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 478         struct gallivm_state * gallivm = bld_base->base.gallivm;
 479         LLVMValueRef cond;
 480         LLVMBasicBlockRef if_block, else_block, endif_block;
 481
 482         cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
 483                 bitcast(bld_base, TGSI_TYPE_UNSIGNED, emit_data->args[0]),
 484                         bld_base->int_bld.zero, "");
 485
 486         endif_block = LLVMAppendBasicBlockInContext(gallivm->context,
 487                                                 ctx->main_fn, "ENDIF");
 488         if_block = LLVMInsertBasicBlockInContext(gallivm->context,
 489                                                 endif_block, "IF");
 490         else_block = LLVMInsertBasicBlockInContext(gallivm->context,
 491                                                 endif_block, "ELSE");
 492         LLVMBuildCondBr(gallivm->builder, cond, if_block, else_block);
 493         LLVMPositionBuilderAtEnd(gallivm->builder, if_block);
 494
 495         ctx->branch_depth++;
 496         ctx->branch[ctx->branch_depth - 1].endif_block = endif_block;
 497         ctx->branch[ctx->branch_depth - 1].if_block = if_block;
 498         ctx->branch[ctx->branch_depth - 1].else_block = else_block;
 499         ctx->branch[ctx->branch_depth - 1].has_else = 0;
 500 }
 501
 502 static void kil_emit(
 503         const struct lp_build_tgsi_action * action,
 504         struct lp_build_tgsi_context * bld_base,
 505         struct lp_build_emit_data * emit_data)
 506 {
 507         unsigned i;
 508         for (i = 0; i < emit_data->arg_count; i++) {
 509                 emit_data->output[i] = lp_build_intrinsic_unary(
 510                         bld_base->base.gallivm->builder,
 511                         action->intr_name,
 512                         emit_data->dst_type, emit_data->args[i]);
 513         }
 514 }
 515
 516 /* coord_arg - index of the source coord vector in the emit_data->args array */
 517 void radeon_llvm_emit_prepare_cube_coords(
 518                 struct lp_build_tgsi_context * bld_base,
 519                 struct lp_build_emit_data * emit_data,
 520                 unsigned coord_arg)
 521 {
 522
 523         unsigned target = emit_data->inst->Texture.Texture;
 524         unsigned opcode = emit_data->inst->Instruction.Opcode;
 525         struct gallivm_state * gallivm = bld_base->base.gallivm;
 526         LLVMBuilderRef builder = gallivm->builder;
 527         LLVMTypeRef type = bld_base->base.elem_type;
 528         LLVMValueRef coords[4];
 529         LLVMValueRef mad_args[3];
 530         LLVMValueRef idx;
 531         unsigned i;
 532
 533         LLVMValueRef v = build_intrinsic(builder, "llvm.AMDGPU.cube",
 534                         LLVMVectorType(type, 4),
 535                         &emit_data->args[coord_arg], 1, LLVMReadNoneAttribute);
 536
 537         for (i = 0; i < 4; ++i) {
 538                 idx = lp_build_const_int32(gallivm, i);
 539                 coords[i] = LLVMBuildExtractElement(builder, v, idx, "");
 540         }
 541
 542         coords[2] = build_intrinsic(builder, "fabs",
 543                         type, &coords[2], 1, LLVMReadNoneAttribute);
 544         coords[2] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_RCP, coords[2]);
 545
 546         mad_args[1] = coords[2];
 547         mad_args[2] = LLVMConstReal(type, 1.5);
 548
 549         mad_args[0] = coords[0];
 550         coords[0] = lp_build_emit_llvm_ternary(bld_base, TGSI_OPCODE_MAD,
 551                         mad_args[0], mad_args[1], mad_args[2]);
 552
 553         mad_args[0] = coords[1];
 554         coords[1] = lp_build_emit_llvm_ternary(bld_base, TGSI_OPCODE_MAD,
 555                         mad_args[0], mad_args[1], mad_args[2]);
 556
 557         /* apply xyz = yxw swizzle to cooords */
 558         coords[2] = coords[3];
 559         coords[3] = coords[1];
 560         coords[1] = coords[0];
 561         coords[0] = coords[3];
 562
 563         /* all cases except simple cube map sampling require special handling
 564          * for coord vector */
 565         if (target != TGSI_TEXTURE_CUBE ||
 566                 opcode != TGSI_OPCODE_TEX) {
 567
 568                 /* load source coord.w component - array_index for cube arrays or
 569                  * compare value for SHADOWCUBE */
 570                 idx = lp_build_const_int32(gallivm, 3);
 571                 coords[3] = LLVMBuildExtractElement(builder,
 572                                 emit_data->args[coord_arg], idx, "");
 573
 574                 /* for cube arrays coord.z = coord.w(array_index) * 8 + face */
 575                 if (target == TGSI_TEXTURE_CUBE_ARRAY ||
 576                         target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
 577
 578                         coords[2] = lp_build_emit_llvm_ternary(bld_base, TGSI_OPCODE_MAD,
 579                                         coords[3], lp_build_const_float(gallivm, 8.0), coords[2]);
 580                 }
 581
 582                 /* for instructions that need additional src (compare/lod/bias),
 583                  * put it in coord.w */
 584                 if (opcode == TGSI_OPCODE_TEX2 ||
 585                         opcode == TGSI_OPCODE_TXB2 ||
 586                         opcode == TGSI_OPCODE_TXL2) {
 587                         coords[3] = emit_data->args[coord_arg + 1];
 588                 }
 589         }
 590
 591         emit_data->args[coord_arg] =
 592                         lp_build_gather_values(bld_base->base.gallivm, coords, 4);
 593 }
 594
 595 static void txd_fetch_args(
 596         struct lp_build_tgsi_context * bld_base,
 597         struct lp_build_emit_data * emit_data)
 598 {
 599         const struct tgsi_full_instruction * inst = emit_data->inst;
 600
 601         LLVMValueRef coords[4];
 602         unsigned chan, src;
 603         for (src = 0; src < 3; src++) {
 604                 for (chan = 0; chan < 4; chan++)
 605                         coords[chan] = lp_build_emit_fetch(bld_base, inst, src, chan);
 606
 607                 emit_data->args[src] = lp_build_gather_values(bld_base->base.gallivm,
 608                                 coords, 4);
 609         }
 610         emit_data->arg_count = 3;
 611         emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
 612 }
 613
 614
 615 static void txp_fetch_args(
 616         struct lp_build_tgsi_context * bld_base,
 617         struct lp_build_emit_data * emit_data)
 618 {
 619         const struct tgsi_full_instruction * inst = emit_data->inst;
 620         LLVMValueRef src_w;
 621         unsigned chan;
 622         LLVMValueRef coords[4];
 623
 624         emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
 625         src_w = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
 626
 627         for (chan = 0; chan < 3; chan++ ) {
 628                 LLVMValueRef arg = lp_build_emit_fetch(bld_base,
 629                                                 emit_data->inst, 0, chan);
 630                 coords[chan] = lp_build_emit_llvm_binary(bld_base,
 631                                         TGSI_OPCODE_DIV, arg, src_w);
 632         }
 633         coords[3] = bld_base->base.one;
 634         emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
 635                                                 coords, 4);
 636         emit_data->arg_count = 1;
 637
 638         if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE ||
 639              inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY ||
 640              inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
 641              inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) &&
 642             inst->Instruction.Opcode != TGSI_OPCODE_TXQ &&
 643             inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) {
 644                 radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, 0);
 645         }
 646 }
 647
 648 static void tex_fetch_args(
 649         struct lp_build_tgsi_context * bld_base,
 650         struct lp_build_emit_data * emit_data)
 651 {
 652         /* XXX: lp_build_swizzle_aos() was failing with wrong arg types,
 653          * when we used CHAN_ALL.  We should be able to get this to work,
 654          * but for now we will swizzle it ourselves
 655         emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
 656                                                  0, CHAN_ALL);
 657
 658         */
 659
 660         const struct tgsi_full_instruction * inst = emit_data->inst;
 661
 662         LLVMValueRef coords[4];
 663         unsigned chan;
 664         for (chan = 0; chan < 4; chan++) {
 665                 coords[chan] = lp_build_emit_fetch(bld_base, inst, 0, chan);
 666         }
 667
 668         emit_data->arg_count = 1;
 669         emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
 670                                                 coords, 4);
 671         emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
 672
 673         if (inst->Instruction.Opcode == TGSI_OPCODE_TEX2 ||
 674                 inst->Instruction.Opcode == TGSI_OPCODE_TXB2 ||
 675                 inst->Instruction.Opcode == TGSI_OPCODE_TXL2) {
 676                 /* These instructions have additional operand that should be packed
 677                  * into the cube coord vector by radeon_llvm_emit_prepare_cube_coords.
 678                  * That operand should be passed as a float value in the args array
 679                  * right after the coord vector. After packing it's not used anymore,
 680                  * that's why arg_count is not increased */
 681                 emit_data->args[1] = lp_build_emit_fetch(bld_base, inst, 1, 0);
 682         }
 683
 684         if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE ||
 685              inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY ||
 686              inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
 687              inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) &&
 688             inst->Instruction.Opcode != TGSI_OPCODE_TXQ &&
 689             inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) {
 690                 radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, 0);
 691         }
 692 }
 693
 694 static void txf_fetch_args(
 695         struct lp_build_tgsi_context * bld_base,
 696         struct lp_build_emit_data * emit_data)
 697 {
 698         const struct tgsi_full_instruction * inst = emit_data->inst;
 699         struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
 700         const struct tgsi_texture_offset * off = inst->TexOffsets;
 701         LLVMTypeRef offset_type = bld_base->int_bld.elem_type;
 702
 703         /* fetch tex coords */
 704         tex_fetch_args(bld_base, emit_data);
 705
 706         /* fetch tex offsets */
 707         if (inst->Texture.NumOffsets) {
 708                 assert(inst->Texture.NumOffsets == 1);
 709
 710                 emit_data->args[1] = LLVMConstBitCast(
 711                         bld->immediates[off->Index][off->SwizzleX],
 712                         offset_type);
 713                 emit_data->args[2] = LLVMConstBitCast(
 714                         bld->immediates[off->Index][off->SwizzleY],
 715                         offset_type);
 716                 emit_data->args[3] = LLVMConstBitCast(
 717                         bld->immediates[off->Index][off->SwizzleZ],
 718                         offset_type);
 719         } else {
 720                 emit_data->args[1] = bld_base->int_bld.zero;
 721                 emit_data->args[2] = bld_base->int_bld.zero;
 722                 emit_data->args[3] = bld_base->int_bld.zero;
 723         }
 724
 725         emit_data->arg_count = 4;
 726 }
 727
 728 static void emit_icmp(
 729                 const struct lp_build_tgsi_action * action,
 730                 struct lp_build_tgsi_context * bld_base,
 731                 struct lp_build_emit_data * emit_data)
 732 {
 733         unsigned pred;
 734         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 735         LLVMContextRef context = bld_base->base.gallivm->context;
 736
 737         switch (emit_data->inst->Instruction.Opcode) {
 738         case TGSI_OPCODE_USEQ: pred = LLVMIntEQ; break;
 739         case TGSI_OPCODE_USNE: pred = LLVMIntNE; break;
 740         case TGSI_OPCODE_USGE: pred = LLVMIntUGE; break;
 741         case TGSI_OPCODE_USLT: pred = LLVMIntULT; break;
 742         case TGSI_OPCODE_ISGE: pred = LLVMIntSGE; break;
 743         case TGSI_OPCODE_ISLT: pred = LLVMIntSLT; break;
 744         default:
 745                 assert(!"unknown instruction");
 746         }
 747
 748         LLVMValueRef v = LLVMBuildICmp(builder, pred,
 749                         emit_data->args[0], emit_data->args[1],"");
 750
 751         v = LLVMBuildSExtOrBitCast(builder, v,
 752                         LLVMInt32TypeInContext(context), "");
 753
 754         emit_data->output[emit_data->chan] = v;
 755 }
 756
 757 static void emit_cmp(
 758                 const struct lp_build_tgsi_action *action,
 759                 struct lp_build_tgsi_context * bld_base,
 760                 struct lp_build_emit_data * emit_data)
 761 {
 762         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 763         LLVMRealPredicate pred;
 764         LLVMValueRef cond;
 765
 766         /* XXX I'm not sure whether to do unordered or ordered comparisons,
 767          * but llvmpipe uses unordered comparisons, so for consistency we use
 768          * unordered.  (The authors of llvmpipe aren't sure about using
 769          * unordered vs ordered comparisons either.
 770          */
 771         switch (emit_data->inst->Instruction.Opcode) {
 772         case TGSI_OPCODE_SGE: pred = LLVMRealUGE; break;
 773         case TGSI_OPCODE_SEQ: pred = LLVMRealUEQ; break;
 774         case TGSI_OPCODE_SLE: pred = LLVMRealULE; break;
 775         case TGSI_OPCODE_SLT: pred = LLVMRealULT; break;
 776         case TGSI_OPCODE_SNE: pred = LLVMRealUNE; break;
 777         case TGSI_OPCODE_SGT: pred = LLVMRealUGT; break;
 778         default: assert(!"unknown instruction");
 779         }
 780
 781         cond = LLVMBuildFCmp(builder,
 782                 pred, emit_data->args[0], emit_data->args[1], "");
 783
 784         emit_data->output[emit_data->chan] = LLVMBuildSelect(builder,
 785                 cond, bld_base->base.one, bld_base->base.zero, "");
 786 }
 787
 788 static void emit_not(
 789                 const struct lp_build_tgsi_action * action,
 790                 struct lp_build_tgsi_context * bld_base,
 791                 struct lp_build_emit_data * emit_data)
 792 {
 793         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 794         LLVMValueRef v = bitcast(bld_base, TGSI_TYPE_UNSIGNED,
 795                         emit_data->args[0]);
 796         emit_data->output[emit_data->chan] = LLVMBuildNot(builder, v, "");
 797 }
 798
 799 static void emit_and(
 800                 const struct lp_build_tgsi_action * action,
 801                 struct lp_build_tgsi_context * bld_base,
 802                 struct lp_build_emit_data * emit_data)
 803 {
 804         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 805         emit_data->output[emit_data->chan] = LLVMBuildAnd(builder,
 806                         emit_data->args[0], emit_data->args[1], "");
 807 }
 808
 809 static void emit_or(
 810                 const struct lp_build_tgsi_action * action,
 811                 struct lp_build_tgsi_context * bld_base,
 812                 struct lp_build_emit_data * emit_data)
 813 {
 814         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 815         emit_data->output[emit_data->chan] = LLVMBuildOr(builder,
 816                         emit_data->args[0], emit_data->args[1], "");
 817 }
 818
 819 static void emit_uadd(
 820                 const struct lp_build_tgsi_action * action,
 821                 struct lp_build_tgsi_context * bld_base,
 822                 struct lp_build_emit_data * emit_data)
 823 {
 824         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 825         emit_data->output[emit_data->chan] = LLVMBuildAdd(builder,
 826                         emit_data->args[0], emit_data->args[1], "");
 827 }
 828
 829 static void emit_udiv(
 830                 const struct lp_build_tgsi_action * action,
 831                 struct lp_build_tgsi_context * bld_base,
 832                 struct lp_build_emit_data * emit_data)
 833 {
 834         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 835         emit_data->output[emit_data->chan] = LLVMBuildUDiv(builder,
 836                         emit_data->args[0], emit_data->args[1], "");
 837 }
 838
 839 static void emit_idiv(
 840                 const struct lp_build_tgsi_action * action,
 841                 struct lp_build_tgsi_context * bld_base,
 842                 struct lp_build_emit_data * emit_data)
 843 {
 844         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 845         emit_data->output[emit_data->chan] = LLVMBuildSDiv(builder,
 846                         emit_data->args[0], emit_data->args[1], "");
 847 }
 848
 849 static void emit_mod(
 850                 const struct lp_build_tgsi_action * action,
 851                 struct lp_build_tgsi_context * bld_base,
 852                 struct lp_build_emit_data * emit_data)
 853 {
 854         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 855         emit_data->output[emit_data->chan] = LLVMBuildSRem(builder,
 856                         emit_data->args[0], emit_data->args[1], "");
 857 }
 858
 859 static void emit_umod(
 860                 const struct lp_build_tgsi_action * action,
 861                 struct lp_build_tgsi_context * bld_base,
 862                 struct lp_build_emit_data * emit_data)
 863 {
 864         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 865         emit_data->output[emit_data->chan] = LLVMBuildURem(builder,
 866                         emit_data->args[0], emit_data->args[1], "");
 867 }
 868
 869 static void emit_shl(
 870                 const struct lp_build_tgsi_action * action,
 871                 struct lp_build_tgsi_context * bld_base,
 872                 struct lp_build_emit_data * emit_data)
 873 {
 874         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 875         emit_data->output[emit_data->chan] = LLVMBuildShl(builder,
 876                         emit_data->args[0], emit_data->args[1], "");
 877 }
 878
 879 static void emit_ushr(
 880                 const struct lp_build_tgsi_action * action,
 881                 struct lp_build_tgsi_context * bld_base,
 882                 struct lp_build_emit_data * emit_data)
 883 {
 884         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 885         emit_data->output[emit_data->chan] = LLVMBuildLShr(builder,
 886                         emit_data->args[0], emit_data->args[1], "");
 887 }
 888 static void emit_ishr(
 889                 const struct lp_build_tgsi_action * action,
 890                 struct lp_build_tgsi_context * bld_base,
 891                 struct lp_build_emit_data * emit_data)
 892 {
 893         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 894         emit_data->output[emit_data->chan] = LLVMBuildAShr(builder,
 895                         emit_data->args[0], emit_data->args[1], "");
 896 }
 897
 898 static void emit_xor(
 899                 const struct lp_build_tgsi_action * action,
 900                 struct lp_build_tgsi_context * bld_base,
 901                 struct lp_build_emit_data * emit_data)
 902 {
 903         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 904         emit_data->output[emit_data->chan] = LLVMBuildXor(builder,
 905                         emit_data->args[0], emit_data->args[1], "");
 906 }
 907
 908 static void emit_ssg(
 909                 const struct lp_build_tgsi_action * action,
 910                 struct lp_build_tgsi_context * bld_base,
 911                 struct lp_build_emit_data * emit_data)
 912 {
 913         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 914
 915         LLVMValueRef cmp, val;
 916
 917         if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_ISSG) {
 918                 cmp = LLVMBuildICmp(builder, LLVMIntSGT, emit_data->args[0], bld_base->int_bld.zero, "");
 919                 val = LLVMBuildSelect(builder, cmp, bld_base->int_bld.one, emit_data->args[0], "");
 920                 cmp = LLVMBuildICmp(builder, LLVMIntSGE, val, bld_base->int_bld.zero, "");
 921                 val = LLVMBuildSelect(builder, cmp, val, LLVMConstInt(bld_base->int_bld.elem_type, -1, true), "");
 922         } else { // float SSG
 923                 cmp = LLVMBuildFCmp(builder, LLVMRealUGT, emit_data->args[0], bld_base->base.zero, "");
 924                 val = LLVMBuildSelect(builder, cmp, bld_base->base.one, emit_data->args[0], "");
 925                 cmp = LLVMBuildFCmp(builder, LLVMRealUGE, val, bld_base->base.zero, "");
 926                 val = LLVMBuildSelect(builder, cmp, val, LLVMConstReal(bld_base->base.elem_type, -1), "");
 927         }
 928
 929         emit_data->output[emit_data->chan] = val;
 930 }
 931
 932 static void emit_ineg(
 933                 const struct lp_build_tgsi_action * action,
 934                 struct lp_build_tgsi_context * bld_base,
 935                 struct lp_build_emit_data * emit_data)
 936 {
 937         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 938         emit_data->output[emit_data->chan] = LLVMBuildNeg(builder,
 939                         emit_data->args[0], "");
 940 }
 941
 942 static void emit_f2i(
 943                 const struct lp_build_tgsi_action * action,
 944                 struct lp_build_tgsi_context * bld_base,
 945                 struct lp_build_emit_data * emit_data)
 946 {
 947         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 948         emit_data->output[emit_data->chan] = LLVMBuildFPToSI(builder,
 949                         emit_data->args[0], bld_base->int_bld.elem_type, "");
 950 }
 951
 952 static void emit_f2u(
 953                 const struct lp_build_tgsi_action * action,
 954                 struct lp_build_tgsi_context * bld_base,
 955                 struct lp_build_emit_data * emit_data)
 956 {
 957         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 958         emit_data->output[emit_data->chan] = LLVMBuildFPToUI(builder,
 959                         emit_data->args[0], bld_base->uint_bld.elem_type, "");
 960 }
 961
 962 static void emit_i2f(
 963                 const struct lp_build_tgsi_action * action,
 964                 struct lp_build_tgsi_context * bld_base,
 965                 struct lp_build_emit_data * emit_data)
 966 {
 967         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 968         emit_data->output[emit_data->chan] = LLVMBuildSIToFP(builder,
 969                         emit_data->args[0], bld_base->base.elem_type, "");
 970 }
 971
 972 static void emit_u2f(
 973                 const struct lp_build_tgsi_action * action,
 974                 struct lp_build_tgsi_context * bld_base,
 975                 struct lp_build_emit_data * emit_data)
 976 {
 977         LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 978         emit_data->output[emit_data->chan] = LLVMBuildUIToFP(builder,
 979                         emit_data->args[0], bld_base->base.elem_type, "");
 980 }
 981
 982 static void emit_immediate(struct lp_build_tgsi_context * bld_base,
 983                 const struct tgsi_full_immediate *imm)
 984 {
 985         unsigned i;
 986         struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
 987
 988         for (i = 0; i < 4; ++i) {
 989                 ctx->soa.immediates[ctx->soa.num_immediates][i] =
 990                                 LLVMConstInt(bld_base->uint_bld.elem_type, imm->u[i].Uint, false   );
 991         }
 992
 993         ctx->soa.num_immediates++;
 994 }
 995
 996 LLVMValueRef
 997 build_intrinsic(LLVMBuilderRef builder,
 998                    const char *name,
 999                    LLVMTypeRef ret_type,
1000                    LLVMValueRef *args,
1001                    unsigned num_args,
1002                    LLVMAttribute attr)
1003 {
1004    LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
1005    LLVMValueRef function;
1006
1007    function = LLVMGetNamedFunction(module, name);
1008    if(!function) {
1009       LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
1010       unsigned i;
1011
1012       assert(num_args <= LP_MAX_FUNC_ARGS);
1013
1014       for(i = 0; i < num_args; ++i) {
1015          assert(args[i]);
1016          arg_types[i] = LLVMTypeOf(args[i]);
1017       }
1018
1019       function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
1020
1021       if (attr)
1022           LLVMAddFunctionAttr(function, attr);
1023    }
1024
1025    return LLVMBuildCall(builder, function, args, num_args, "");
1026 }
1027
1028 static void build_tgsi_intrinsic(
1029  const struct lp_build_tgsi_action * action,
1030  struct lp_build_tgsi_context * bld_base,
1031  struct lp_build_emit_data * emit_data,
1032  LLVMAttribute attr)
1033 {
1034    struct lp_build_context * base = &bld_base->base;
1035    emit_data->output[emit_data->chan] = build_intrinsic(
1036                base->gallivm->builder, action->intr_name,
1037                emit_data->dst_type, emit_data->args,
1038                emit_data->arg_count, attr);
1039 }
1040 void
1041 build_tgsi_intrinsic_nomem(
1042  const struct lp_build_tgsi_action * action,
1043  struct lp_build_tgsi_context * bld_base,
1044  struct lp_build_emit_data * emit_data)
1045 {
1046         build_tgsi_intrinsic(action, bld_base, emit_data, LLVMReadNoneAttribute);
1047 }
1048
1049 static void build_tgsi_intrinsic_readonly(
1050  const struct lp_build_tgsi_action * action,
1051  struct lp_build_tgsi_context * bld_base,
1052  struct lp_build_emit_data * emit_data)
1053 {
1054         build_tgsi_intrinsic(action, bld_base, emit_data, LLVMReadOnlyAttribute);
1055 }
1056
1057 void radeon_llvm_context_init(struct radeon_llvm_context * ctx)
1058 {
1059         struct lp_type type;
1060         LLVMTypeRef main_fn_type;
1061         LLVMBasicBlockRef main_fn_body;
1062
1063         /* Initialize the gallivm object:
1064          * We are only using the module, context, and builder fields of this struct.
1065          * This should be enough for us to be able to pass our gallivm struct to the
1066          * helper functions in the gallivm module.
1067          */
1068         memset(&ctx->gallivm, 0, sizeof (ctx->gallivm));
1069         memset(&ctx->soa, 0, sizeof(ctx->soa));
1070         ctx->gallivm.context = LLVMContextCreate();
1071         ctx->gallivm.module = LLVMModuleCreateWithNameInContext("tgsi",
1072                                                 ctx->gallivm.context);
1073         ctx->gallivm.builder = LLVMCreateBuilderInContext(ctx->gallivm.context);
1074
1075         /* Setup the module */
1076         main_fn_type = LLVMFunctionType(LLVMVoidTypeInContext(ctx->gallivm.context),
1077                                          NULL, 0, 0);
1078         ctx->main_fn = LLVMAddFunction(ctx->gallivm.module, "main", main_fn_type);
1079         main_fn_body = LLVMAppendBasicBlockInContext(ctx->gallivm.context,
1080                         ctx->main_fn, "main_body");
1081          LLVMPositionBuilderAtEnd(ctx->gallivm.builder, main_fn_body);
1082
1083         ctx->store_output_intr = "llvm.AMDGPU.store.output.";
1084         ctx->swizzle_intr = "llvm.AMDGPU.swizzle";
1085         struct lp_build_tgsi_context * bld_base = &ctx->soa.bld_base;
1086
1087         /* XXX: We need to revisit this.I think the correct way to do this is
1088          * to use length = 4 here and use the elem_bld for everything. */
1089         type.floating = TRUE;
1090         type.sign = TRUE;
1091         type.width = 32;
1092         type.length = 1;
1093
1094         lp_build_context_init(&bld_base->base, &ctx->gallivm, type);
1095         lp_build_context_init(&ctx->soa.bld_base.uint_bld, &ctx->gallivm, lp_uint_type(type));
1096         lp_build_context_init(&ctx->soa.bld_base.int_bld, &ctx->gallivm, lp_int_type(type));
1097
1098         bld_base->soa = 1;
1099         bld_base->emit_store = emit_store;
1100         bld_base->emit_swizzle = emit_swizzle;
1101         bld_base->emit_declaration = emit_declaration;
1102         bld_base->emit_immediate = emit_immediate;
1103
1104         bld_base->emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = emit_fetch_immediate;
1105         bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_input;
1106         bld_base->emit_fetch_funcs[TGSI_FILE_TEMPORARY] = emit_fetch_temporary;
1107         bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = emit_fetch_output;
1108
1109         /* Allocate outputs */
1110         ctx->soa.outputs = ctx->outputs;
1111
1112         /* XXX: Is there a better way to initialize all this ? */
1113
1114         lp_set_default_actions(bld_base);
1115
1116         bld_base->op_actions[TGSI_OPCODE_ABS].emit = build_tgsi_intrinsic_readonly;
1117         bld_base->op_actions[TGSI_OPCODE_ABS].intr_name = "fabs";
1118         bld_base->op_actions[TGSI_OPCODE_ARL].emit = build_tgsi_intrinsic_nomem;
1119         bld_base->op_actions[TGSI_OPCODE_ARL].intr_name = "llvm.AMDGPU.arl";
1120         bld_base->op_actions[TGSI_OPCODE_AND].emit = emit_and;
1121         bld_base->op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit;
1122         bld_base->op_actions[TGSI_OPCODE_BRK].emit = brk_emit;
1123         bld_base->op_actions[TGSI_OPCODE_CEIL].emit = build_tgsi_intrinsic_readonly;
1124         bld_base->op_actions[TGSI_OPCODE_CEIL].intr_name = "ceil";
1125         bld_base->op_actions[TGSI_OPCODE_CLAMP].emit = build_tgsi_intrinsic_nomem;
1126         bld_base->op_actions[TGSI_OPCODE_CLAMP].intr_name = "llvm.AMDIL.clamp.";
1127         bld_base->op_actions[TGSI_OPCODE_CMP].emit = build_tgsi_intrinsic_nomem;
1128         bld_base->op_actions[TGSI_OPCODE_CMP].intr_name = "llvm.AMDGPU.cndlt";
1129         bld_base->op_actions[TGSI_OPCODE_CONT].emit = cont_emit;
1130         bld_base->op_actions[TGSI_OPCODE_COS].emit = build_tgsi_intrinsic_readonly;
1131         bld_base->op_actions[TGSI_OPCODE_COS].intr_name = "llvm.cos.f32";
1132         bld_base->op_actions[TGSI_OPCODE_DDX].intr_name = "llvm.AMDGPU.ddx";
1133         bld_base->op_actions[TGSI_OPCODE_DDX].fetch_args = tex_fetch_args;
1134         bld_base->op_actions[TGSI_OPCODE_DDY].intr_name = "llvm.AMDGPU.ddy";
1135         bld_base->op_actions[TGSI_OPCODE_DDY].fetch_args = tex_fetch_args;
1136         bld_base->op_actions[TGSI_OPCODE_ELSE].emit = else_emit;
1137         bld_base->op_actions[TGSI_OPCODE_ENDIF].emit = endif_emit;
1138         bld_base->op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit;
1139         bld_base->op_actions[TGSI_OPCODE_EX2].emit = build_tgsi_intrinsic_nomem;
1140         bld_base->op_actions[TGSI_OPCODE_EX2].intr_name = "llvm.AMDIL.exp.";
1141         bld_base->op_actions[TGSI_OPCODE_FLR].emit = build_tgsi_intrinsic_readonly;
1142         bld_base->op_actions[TGSI_OPCODE_FLR].intr_name = "floor";
1143         bld_base->op_actions[TGSI_OPCODE_FRC].emit = build_tgsi_intrinsic_nomem;
1144         bld_base->op_actions[TGSI_OPCODE_FRC].intr_name = "llvm.AMDIL.fraction.";
1145         bld_base->op_actions[TGSI_OPCODE_F2I].emit = emit_f2i;
1146         bld_base->op_actions[TGSI_OPCODE_F2U].emit = emit_f2u;
1147         bld_base->op_actions[TGSI_OPCODE_IABS].emit = build_tgsi_intrinsic_nomem;
1148         bld_base->op_actions[TGSI_OPCODE_IABS].intr_name = "llvm.AMDIL.abs.";
1149         bld_base->op_actions[TGSI_OPCODE_IDIV].emit = emit_idiv;
1150         bld_base->op_actions[TGSI_OPCODE_IF].emit = if_emit;
1151         bld_base->op_actions[TGSI_OPCODE_IMAX].emit = build_tgsi_intrinsic_nomem;
1152         bld_base->op_actions[TGSI_OPCODE_IMAX].intr_name = "llvm.AMDGPU.imax";
1153         bld_base->op_actions[TGSI_OPCODE_IMIN].emit = build_tgsi_intrinsic_nomem;
1154         bld_base->op_actions[TGSI_OPCODE_IMIN].intr_name = "llvm.AMDGPU.imin";
1155         bld_base->op_actions[TGSI_OPCODE_INEG].emit = emit_ineg;
1156         bld_base->op_actions[TGSI_OPCODE_ISHR].emit = emit_ishr;
1157         bld_base->op_actions[TGSI_OPCODE_ISGE].emit = emit_icmp;
1158         bld_base->op_actions[TGSI_OPCODE_ISLT].emit = emit_icmp;
1159         bld_base->op_actions[TGSI_OPCODE_ISSG].emit = emit_ssg;
1160         bld_base->op_actions[TGSI_OPCODE_I2F].emit = emit_i2f;
1161         bld_base->op_actions[TGSI_OPCODE_KIL].emit = kil_emit;
1162         bld_base->op_actions[TGSI_OPCODE_KIL].intr_name = "llvm.AMDGPU.kill";
1163         bld_base->op_actions[TGSI_OPCODE_KILP].emit = lp_build_tgsi_intrinsic;
1164         bld_base->op_actions[TGSI_OPCODE_KILP].intr_name = "llvm.AMDGPU.kilp";
1165         bld_base->op_actions[TGSI_OPCODE_LG2].emit = build_tgsi_intrinsic_readonly;
1166         bld_base->op_actions[TGSI_OPCODE_LG2].intr_name = "llvm.log2.f32";
1167         bld_base->op_actions[TGSI_OPCODE_LRP].emit = build_tgsi_intrinsic_nomem;
1168         bld_base->op_actions[TGSI_OPCODE_LRP].intr_name = "llvm.AMDGPU.lrp";
1169         bld_base->op_actions[TGSI_OPCODE_MOD].emit = emit_mod;
1170         bld_base->op_actions[TGSI_OPCODE_NOT].emit = emit_not;
1171         bld_base->op_actions[TGSI_OPCODE_OR].emit = emit_or;
1172         bld_base->op_actions[TGSI_OPCODE_POW].emit = build_tgsi_intrinsic_readonly;
1173         bld_base->op_actions[TGSI_OPCODE_POW].intr_name = "llvm.pow.f32";
1174         bld_base->op_actions[TGSI_OPCODE_ROUND].emit = build_tgsi_intrinsic_nomem;
1175         bld_base->op_actions[TGSI_OPCODE_ROUND].intr_name = "llvm.AMDIL.round.nearest.";
1176         bld_base->op_actions[TGSI_OPCODE_SGE].emit = emit_cmp;
1177         bld_base->op_actions[TGSI_OPCODE_SEQ].emit = emit_cmp;
1178         bld_base->op_actions[TGSI_OPCODE_SHL].emit = emit_shl;
1179         bld_base->op_actions[TGSI_OPCODE_SLE].emit = emit_cmp;
1180         bld_base->op_actions[TGSI_OPCODE_SLT].emit = emit_cmp;
1181         bld_base->op_actions[TGSI_OPCODE_SNE].emit = emit_cmp;
1182         bld_base->op_actions[TGSI_OPCODE_SGT].emit = emit_cmp;
1183         bld_base->op_actions[TGSI_OPCODE_SIN].emit = build_tgsi_intrinsic_readonly;
1184         bld_base->op_actions[TGSI_OPCODE_SIN].intr_name = "llvm.sin.f32";
1185         bld_base->op_actions[TGSI_OPCODE_SSG].emit = emit_ssg;
1186         bld_base->op_actions[TGSI_OPCODE_TEX].fetch_args = tex_fetch_args;
1187         bld_base->op_actions[TGSI_OPCODE_TEX].intr_name = "llvm.AMDGPU.tex";
1188         bld_base->op_actions[TGSI_OPCODE_TEX2].fetch_args = tex_fetch_args;
1189         bld_base->op_actions[TGSI_OPCODE_TEX2].intr_name = "llvm.AMDGPU.tex";
1190         bld_base->op_actions[TGSI_OPCODE_TXB].fetch_args = tex_fetch_args;
1191         bld_base->op_actions[TGSI_OPCODE_TXB].intr_name = "llvm.AMDGPU.txb";
1192         bld_base->op_actions[TGSI_OPCODE_TXB2].fetch_args = tex_fetch_args;
1193         bld_base->op_actions[TGSI_OPCODE_TXB2].intr_name = "llvm.AMDGPU.txb";
1194         bld_base->op_actions[TGSI_OPCODE_TXD].fetch_args = txd_fetch_args;
1195         bld_base->op_actions[TGSI_OPCODE_TXD].intr_name = "llvm.AMDGPU.txd";
1196         bld_base->op_actions[TGSI_OPCODE_TXF].fetch_args = txf_fetch_args;
1197         bld_base->op_actions[TGSI_OPCODE_TXF].intr_name = "llvm.AMDGPU.txf";
1198         bld_base->op_actions[TGSI_OPCODE_TXL].fetch_args = tex_fetch_args;
1199         bld_base->op_actions[TGSI_OPCODE_TXL].intr_name = "llvm.AMDGPU.txl";
1200         bld_base->op_actions[TGSI_OPCODE_TXL2].fetch_args = tex_fetch_args;
1201         bld_base->op_actions[TGSI_OPCODE_TXL2].intr_name = "llvm.AMDGPU.txl";
1202         bld_base->op_actions[TGSI_OPCODE_TXP].fetch_args = txp_fetch_args;
1203         bld_base->op_actions[TGSI_OPCODE_TXP].intr_name = "llvm.AMDGPU.tex";
1204         bld_base->op_actions[TGSI_OPCODE_TXQ].fetch_args = tex_fetch_args;
1205         bld_base->op_actions[TGSI_OPCODE_TXQ].intr_name = "llvm.AMDGPU.txq";
1206         bld_base->op_actions[TGSI_OPCODE_TRUNC].emit = build_tgsi_intrinsic_nomem;
1207         bld_base->op_actions[TGSI_OPCODE_TRUNC].intr_name = "llvm.AMDGPU.trunc";
1208         bld_base->op_actions[TGSI_OPCODE_UADD].emit = emit_uadd;
1209         bld_base->op_actions[TGSI_OPCODE_UDIV].emit = emit_udiv;
1210         bld_base->op_actions[TGSI_OPCODE_UMAX].emit = build_tgsi_intrinsic_nomem;
1211         bld_base->op_actions[TGSI_OPCODE_UMAX].intr_name = "llvm.AMDGPU.umax";
1212         bld_base->op_actions[TGSI_OPCODE_UMIN].emit = build_tgsi_intrinsic_nomem;
1213         bld_base->op_actions[TGSI_OPCODE_UMIN].intr_name = "llvm.AMDGPU.umin";
1214         bld_base->op_actions[TGSI_OPCODE_UMOD].emit = emit_umod;
1215         bld_base->op_actions[TGSI_OPCODE_USEQ].emit = emit_icmp;
1216         bld_base->op_actions[TGSI_OPCODE_USGE].emit = emit_icmp;
1217         bld_base->op_actions[TGSI_OPCODE_USHR].emit = emit_ushr;
1218         bld_base->op_actions[TGSI_OPCODE_USLT].emit = emit_icmp;
1219         bld_base->op_actions[TGSI_OPCODE_USNE].emit = emit_icmp;
1220         bld_base->op_actions[TGSI_OPCODE_U2F].emit = emit_u2f;
1221         bld_base->op_actions[TGSI_OPCODE_XOR].emit = emit_xor;
1222
1223         bld_base->rsq_action.emit = build_tgsi_intrinsic_nomem;
1224         bld_base->rsq_action.intr_name = "llvm.AMDGPU.rsq";
1225 }
1226
1227 void radeon_llvm_finalize_module(struct radeon_llvm_context * ctx)
1228 {
1229         struct gallivm_state * gallivm = ctx->soa.bld_base.base.gallivm;
1230         /* End the main function with Return*/
1231         LLVMBuildRetVoid(gallivm->builder);
1232
1233         /* Create the pass manager */
1234         ctx->gallivm.passmgr = LLVMCreateFunctionPassManagerForModule(
1235                                                         gallivm->module);
1236
1237         /* This pass should eliminate all the load and store instructions */
1238         LLVMAddPromoteMemoryToRegisterPass(gallivm->passmgr);
1239
1240         /* Add some optimization passes */
1241         LLVMAddScalarReplAggregatesPass(gallivm->passmgr);
1242         LLVMAddCFGSimplificationPass(gallivm->passmgr);
1243
1244         /* Run the passs */
1245         LLVMRunFunctionPassManager(gallivm->passmgr, ctx->main_fn);
1246
1247         LLVMDisposeBuilder(gallivm->builder);
1248         LLVMDisposePassManager(gallivm->passmgr);
1249
1250 }
1251
1252 void radeon_llvm_dispose(struct radeon_llvm_context * ctx)
1253 {
1254         LLVMDisposeModule(ctx->soa.bld_base.base.gallivm->module);
1255         LLVMContextDispose(ctx->soa.bld_base.base.gallivm->context);
1256 }