src/gallium/drivers/radeonsi/si_shader_tgsi_setup.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "si_shader_internal.h"
  26 #include "si_pipe.h"
  27 #include "ac_llvm_util.h"
  28 #include "util/u_memory.h"
  29
  30 enum si_llvm_calling_convention {
  31         RADEON_LLVM_AMDGPU_VS = 87,
  32         RADEON_LLVM_AMDGPU_GS = 88,
  33         RADEON_LLVM_AMDGPU_PS = 89,
  34         RADEON_LLVM_AMDGPU_CS = 90,
  35         RADEON_LLVM_AMDGPU_HS = 93,
  36 };
  37
  38 struct si_llvm_diagnostics {
  39         struct pipe_debug_callback *debug;
  40         unsigned retval;
  41 };
  42
  43 static void si_diagnostic_handler(LLVMDiagnosticInfoRef di, void *context)
  44 {
  45         struct si_llvm_diagnostics *diag = (struct si_llvm_diagnostics *)context;
  46         LLVMDiagnosticSeverity severity = LLVMGetDiagInfoSeverity(di);
  47         char *description = LLVMGetDiagInfoDescription(di);
  48         const char *severity_str = NULL;
  49
  50         switch (severity) {
  51         case LLVMDSError:
  52                 severity_str = "error";
  53                 break;
  54         case LLVMDSWarning:
  55                 severity_str = "warning";
  56                 break;
  57         case LLVMDSRemark:
  58                 severity_str = "remark";
  59                 break;
  60         case LLVMDSNote:
  61                 severity_str = "note";
  62                 break;
  63         default:
  64                 severity_str = "unknown";
  65         }
  66
  67         pipe_debug_message(diag->debug, SHADER_INFO,
  68                            "LLVM diagnostic (%s): %s", severity_str, description);
  69
  70         if (severity == LLVMDSError) {
  71                 diag->retval = 1;
  72                 fprintf(stderr,"LLVM triggered Diagnostic Handler: %s\n", description);
  73         }
  74
  75         LLVMDisposeMessage(description);
  76 }
  77
  78 /**
  79  * Compile an LLVM module to machine code.
  80  *
  81  * @returns 0 for success, 1 for failure
  82  */
  83 unsigned si_llvm_compile(LLVMModuleRef M, struct si_shader_binary *binary,
  84                          struct ac_llvm_compiler *compiler,
  85                          struct pipe_debug_callback *debug,
  86                          bool less_optimized, unsigned wave_size)
  87 {
  88         struct ac_compiler_passes *passes = compiler->passes;
  89
  90         if (wave_size == 32)
  91                 passes = compiler->passes_wave32;
  92         else if (less_optimized && compiler->low_opt_passes)
  93                 passes = compiler->low_opt_passes;
  94
  95         struct si_llvm_diagnostics diag;
  96         LLVMContextRef llvm_ctx;
  97
  98         diag.debug = debug;
  99         diag.retval = 0;
 100
 101         /* Setup Diagnostic Handler*/
 102         llvm_ctx = LLVMGetModuleContext(M);
 103
 104         LLVMContextSetDiagnosticHandler(llvm_ctx, si_diagnostic_handler, &diag);
 105
 106         /* Compile IR. */
 107         if (!ac_compile_module_to_elf(passes, M, (char **)&binary->elf_buffer,
 108                                       &binary->elf_size))
 109                 diag.retval = 1;
 110
 111         if (diag.retval != 0)
 112                 pipe_debug_message(debug, SHADER_INFO, "LLVM compile failed");
 113         return diag.retval;
 114 }
 115
 116 void si_shader_binary_clean(struct si_shader_binary *binary)
 117 {
 118         free((void *)binary->elf_buffer);
 119         binary->elf_buffer = NULL;
 120
 121         free(binary->llvm_ir_string);
 122         binary->llvm_ir_string = NULL;
 123 }
 124
 125 LLVMTypeRef tgsi2llvmtype(struct lp_build_tgsi_context *bld_base,
 126                           enum tgsi_opcode_type type)
 127 {
 128         struct si_shader_context *ctx = si_shader_context(bld_base);
 129
 130         switch (type) {
 131         case TGSI_TYPE_UNSIGNED:
 132         case TGSI_TYPE_SIGNED:
 133                 return ctx->ac.i32;
 134         case TGSI_TYPE_UNSIGNED64:
 135         case TGSI_TYPE_SIGNED64:
 136                 return ctx->ac.i64;
 137         case TGSI_TYPE_DOUBLE:
 138                 return ctx->ac.f64;
 139         case TGSI_TYPE_UNTYPED:
 140         case TGSI_TYPE_FLOAT:
 141                 return ctx->ac.f32;
 142         default: break;
 143         }
 144         return 0;
 145 }
 146
 147 LLVMValueRef bitcast(struct lp_build_tgsi_context *bld_base,
 148                      enum tgsi_opcode_type type, LLVMValueRef value)
 149 {
 150         struct si_shader_context *ctx = si_shader_context(bld_base);
 151         LLVMTypeRef dst_type = tgsi2llvmtype(bld_base, type);
 152
 153         if (dst_type)
 154                 return LLVMBuildBitCast(ctx->ac.builder, value, dst_type, "");
 155         else
 156                 return value;
 157 }
 158
 159 /**
 160  * Return a value that is equal to the given i32 \p index if it lies in [0,num)
 161  * or an undefined value in the same interval otherwise.
 162  */
 163 LLVMValueRef si_llvm_bound_index(struct si_shader_context *ctx,
 164                                  LLVMValueRef index,
 165                                  unsigned num)
 166 {
 167         LLVMBuilderRef builder = ctx->ac.builder;
 168         LLVMValueRef c_max = LLVMConstInt(ctx->i32, num - 1, 0);
 169         LLVMValueRef cc;
 170
 171         if (util_is_power_of_two_or_zero(num)) {
 172                 index = LLVMBuildAnd(builder, index, c_max, "");
 173         } else {
 174                 /* In theory, this MAX pattern should result in code that is
 175                  * as good as the bit-wise AND above.
 176                  *
 177                  * In practice, LLVM generates worse code (at the time of
 178                  * writing), because its value tracking is not strong enough.
 179                  */
 180                 cc = LLVMBuildICmp(builder, LLVMIntULE, index, c_max, "");
 181                 index = LLVMBuildSelect(builder, cc, index, c_max, "");
 182         }
 183
 184         return index;
 185 }
 186
 187 static LLVMValueRef emit_swizzle(struct lp_build_tgsi_context *bld_base,
 188                                  LLVMValueRef value,
 189                                  unsigned swizzle_x,
 190                                  unsigned swizzle_y,
 191                                  unsigned swizzle_z,
 192                                  unsigned swizzle_w)
 193 {
 194         struct si_shader_context *ctx = si_shader_context(bld_base);
 195         LLVMValueRef swizzles[4];
 196
 197         swizzles[0] = LLVMConstInt(ctx->i32, swizzle_x, 0);
 198         swizzles[1] = LLVMConstInt(ctx->i32, swizzle_y, 0);
 199         swizzles[2] = LLVMConstInt(ctx->i32, swizzle_z, 0);
 200         swizzles[3] = LLVMConstInt(ctx->i32, swizzle_w, 0);
 201
 202         return LLVMBuildShuffleVector(ctx->ac.builder,
 203                                       value,
 204                                       LLVMGetUndef(LLVMTypeOf(value)),
 205                                       LLVMConstVector(swizzles, 4), "");
 206 }
 207
 208 /**
 209  * Return the description of the array covering the given temporary register
 210  * index.
 211  */
 212 static unsigned
 213 get_temp_array_id(struct lp_build_tgsi_context *bld_base,
 214                   unsigned reg_index,
 215                   const struct tgsi_ind_register *reg)
 216 {
 217         struct si_shader_context *ctx = si_shader_context(bld_base);
 218         unsigned num_arrays = ctx->bld_base.info->array_max[TGSI_FILE_TEMPORARY];
 219         unsigned i;
 220
 221         if (reg && reg->ArrayID > 0 && reg->ArrayID <= num_arrays)
 222                 return reg->ArrayID;
 223
 224         for (i = 0; i < num_arrays; i++) {
 225                 const struct tgsi_array_info *array = &ctx->temp_arrays[i];
 226
 227                 if (reg_index >= array->range.First && reg_index <= array->range.Last)
 228                         return i + 1;
 229         }
 230
 231         return 0;
 232 }
 233
 234 static struct tgsi_declaration_range
 235 get_array_range(struct lp_build_tgsi_context *bld_base,
 236                 unsigned File, unsigned reg_index,
 237                 const struct tgsi_ind_register *reg)
 238 {
 239         struct si_shader_context *ctx = si_shader_context(bld_base);
 240         struct tgsi_declaration_range range;
 241
 242         if (File == TGSI_FILE_TEMPORARY) {
 243                 unsigned array_id = get_temp_array_id(bld_base, reg_index, reg);
 244                 if (array_id)
 245                         return ctx->temp_arrays[array_id - 1].range;
 246         }
 247
 248         range.First = 0;
 249         range.Last = bld_base->info->file_max[File];
 250         return range;
 251 }
 252
 253 /**
 254  * For indirect registers, construct a pointer directly to the requested
 255  * element using getelementptr if possible.
 256  *
 257  * Returns NULL if the insertelement/extractelement fallback for array access
 258  * must be used.
 259  */
 260 static LLVMValueRef
 261 get_pointer_into_array(struct si_shader_context *ctx,
 262                        unsigned file,
 263                        unsigned swizzle,
 264                        unsigned reg_index,
 265                        const struct tgsi_ind_register *reg_indirect)
 266 {
 267         unsigned array_id;
 268         struct tgsi_array_info *array;
 269         LLVMValueRef idxs[2];
 270         LLVMValueRef index;
 271         LLVMValueRef alloca;
 272
 273         if (file != TGSI_FILE_TEMPORARY)
 274                 return NULL;
 275
 276         array_id = get_temp_array_id(&ctx->bld_base, reg_index, reg_indirect);
 277         if (!array_id)
 278                 return NULL;
 279
 280         alloca = ctx->temp_array_allocas[array_id - 1];
 281         if (!alloca)
 282                 return NULL;
 283
 284         array = &ctx->temp_arrays[array_id - 1];
 285
 286         if (!(array->writemask & (1 << swizzle)))
 287                 return ctx->undef_alloca;
 288
 289         index = si_get_indirect_index(ctx, reg_indirect, 1,
 290                                       reg_index - ctx->temp_arrays[array_id - 1].range.First);
 291
 292         /* Ensure that the index is within a valid range, to guard against
 293          * VM faults and overwriting critical data (e.g. spilled resource
 294          * descriptors).
 295          *
 296          * TODO It should be possible to avoid the additional instructions
 297          * if LLVM is changed so that it guarantuees:
 298          * 1. the scratch space descriptor isolates the current wave (this
 299          *    could even save the scratch offset SGPR at the cost of an
 300          *    additional SALU instruction)
 301          * 2. the memory for allocas must be allocated at the _end_ of the
 302          *    scratch space (after spilled registers)
 303          */
 304         index = si_llvm_bound_index(ctx, index, array->range.Last - array->range.First + 1);
 305
 306         index = ac_build_imad(&ctx->ac, index,
 307                               LLVMConstInt(ctx->i32, util_bitcount(array->writemask), 0),
 308                               LLVMConstInt(ctx->i32,
 309                                            util_bitcount(array->writemask & ((1 << swizzle) - 1)), 0));
 310         idxs[0] = ctx->i32_0;
 311         idxs[1] = index;
 312         return LLVMBuildGEP(ctx->ac.builder, alloca, idxs, 2, "");
 313 }
 314
 315 LLVMValueRef
 316 si_llvm_emit_fetch_64bit(struct lp_build_tgsi_context *bld_base,
 317                          LLVMTypeRef type,
 318                          LLVMValueRef ptr,
 319                          LLVMValueRef ptr2)
 320 {
 321         struct si_shader_context *ctx = si_shader_context(bld_base);
 322         LLVMValueRef values[2] = {
 323                 ac_to_integer(&ctx->ac, ptr),
 324                 ac_to_integer(&ctx->ac, ptr2),
 325         };
 326         LLVMValueRef result = ac_build_gather_values(&ctx->ac, values, 2);
 327         return LLVMBuildBitCast(ctx->ac.builder, result, type, "");
 328 }
 329
 330 static LLVMValueRef
 331 emit_array_fetch(struct lp_build_tgsi_context *bld_base,
 332                  unsigned File, enum tgsi_opcode_type type,
 333                  struct tgsi_declaration_range range,
 334                  unsigned swizzle_in)
 335 {
 336         struct si_shader_context *ctx = si_shader_context(bld_base);
 337         unsigned i, size = range.Last - range.First + 1;
 338         LLVMTypeRef vec = LLVMVectorType(tgsi2llvmtype(bld_base, type), size);
 339         LLVMValueRef result = LLVMGetUndef(vec);
 340         unsigned swizzle = swizzle_in;
 341         struct tgsi_full_src_register tmp_reg = {};
 342         tmp_reg.Register.File = File;
 343         if (tgsi_type_is_64bit(type))
 344                 swizzle |= (swizzle_in + 1) << 16;
 345
 346         for (i = 0; i < size; ++i) {
 347                 tmp_reg.Register.Index = i + range.First;
 348
 349                 LLVMValueRef temp = si_llvm_emit_fetch(bld_base, &tmp_reg, type, swizzle);
 350                 result = LLVMBuildInsertElement(ctx->ac.builder, result, temp,
 351                         LLVMConstInt(ctx->i32, i, 0), "array_vector");
 352         }
 353         return result;
 354 }
 355
 356 static LLVMValueRef
 357 load_value_from_array(struct lp_build_tgsi_context *bld_base,
 358                       unsigned file,
 359                       enum tgsi_opcode_type type,
 360                       unsigned swizzle,
 361                       unsigned reg_index,
 362                       const struct tgsi_ind_register *reg_indirect)
 363 {
 364         struct si_shader_context *ctx = si_shader_context(bld_base);
 365         LLVMBuilderRef builder = ctx->ac.builder;
 366         LLVMValueRef ptr;
 367
 368         ptr = get_pointer_into_array(ctx, file, swizzle, reg_index, reg_indirect);
 369         if (ptr) {
 370                 LLVMValueRef val = LLVMBuildLoad(builder, ptr, "");
 371                 if (tgsi_type_is_64bit(type)) {
 372                         LLVMValueRef ptr_hi, val_hi;
 373                         ptr_hi = LLVMBuildGEP(builder, ptr, &ctx->i32_1, 1, "");
 374                         val_hi = LLVMBuildLoad(builder, ptr_hi, "");
 375                         val = si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
 376                                                        val, val_hi);
 377                 }
 378
 379                 return val;
 380         } else {
 381                 struct tgsi_declaration_range range =
 382                         get_array_range(bld_base, file, reg_index, reg_indirect);
 383                 LLVMValueRef index =
 384                         si_get_indirect_index(ctx, reg_indirect, 1, reg_index - range.First);
 385                 LLVMValueRef array =
 386                         emit_array_fetch(bld_base, file, type, range, swizzle);
 387                 return LLVMBuildExtractElement(builder, array, index, "");
 388         }
 389 }
 390
 391 static void
 392 store_value_to_array(struct lp_build_tgsi_context *bld_base,
 393                      LLVMValueRef value,
 394                      unsigned file,
 395                      unsigned chan_index,
 396                      unsigned reg_index,
 397                      const struct tgsi_ind_register *reg_indirect)
 398 {
 399         struct si_shader_context *ctx = si_shader_context(bld_base);
 400         LLVMBuilderRef builder = ctx->ac.builder;
 401         LLVMValueRef ptr;
 402
 403         ptr = get_pointer_into_array(ctx, file, chan_index, reg_index, reg_indirect);
 404         if (ptr) {
 405                 LLVMBuildStore(builder, value, ptr);
 406         } else {
 407                 unsigned i, size;
 408                 struct tgsi_declaration_range range = get_array_range(bld_base, file, reg_index, reg_indirect);
 409                 LLVMValueRef index = si_get_indirect_index(ctx, reg_indirect, 1, reg_index - range.First);
 410                 LLVMValueRef array =
 411                         emit_array_fetch(bld_base, file, TGSI_TYPE_FLOAT, range, chan_index);
 412                 LLVMValueRef temp_ptr;
 413
 414                 array = LLVMBuildInsertElement(builder, array, value, index, "");
 415
 416                 size = range.Last - range.First + 1;
 417                 for (i = 0; i < size; ++i) {
 418                         switch(file) {
 419                         case TGSI_FILE_OUTPUT:
 420                                 temp_ptr = ctx->outputs[i + range.First][chan_index];
 421                                 break;
 422
 423                         case TGSI_FILE_TEMPORARY:
 424                                 if (range.First + i >= ctx->temps_count)
 425                                         continue;
 426                                 temp_ptr = ctx->temps[(i + range.First) * TGSI_NUM_CHANNELS + chan_index];
 427                                 break;
 428
 429                         default:
 430                                 continue;
 431                         }
 432                         value = LLVMBuildExtractElement(builder, array,
 433                                 LLVMConstInt(ctx->i32, i, 0), "");
 434                         LLVMBuildStore(builder, value, temp_ptr);
 435                 }
 436         }
 437 }
 438
 439 /* If this is true, preload FS inputs at the beginning of shaders. Otherwise,
 440  * reload them at each use. This must be true if the shader is using
 441  * derivatives and KILL, because KILL can leave the WQM and then a lazy
 442  * input load isn't in the WQM anymore.
 443  */
 444 static bool si_preload_fs_inputs(struct si_shader_context *ctx)
 445 {
 446         struct si_shader_selector *sel = ctx->shader->selector;
 447
 448         return sel->info.uses_derivatives &&
 449                sel->info.uses_kill;
 450 }
 451
 452 static LLVMValueRef
 453 get_output_ptr(struct lp_build_tgsi_context *bld_base, unsigned index,
 454                unsigned chan)
 455 {
 456         struct si_shader_context *ctx = si_shader_context(bld_base);
 457
 458         assert(index <= ctx->bld_base.info->file_max[TGSI_FILE_OUTPUT]);
 459         return ctx->outputs[index][chan];
 460 }
 461
 462 LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base,
 463                                 const struct tgsi_full_src_register *reg,
 464                                 enum tgsi_opcode_type type,
 465                                 unsigned swizzle_in)
 466 {
 467         struct si_shader_context *ctx = si_shader_context(bld_base);
 468         LLVMBuilderRef builder = ctx->ac.builder;
 469         LLVMValueRef result = NULL, ptr, ptr2;
 470         unsigned swizzle = swizzle_in & 0xffff;
 471
 472         if (swizzle_in == ~0) {
 473                 LLVMValueRef values[TGSI_NUM_CHANNELS];
 474                 unsigned chan;
 475                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 476                         values[chan] = si_llvm_emit_fetch(bld_base, reg, type, chan);
 477                 }
 478                 return ac_build_gather_values(&ctx->ac, values,
 479                                               TGSI_NUM_CHANNELS);
 480         }
 481
 482         if (reg->Register.Indirect) {
 483                 LLVMValueRef load = load_value_from_array(bld_base, reg->Register.File, type,
 484                                 swizzle, reg->Register.Index, &reg->Indirect);
 485                 return bitcast(bld_base, type, load);
 486         }
 487
 488         switch(reg->Register.File) {
 489         case TGSI_FILE_IMMEDIATE: {
 490                 LLVMTypeRef ctype = tgsi2llvmtype(bld_base, type);
 491                 if (tgsi_type_is_64bit(type)) {
 492                         result = LLVMGetUndef(LLVMVectorType(ctx->i32, 2));
 493                         result = LLVMConstInsertElement(result,
 494                                                         ctx->imms[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle],
 495                                                         ctx->i32_0);
 496                         result = LLVMConstInsertElement(result,
 497                                                         ctx->imms[reg->Register.Index * TGSI_NUM_CHANNELS + (swizzle_in >> 16)],
 498                                                         ctx->i32_1);
 499                         return LLVMConstBitCast(result, ctype);
 500                 } else {
 501                         return LLVMConstBitCast(ctx->imms[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle], ctype);
 502                 }
 503         }
 504
 505         case TGSI_FILE_INPUT: {
 506                 unsigned index = reg->Register.Index;
 507                 LLVMValueRef input[4];
 508
 509                 /* I don't think doing this for vertex shaders is beneficial.
 510                  * For those, we want to make sure the VMEM loads are executed
 511                  * only once. Fragment shaders don't care much, because
 512                  * v_interp instructions are much cheaper than VMEM loads.
 513                  */
 514                 if (!si_preload_fs_inputs(ctx) &&
 515                     ctx->bld_base.info->processor == PIPE_SHADER_FRAGMENT)
 516                         ctx->load_input(ctx, index, &ctx->input_decls[index], input);
 517                 else
 518                         memcpy(input, &ctx->inputs[index * 4], sizeof(input));
 519
 520                 result = input[swizzle];
 521
 522                 if (tgsi_type_is_64bit(type)) {
 523                         ptr = result;
 524                         ptr2 = input[swizzle_in >> 16];
 525                         return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
 526                                                         ptr, ptr2);
 527                 }
 528                 break;
 529         }
 530
 531         case TGSI_FILE_TEMPORARY:
 532                 if (reg->Register.Index >= ctx->temps_count)
 533                         return LLVMGetUndef(tgsi2llvmtype(bld_base, type));
 534                 ptr = ctx->temps[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle];
 535                 if (tgsi_type_is_64bit(type)) {
 536                         ptr2 = ctx->temps[reg->Register.Index * TGSI_NUM_CHANNELS + (swizzle_in >> 16)];
 537                         return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
 538                                                         LLVMBuildLoad(builder, ptr, ""),
 539                                                         LLVMBuildLoad(builder, ptr2, ""));
 540                 }
 541                 result = LLVMBuildLoad(builder, ptr, "");
 542                 break;
 543
 544         case TGSI_FILE_OUTPUT:
 545                 ptr = get_output_ptr(bld_base, reg->Register.Index, swizzle);
 546                 if (tgsi_type_is_64bit(type)) {
 547                         ptr2 = get_output_ptr(bld_base, reg->Register.Index, (swizzle_in >> 16));
 548                         return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
 549                                                         LLVMBuildLoad(builder, ptr, ""),
 550                                                         LLVMBuildLoad(builder, ptr2, ""));
 551                 }
 552                 result = LLVMBuildLoad(builder, ptr, "");
 553                 break;
 554
 555         default:
 556                 return LLVMGetUndef(tgsi2llvmtype(bld_base, type));
 557         }
 558
 559         return bitcast(bld_base, type, result);
 560 }
 561
 562 static LLVMValueRef fetch_system_value(struct lp_build_tgsi_context *bld_base,
 563                                        const struct tgsi_full_src_register *reg,
 564                                        enum tgsi_opcode_type type,
 565                                        unsigned swizzle_in)
 566 {
 567         struct si_shader_context *ctx = si_shader_context(bld_base);
 568         LLVMBuilderRef builder = ctx->ac.builder;
 569         LLVMValueRef cval = ctx->system_values[reg->Register.Index];
 570         unsigned swizzle = swizzle_in & 0xffff;
 571
 572         if (tgsi_type_is_64bit(type)) {
 573                 LLVMValueRef lo, hi;
 574
 575                 assert(swizzle == 0 || swizzle == 2);
 576
 577                 lo = LLVMBuildExtractElement(
 578                         builder, cval, LLVMConstInt(ctx->i32, swizzle, 0), "");
 579                 hi = LLVMBuildExtractElement(
 580                         builder, cval, LLVMConstInt(ctx->i32, (swizzle_in >> 16), 0), "");
 581
 582                 return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
 583                                                 lo, hi);
 584         }
 585
 586         if (LLVMGetTypeKind(LLVMTypeOf(cval)) == LLVMVectorTypeKind) {
 587                 cval = LLVMBuildExtractElement(
 588                         builder, cval, LLVMConstInt(ctx->i32, swizzle, 0), "");
 589         } else {
 590                 assert(swizzle == 0);
 591         }
 592
 593         return bitcast(bld_base, type, cval);
 594 }
 595
 596 static void emit_declaration(struct lp_build_tgsi_context *bld_base,
 597                              const struct tgsi_full_declaration *decl)
 598 {
 599         struct si_shader_context *ctx = si_shader_context(bld_base);
 600         LLVMBuilderRef builder = ctx->ac.builder;
 601         unsigned first, last, i;
 602         switch(decl->Declaration.File) {
 603         case TGSI_FILE_ADDRESS:
 604         {
 605                  unsigned idx;
 606                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 607                         unsigned chan;
 608                         for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 609                                  ctx->addrs[idx][chan] = ac_build_alloca_undef(
 610                                         &ctx->ac, ctx->i32, "");
 611                         }
 612                 }
 613                 break;
 614         }
 615
 616         case TGSI_FILE_TEMPORARY:
 617         {
 618                 char name[18] = "";
 619                 LLVMValueRef array_alloca = NULL;
 620                 unsigned decl_size;
 621                 unsigned writemask = decl->Declaration.UsageMask;
 622                 first = decl->Range.First;
 623                 last = decl->Range.Last;
 624                 decl_size = 4 * ((last - first) + 1);
 625
 626                 if (decl->Declaration.Array) {
 627                         unsigned id = decl->Array.ArrayID - 1;
 628                         unsigned array_size;
 629
 630                         writemask &= ctx->temp_arrays[id].writemask;
 631                         ctx->temp_arrays[id].writemask = writemask;
 632                         array_size = ((last - first) + 1) * util_bitcount(writemask);
 633
 634                         /* If the array has more than 16 elements, store it
 635                          * in memory using an alloca that spans the entire
 636                          * array.
 637                          *
 638                          * Otherwise, store each array element individually.
 639                          * We will then generate vectors (per-channel, up to
 640                          * <16 x float> if the usagemask is a single bit) for
 641                          * indirect addressing.
 642                          *
 643                          * Note that 16 is the number of vector elements that
 644                          * LLVM will store in a register, so theoretically an
 645                          * array with up to 4 * 16 = 64 elements could be
 646                          * handled this way, but whether that's a good idea
 647                          * depends on VGPR register pressure elsewhere.
 648                          *
 649                          * FIXME: We shouldn't need to have the non-alloca
 650                          * code path for arrays. LLVM should be smart enough to
 651                          * promote allocas into registers when profitable.
 652                          */
 653                         if (array_size > 16 ||
 654                             !ctx->screen->llvm_has_working_vgpr_indexing) {
 655                                 array_alloca = ac_build_alloca_undef(&ctx->ac,
 656                                         LLVMArrayType(ctx->f32,
 657                                                       array_size), "array");
 658                                 ctx->temp_array_allocas[id] = array_alloca;
 659                         }
 660                 }
 661
 662                 if (!ctx->temps_count) {
 663                         ctx->temps_count = bld_base->info->file_max[TGSI_FILE_TEMPORARY] + 1;
 664                         ctx->temps = MALLOC(TGSI_NUM_CHANNELS * ctx->temps_count * sizeof(LLVMValueRef));
 665                 }
 666                 if (!array_alloca) {
 667                         for (i = 0; i < decl_size; ++i) {
 668 #ifndef NDEBUG
 669                                 snprintf(name, sizeof(name), "TEMP%d.%c",
 670                                          first + i / 4, "xyzw"[i % 4]);
 671 #endif
 672                                 ctx->temps[first * TGSI_NUM_CHANNELS + i] =
 673                                         ac_build_alloca_undef(&ctx->ac,
 674                                                               ctx->f32,
 675                                                               name);
 676                         }
 677                 } else {
 678                         LLVMValueRef idxs[2] = {
 679                                 ctx->i32_0,
 680                                 NULL
 681                         };
 682                         unsigned j = 0;
 683
 684                         if (writemask != TGSI_WRITEMASK_XYZW &&
 685                             !ctx->undef_alloca) {
 686                                 /* Create a dummy alloca. We use it so that we
 687                                  * have a pointer that is safe to load from if
 688                                  * a shader ever reads from a channel that
 689                                  * it never writes to.
 690                                  */
 691                                 ctx->undef_alloca = ac_build_alloca_undef(
 692                                         &ctx->ac, ctx->f32, "undef");
 693                         }
 694
 695                         for (i = 0; i < decl_size; ++i) {
 696                                 LLVMValueRef ptr;
 697                                 if (writemask & (1 << (i % 4))) {
 698 #ifndef NDEBUG
 699                                         snprintf(name, sizeof(name), "TEMP%d.%c",
 700                                                  first + i / 4, "xyzw"[i % 4]);
 701 #endif
 702                                         idxs[1] = LLVMConstInt(ctx->i32, j, 0);
 703                                         ptr = LLVMBuildGEP(builder, array_alloca, idxs, 2, name);
 704                                         j++;
 705                                 } else {
 706                                         ptr = ctx->undef_alloca;
 707                                 }
 708                                 ctx->temps[first * TGSI_NUM_CHANNELS + i] = ptr;
 709                         }
 710                 }
 711                 break;
 712         }
 713         case TGSI_FILE_INPUT:
 714         {
 715                 unsigned idx;
 716                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 717                         if (ctx->load_input &&
 718                             ctx->input_decls[idx].Declaration.File != TGSI_FILE_INPUT) {
 719                                 ctx->input_decls[idx] = *decl;
 720                                 ctx->input_decls[idx].Range.First = idx;
 721                                 ctx->input_decls[idx].Range.Last = idx;
 722                                 ctx->input_decls[idx].Semantic.Index += idx - decl->Range.First;
 723
 724                                 if (si_preload_fs_inputs(ctx) ||
 725                                     bld_base->info->processor != PIPE_SHADER_FRAGMENT)
 726                                         ctx->load_input(ctx, idx, &ctx->input_decls[idx],
 727                                                         &ctx->inputs[idx * 4]);
 728                         }
 729                 }
 730         }
 731         break;
 732
 733         case TGSI_FILE_SYSTEM_VALUE:
 734         {
 735                 unsigned idx;
 736                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 737                         si_load_system_value(ctx, idx, decl);
 738                 }
 739         }
 740         break;
 741
 742         case TGSI_FILE_OUTPUT:
 743         {
 744                 char name[16] = "";
 745                 unsigned idx;
 746                 for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) {
 747                         unsigned chan;
 748                         assert(idx < RADEON_LLVM_MAX_OUTPUTS);
 749                         if (ctx->outputs[idx][0])
 750                                 continue;
 751                         for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 752 #ifndef NDEBUG
 753                                 snprintf(name, sizeof(name), "OUT%d.%c",
 754                                          idx, "xyzw"[chan % 4]);
 755 #endif
 756                                 ctx->outputs[idx][chan] = ac_build_alloca_undef(
 757                                         &ctx->ac, ctx->f32, name);
 758                         }
 759                 }
 760                 break;
 761         }
 762
 763         case TGSI_FILE_MEMORY:
 764                 si_tgsi_declare_compute_memory(ctx, decl);
 765                 break;
 766
 767         default:
 768                 break;
 769         }
 770 }
 771
 772 void si_llvm_emit_store(struct lp_build_tgsi_context *bld_base,
 773                         const struct tgsi_full_instruction *inst,
 774                         const struct tgsi_opcode_info *info,
 775                         unsigned index,
 776                         LLVMValueRef dst[4])
 777 {
 778         struct si_shader_context *ctx = si_shader_context(bld_base);
 779         const struct tgsi_full_dst_register *reg = &inst->Dst[index];
 780         LLVMBuilderRef builder = ctx->ac.builder;
 781         LLVMValueRef temp_ptr, temp_ptr2 = NULL;
 782         bool is_vec_store = false;
 783         enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode, index);
 784
 785         if (dst[0]) {
 786                 LLVMTypeKind k = LLVMGetTypeKind(LLVMTypeOf(dst[0]));
 787                 is_vec_store = (k == LLVMVectorTypeKind);
 788         }
 789
 790         if (is_vec_store) {
 791                 LLVMValueRef values[4] = {};
 792                 uint32_t writemask = reg->Register.WriteMask;
 793                 while (writemask) {
 794                         unsigned chan = u_bit_scan(&writemask);
 795                         LLVMValueRef index = LLVMConstInt(ctx->i32, chan, 0);
 796                         values[chan]  = LLVMBuildExtractElement(ctx->ac.builder,
 797                                                         dst[0], index, "");
 798                 }
 799                 bld_base->emit_store(bld_base, inst, info, index, values);
 800                 return;
 801         }
 802
 803         uint32_t writemask = reg->Register.WriteMask;
 804         while (writemask) {
 805                 unsigned chan_index = u_bit_scan(&writemask);
 806                 LLVMValueRef value = dst[chan_index];
 807
 808                 if (tgsi_type_is_64bit(dtype) && (chan_index == 1 || chan_index == 3))
 809                         continue;
 810                 if (inst->Instruction.Saturate)
 811                         value = ac_build_clamp(&ctx->ac, value);
 812
 813                 if (reg->Register.File == TGSI_FILE_ADDRESS) {
 814                         temp_ptr = ctx->addrs[reg->Register.Index][chan_index];
 815                         LLVMBuildStore(builder, value, temp_ptr);
 816                         continue;
 817                 }
 818
 819                 if (!tgsi_type_is_64bit(dtype))
 820                         value = ac_to_float(&ctx->ac, value);
 821
 822                 if (reg->Register.Indirect) {
 823                         unsigned file = reg->Register.File;
 824                         unsigned reg_index = reg->Register.Index;
 825                         store_value_to_array(bld_base, value, file, chan_index,
 826                                              reg_index, &reg->Indirect);
 827                 } else {
 828                         switch(reg->Register.File) {
 829                         case TGSI_FILE_OUTPUT:
 830                                 temp_ptr = ctx->outputs[reg->Register.Index][chan_index];
 831                                 if (tgsi_type_is_64bit(dtype))
 832                                         temp_ptr2 = ctx->outputs[reg->Register.Index][chan_index + 1];
 833                                 break;
 834
 835                         case TGSI_FILE_TEMPORARY:
 836                         {
 837                                 if (reg->Register.Index >= ctx->temps_count)
 838                                         continue;
 839
 840                                 temp_ptr = ctx->temps[ TGSI_NUM_CHANNELS * reg->Register.Index + chan_index];
 841                                 if (tgsi_type_is_64bit(dtype))
 842                                         temp_ptr2 = ctx->temps[ TGSI_NUM_CHANNELS * reg->Register.Index + chan_index + 1];
 843
 844                                 break;
 845                         }
 846                         default:
 847                                 return;
 848                         }
 849                         if (!tgsi_type_is_64bit(dtype))
 850                                 LLVMBuildStore(builder, value, temp_ptr);
 851                         else {
 852                                 LLVMValueRef ptr = LLVMBuildBitCast(builder, value,
 853                                                                     LLVMVectorType(ctx->i32, 2), "");
 854                                 LLVMValueRef val2;
 855                                 value = LLVMBuildExtractElement(builder, ptr,
 856                                                                 ctx->i32_0, "");
 857                                 val2 = LLVMBuildExtractElement(builder, ptr,
 858                                                                ctx->i32_1, "");
 859
 860                                 LLVMBuildStore(builder, ac_to_float(&ctx->ac, value), temp_ptr);
 861                                 LLVMBuildStore(builder, ac_to_float(&ctx->ac, val2), temp_ptr2);
 862                         }
 863                 }
 864         }
 865 }
 866
 867 static int get_line(int pc)
 868 {
 869         /* Subtract 1 so that the number shown is that of the corresponding
 870          * opcode in the TGSI dump, e.g. an if block has the same suffix as
 871          * the instruction number of the corresponding TGSI IF.
 872          */
 873         return pc - 1;
 874 }
 875
 876 static void bgnloop_emit(const struct lp_build_tgsi_action *action,
 877                          struct lp_build_tgsi_context *bld_base,
 878                          struct lp_build_emit_data *emit_data)
 879 {
 880         struct si_shader_context *ctx = si_shader_context(bld_base);
 881         ac_build_bgnloop(&ctx->ac, get_line(bld_base->pc));
 882 }
 883
 884 static void brk_emit(const struct lp_build_tgsi_action *action,
 885                      struct lp_build_tgsi_context *bld_base,
 886                      struct lp_build_emit_data *emit_data)
 887 {
 888         struct si_shader_context *ctx = si_shader_context(bld_base);
 889         ac_build_break(&ctx->ac);
 890 }
 891
 892 static void cont_emit(const struct lp_build_tgsi_action *action,
 893                       struct lp_build_tgsi_context *bld_base,
 894                       struct lp_build_emit_data *emit_data)
 895 {
 896         struct si_shader_context *ctx = si_shader_context(bld_base);
 897         ac_build_continue(&ctx->ac);
 898 }
 899
 900 static void else_emit(const struct lp_build_tgsi_action *action,
 901                       struct lp_build_tgsi_context *bld_base,
 902                       struct lp_build_emit_data *emit_data)
 903 {
 904         struct si_shader_context *ctx = si_shader_context(bld_base);
 905         ac_build_else(&ctx->ac, get_line(bld_base->pc));
 906 }
 907
 908 static void endif_emit(const struct lp_build_tgsi_action *action,
 909                        struct lp_build_tgsi_context *bld_base,
 910                        struct lp_build_emit_data *emit_data)
 911 {
 912         struct si_shader_context *ctx = si_shader_context(bld_base);
 913         ac_build_endif(&ctx->ac, get_line(bld_base->pc));
 914 }
 915
 916 static void endloop_emit(const struct lp_build_tgsi_action *action,
 917                          struct lp_build_tgsi_context *bld_base,
 918                          struct lp_build_emit_data *emit_data)
 919 {
 920         struct si_shader_context *ctx = si_shader_context(bld_base);
 921         ac_build_endloop(&ctx->ac, get_line(bld_base->pc));
 922 }
 923
 924 static void if_emit(const struct lp_build_tgsi_action *action,
 925                     struct lp_build_tgsi_context *bld_base,
 926                     struct lp_build_emit_data *emit_data)
 927 {
 928         struct si_shader_context *ctx = si_shader_context(bld_base);
 929         ac_build_if(&ctx->ac, emit_data->args[0], get_line(bld_base->pc));
 930 }
 931
 932 static void uif_emit(const struct lp_build_tgsi_action *action,
 933                      struct lp_build_tgsi_context *bld_base,
 934                      struct lp_build_emit_data *emit_data)
 935 {
 936         struct si_shader_context *ctx = si_shader_context(bld_base);
 937         ac_build_uif(&ctx->ac, emit_data->args[0], get_line(bld_base->pc));
 938 }
 939
 940 static void emit_immediate(struct lp_build_tgsi_context *bld_base,
 941                            const struct tgsi_full_immediate *imm)
 942 {
 943         unsigned i;
 944         struct si_shader_context *ctx = si_shader_context(bld_base);
 945
 946         for (i = 0; i < 4; ++i) {
 947                 ctx->imms[ctx->imms_num * TGSI_NUM_CHANNELS + i] =
 948                                 LLVMConstInt(ctx->i32, imm->u[i].Uint, false   );
 949         }
 950
 951         ctx->imms_num++;
 952 }
 953
 954 void si_llvm_context_init(struct si_shader_context *ctx,
 955                           struct si_screen *sscreen,
 956                           struct ac_llvm_compiler *compiler,
 957                           unsigned wave_size,
 958                           unsigned ballot_mask_bits)
 959 {
 960         struct lp_type type;
 961
 962         /* Initialize the gallivm object:
 963          * We are only using the module, context, and builder fields of this struct.
 964          * This should be enough for us to be able to pass our gallivm struct to the
 965          * helper functions in the gallivm module.
 966          */
 967         memset(ctx, 0, sizeof(*ctx));
 968         ctx->screen = sscreen;
 969         ctx->compiler = compiler;
 970
 971         ac_llvm_context_init(&ctx->ac, compiler, sscreen->info.chip_class,
 972                              sscreen->info.family,
 973                              AC_FLOAT_MODE_NO_SIGNED_ZEROS_FP_MATH,
 974                              wave_size, ballot_mask_bits);
 975
 976         ctx->gallivm.context = ctx->ac.context;
 977         ctx->gallivm.module = ctx->ac.module;
 978         ctx->gallivm.builder = ctx->ac.builder;
 979
 980         struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
 981
 982         type.floating = true;
 983         type.fixed = false;
 984         type.sign = true;
 985         type.norm = false;
 986         type.width = 32;
 987         type.length = 1;
 988
 989         lp_build_context_init(&bld_base->base, &ctx->gallivm, type);
 990         lp_build_context_init(&ctx->bld_base.uint_bld, &ctx->gallivm, lp_uint_type(type));
 991         lp_build_context_init(&ctx->bld_base.int_bld, &ctx->gallivm, lp_int_type(type));
 992         type.width *= 2;
 993         lp_build_context_init(&ctx->bld_base.dbl_bld, &ctx->gallivm, type);
 994         lp_build_context_init(&ctx->bld_base.uint64_bld, &ctx->gallivm, lp_uint_type(type));
 995         lp_build_context_init(&ctx->bld_base.int64_bld, &ctx->gallivm, lp_int_type(type));
 996
 997         bld_base->soa = 1;
 998         bld_base->emit_swizzle = emit_swizzle;
 999         bld_base->emit_declaration = emit_declaration;
1000         bld_base->emit_immediate = emit_immediate;
1001
1002         bld_base->op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit;
1003         bld_base->op_actions[TGSI_OPCODE_BRK].emit = brk_emit;
1004         bld_base->op_actions[TGSI_OPCODE_CONT].emit = cont_emit;
1005         bld_base->op_actions[TGSI_OPCODE_IF].emit = if_emit;
1006         bld_base->op_actions[TGSI_OPCODE_UIF].emit = uif_emit;
1007         bld_base->op_actions[TGSI_OPCODE_ELSE].emit = else_emit;
1008         bld_base->op_actions[TGSI_OPCODE_ENDIF].emit = endif_emit;
1009         bld_base->op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit;
1010
1011         si_shader_context_init_alu(&ctx->bld_base);
1012         si_shader_context_init_mem(ctx);
1013
1014         ctx->voidt = LLVMVoidTypeInContext(ctx->ac.context);
1015         ctx->i1 = LLVMInt1TypeInContext(ctx->ac.context);
1016         ctx->i8 = LLVMInt8TypeInContext(ctx->ac.context);
1017         ctx->i32 = LLVMInt32TypeInContext(ctx->ac.context);
1018         ctx->i64 = LLVMInt64TypeInContext(ctx->ac.context);
1019         ctx->i128 = LLVMIntTypeInContext(ctx->ac.context, 128);
1020         ctx->f32 = LLVMFloatTypeInContext(ctx->ac.context);
1021         ctx->v2i32 = LLVMVectorType(ctx->i32, 2);
1022         ctx->v4i32 = LLVMVectorType(ctx->i32, 4);
1023         ctx->v4f32 = LLVMVectorType(ctx->f32, 4);
1024         ctx->v8i32 = LLVMVectorType(ctx->i32, 8);
1025
1026         ctx->i32_0 = LLVMConstInt(ctx->i32, 0, 0);
1027         ctx->i32_1 = LLVMConstInt(ctx->i32, 1, 0);
1028         ctx->i1false = LLVMConstInt(ctx->i1, 0, 0);
1029         ctx->i1true = LLVMConstInt(ctx->i1, 1, 0);
1030 }
1031
1032 /* Set the context to a certain TGSI shader. Can be called repeatedly
1033  * to change the shader. */
1034 void si_llvm_context_set_ir(struct si_shader_context *ctx,
1035                             struct si_shader *shader)
1036 {
1037         struct si_shader_selector *sel = shader->selector;
1038         const struct tgsi_shader_info *info = &sel->info;
1039
1040         ctx->shader = shader;
1041         ctx->type = sel->type;
1042         ctx->bld_base.info = info;
1043
1044         /* Clean up the old contents. */
1045         FREE(ctx->temp_arrays);
1046         ctx->temp_arrays = NULL;
1047         FREE(ctx->temp_array_allocas);
1048         ctx->temp_array_allocas = NULL;
1049
1050         FREE(ctx->imms);
1051         ctx->imms = NULL;
1052         ctx->imms_num = 0;
1053
1054         FREE(ctx->temps);
1055         ctx->temps = NULL;
1056         ctx->temps_count = 0;
1057
1058         ctx->num_const_buffers = util_last_bit(info->const_buffers_declared);
1059         ctx->num_shader_buffers = util_last_bit(info->shader_buffers_declared);
1060
1061         ctx->num_samplers = util_last_bit(info->samplers_declared);
1062         ctx->num_images = util_last_bit(info->images_declared);
1063
1064         if (sel->nir)
1065                 return;
1066
1067         if (info->array_max[TGSI_FILE_TEMPORARY] > 0) {
1068                 int size = info->array_max[TGSI_FILE_TEMPORARY];
1069
1070                 ctx->temp_arrays = CALLOC(size, sizeof(ctx->temp_arrays[0]));
1071                 ctx->temp_array_allocas = CALLOC(size, sizeof(ctx->temp_array_allocas[0]));
1072
1073                 tgsi_scan_arrays(sel->tokens, TGSI_FILE_TEMPORARY, size,
1074                                  ctx->temp_arrays);
1075         }
1076         if (info->file_max[TGSI_FILE_IMMEDIATE] >= 0) {
1077                 int size = info->file_max[TGSI_FILE_IMMEDIATE] + 1;
1078                 ctx->imms = MALLOC(size * TGSI_NUM_CHANNELS * sizeof(LLVMValueRef));
1079         }
1080
1081         /* Re-set these to start with a clean slate. */
1082         ctx->bld_base.num_instructions = 0;
1083         ctx->bld_base.pc = 0;
1084         memset(ctx->outputs, 0, sizeof(ctx->outputs));
1085
1086         ctx->bld_base.emit_store = si_llvm_emit_store;
1087         ctx->bld_base.emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = si_llvm_emit_fetch;
1088         ctx->bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = si_llvm_emit_fetch;
1089         ctx->bld_base.emit_fetch_funcs[TGSI_FILE_TEMPORARY] = si_llvm_emit_fetch;
1090         ctx->bld_base.emit_fetch_funcs[TGSI_FILE_OUTPUT] = si_llvm_emit_fetch;
1091         ctx->bld_base.emit_fetch_funcs[TGSI_FILE_SYSTEM_VALUE] = fetch_system_value;
1092 }
1093
1094 void si_llvm_create_func(struct si_shader_context *ctx,
1095                          const char *name,
1096                          LLVMTypeRef *return_types, unsigned num_return_elems,
1097                          LLVMTypeRef *ParamTypes, unsigned ParamCount)
1098 {
1099         LLVMTypeRef main_fn_type, ret_type;
1100         LLVMBasicBlockRef main_fn_body;
1101         enum si_llvm_calling_convention call_conv;
1102         enum pipe_shader_type real_shader_type;
1103
1104         if (num_return_elems)
1105                 ret_type = LLVMStructTypeInContext(ctx->ac.context,
1106                                                    return_types,
1107                                                    num_return_elems, true);
1108         else
1109                 ret_type = ctx->voidt;
1110
1111         /* Setup the function */
1112         ctx->return_type = ret_type;
1113         main_fn_type = LLVMFunctionType(ret_type, ParamTypes, ParamCount, 0);
1114         ctx->main_fn = LLVMAddFunction(ctx->gallivm.module, name, main_fn_type);
1115         main_fn_body = LLVMAppendBasicBlockInContext(ctx->ac.context,
1116                         ctx->main_fn, "main_body");
1117         LLVMPositionBuilderAtEnd(ctx->ac.builder, main_fn_body);
1118
1119         real_shader_type = ctx->type;
1120
1121         /* LS is merged into HS (TCS), and ES is merged into GS. */
1122         if (ctx->screen->info.chip_class >= GFX9) {
1123                 if (ctx->shader->key.as_ls)
1124                         real_shader_type = PIPE_SHADER_TESS_CTRL;
1125                 else if (ctx->shader->key.as_es || ctx->shader->key.as_ngg)
1126                         real_shader_type = PIPE_SHADER_GEOMETRY;
1127         }
1128
1129         switch (real_shader_type) {
1130         case PIPE_SHADER_VERTEX:
1131         case PIPE_SHADER_TESS_EVAL:
1132                 call_conv = RADEON_LLVM_AMDGPU_VS;
1133                 break;
1134         case PIPE_SHADER_TESS_CTRL:
1135                 call_conv = RADEON_LLVM_AMDGPU_HS;
1136                 break;
1137         case PIPE_SHADER_GEOMETRY:
1138                 call_conv = RADEON_LLVM_AMDGPU_GS;
1139                 break;
1140         case PIPE_SHADER_FRAGMENT:
1141                 call_conv = RADEON_LLVM_AMDGPU_PS;
1142                 break;
1143         case PIPE_SHADER_COMPUTE:
1144                 call_conv = RADEON_LLVM_AMDGPU_CS;
1145                 break;
1146         default:
1147                 unreachable("Unhandle shader type");
1148         }
1149
1150         LLVMSetFunctionCallConv(ctx->main_fn, call_conv);
1151 }
1152
1153 void si_llvm_optimize_module(struct si_shader_context *ctx)
1154 {
1155         /* Dump LLVM IR before any optimization passes */
1156         if (ctx->screen->debug_flags & DBG(PREOPT_IR) &&
1157             si_can_dump_shader(ctx->screen, ctx->type))
1158                 LLVMDumpModule(ctx->gallivm.module);
1159
1160         /* Run the pass */
1161         LLVMRunPassManager(ctx->compiler->passmgr, ctx->gallivm.module);
1162         LLVMDisposeBuilder(ctx->ac.builder);
1163 }
1164
1165 void si_llvm_dispose(struct si_shader_context *ctx)
1166 {
1167         LLVMDisposeModule(ctx->gallivm.module);
1168         LLVMContextDispose(ctx->gallivm.context);
1169         FREE(ctx->temp_arrays);
1170         ctx->temp_arrays = NULL;
1171         FREE(ctx->temp_array_allocas);
1172         ctx->temp_array_allocas = NULL;
1173         FREE(ctx->temps);
1174         ctx->temps = NULL;
1175         ctx->temps_count = 0;
1176         FREE(ctx->imms);
1177         ctx->imms = NULL;
1178         ctx->imms_num = 0;
1179         ac_llvm_context_dispose(&ctx->ac);
1180 }