X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_shader_tgsi_setup.c;h=39abd4b18f63aa730da1ef09c7de037e94276cb7;hb=6c2fa01b9c22dc5f4b701a4a8e701c92b6ac182d;hp=c733f5a812160d3a96ce23b096e9f3bbbee2ab25;hpb=130e198c496382d5fb01b0a36423e0161421abf5;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/si_shader_tgsi_setup.c b/src/gallium/drivers/radeonsi/si_shader_tgsi_setup.c index c733f5a8121..39abd4b18f6 100644 --- a/src/gallium/drivers/radeonsi/si_shader_tgsi_setup.c +++ b/src/gallium/drivers/radeonsi/si_shader_tgsi_setup.c @@ -1,5 +1,6 @@ /* * Copyright 2016 Advanced Micro Devices, Inc. + * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -23,143 +24,17 @@ #include "si_shader_internal.h" #include "si_pipe.h" - -#include "gallivm/lp_bld_const.h" -#include "gallivm/lp_bld_gather.h" -#include "gallivm/lp_bld_flow.h" -#include "gallivm/lp_bld_init.h" -#include "gallivm/lp_bld_intr.h" -#include "gallivm/lp_bld_misc.h" -#include "gallivm/lp_bld_swizzle.h" -#include "tgsi/tgsi_info.h" -#include "tgsi/tgsi_parse.h" -#include "util/u_math.h" +#include "ac_llvm_util.h" #include "util/u_memory.h" -#include "util/u_debug.h" - -#include -#include -#include -#include - -/* Data for if/else/endif and bgnloop/endloop control flow structures. - */ -struct si_llvm_flow { - /* Loop exit or next part of if/else/endif. */ - LLVMBasicBlockRef next_block; - LLVMBasicBlockRef loop_entry_block; -}; - -#define CPU_STRING_LEN 30 -#define FS_STRING_LEN 30 -#define TRIPLE_STRING_LEN 7 - -/** - * Shader types for the LLVM backend. - */ -enum si_llvm_shader_type { - RADEON_LLVM_SHADER_PS = 0, - RADEON_LLVM_SHADER_VS = 1, - RADEON_LLVM_SHADER_GS = 2, - RADEON_LLVM_SHADER_CS = 3, -}; enum si_llvm_calling_convention { RADEON_LLVM_AMDGPU_VS = 87, RADEON_LLVM_AMDGPU_GS = 88, RADEON_LLVM_AMDGPU_PS = 89, RADEON_LLVM_AMDGPU_CS = 90, + RADEON_LLVM_AMDGPU_HS = 93, }; -void si_llvm_add_attribute(LLVMValueRef F, const char *name, int value) -{ - char str[16]; - - snprintf(str, sizeof(str), "%i", value); - LLVMAddTargetDependentFunctionAttr(F, name, str); -} - -/** - * Set the shader type we want to compile - * - * @param type shader type to set - */ -void si_llvm_shader_type(LLVMValueRef F, unsigned type) -{ - enum si_llvm_shader_type llvm_type; - enum si_llvm_calling_convention calling_conv; - - switch (type) { - case PIPE_SHADER_VERTEX: - case PIPE_SHADER_TESS_CTRL: - case PIPE_SHADER_TESS_EVAL: - llvm_type = RADEON_LLVM_SHADER_VS; - calling_conv = RADEON_LLVM_AMDGPU_VS; - break; - case PIPE_SHADER_GEOMETRY: - llvm_type = RADEON_LLVM_SHADER_GS; - calling_conv = RADEON_LLVM_AMDGPU_GS; - break; - case PIPE_SHADER_FRAGMENT: - llvm_type = RADEON_LLVM_SHADER_PS; - calling_conv = RADEON_LLVM_AMDGPU_PS; - break; - case PIPE_SHADER_COMPUTE: - llvm_type = RADEON_LLVM_SHADER_CS; - calling_conv = RADEON_LLVM_AMDGPU_CS; - break; - default: - unreachable("Unhandle shader type"); - } - - if (HAVE_LLVM >= 0x309) - LLVMSetFunctionCallConv(F, calling_conv); - else - si_llvm_add_attribute(F, "ShaderType", llvm_type); -} - -static void init_amdgpu_target() -{ - gallivm_init_llvm_targets(); - LLVMInitializeAMDGPUTargetInfo(); - LLVMInitializeAMDGPUTarget(); - LLVMInitializeAMDGPUTargetMC(); - LLVMInitializeAMDGPUAsmPrinter(); - - /* For inline assembly. */ - LLVMInitializeAMDGPUAsmParser(); - - if (HAVE_LLVM >= 0x0400) { - /* - * Workaround for bug in llvm 4.0 that causes image intrinsics - * to disappear. - * https://reviews.llvm.org/D26348 - */ - const char *argv[2] = {"mesa", "-simplifycfg-sink-common=false"}; - LLVMParseCommandLineOptions(2, argv, NULL); - } -} - -static once_flag init_amdgpu_target_once_flag = ONCE_FLAG_INIT; - -LLVMTargetRef si_llvm_get_amdgpu_target(const char *triple) -{ - LLVMTargetRef target = NULL; - char *err_message = NULL; - - call_once(&init_amdgpu_target_once_flag, init_amdgpu_target); - - if (LLVMGetTargetFromTriple(triple, &target, &err_message)) { - fprintf(stderr, "Cannot find target for triple %s ", triple); - if (err_message) { - fprintf(stderr, "%s\n", err_message); - } - LLVMDisposeMessage(err_message); - return NULL; - } - return target; -} - struct si_llvm_diagnostics { struct pipe_debug_callback *debug; unsigned retval; @@ -205,17 +80,20 @@ static void si_diagnostic_handler(LLVMDiagnosticInfoRef di, void *context) * * @returns 0 for success, 1 for failure */ -unsigned si_llvm_compile(LLVMModuleRef M, struct ac_shader_binary *binary, - LLVMTargetMachineRef tm, - struct pipe_debug_callback *debug) +unsigned si_llvm_compile(LLVMModuleRef M, struct si_shader_binary *binary, + struct ac_llvm_compiler *compiler, + struct pipe_debug_callback *debug, + bool less_optimized, unsigned wave_size) { + struct ac_compiler_passes *passes = compiler->passes; + + if (wave_size == 32) + passes = compiler->passes_wave32; + else if (less_optimized && compiler->low_opt_passes) + passes = compiler->low_opt_passes; + struct si_llvm_diagnostics diag; - char *err; LLVMContextRef llvm_ctx; - LLVMMemoryBufferRef out_buffer; - unsigned buffer_size; - const char *buffer_data; - LLVMBool mem_err; diag.debug = debug; diag.retval = 0; @@ -225,52 +103,42 @@ unsigned si_llvm_compile(LLVMModuleRef M, struct ac_shader_binary *binary, LLVMContextSetDiagnosticHandler(llvm_ctx, si_diagnostic_handler, &diag); - /* Compile IR*/ - mem_err = LLVMTargetMachineEmitToMemoryBuffer(tm, M, LLVMObjectFile, &err, - &out_buffer); - - /* Process Errors/Warnings */ - if (mem_err) { - fprintf(stderr, "%s: %s", __FUNCTION__, err); - pipe_debug_message(debug, SHADER_INFO, - "LLVM emit error: %s", err); - FREE(err); + /* Compile IR. */ + if (!ac_compile_module_to_elf(passes, M, (char **)&binary->elf_buffer, + &binary->elf_size)) diag.retval = 1; - goto out; - } - /* Extract Shader Code*/ - buffer_size = LLVMGetBufferSize(out_buffer); - buffer_data = LLVMGetBufferStart(out_buffer); - - ac_elf_read(buffer_data, buffer_size, binary); - - /* Clean up */ - LLVMDisposeMemoryBuffer(out_buffer); - -out: if (diag.retval != 0) pipe_debug_message(debug, SHADER_INFO, "LLVM compile failed"); return diag.retval; } +void si_shader_binary_clean(struct si_shader_binary *binary) +{ + free((void *)binary->elf_buffer); + binary->elf_buffer = NULL; + + free(binary->llvm_ir_string); + binary->llvm_ir_string = NULL; +} + LLVMTypeRef tgsi2llvmtype(struct lp_build_tgsi_context *bld_base, enum tgsi_opcode_type type) { - LLVMContextRef ctx = bld_base->base.gallivm->context; + struct si_shader_context *ctx = si_shader_context(bld_base); switch (type) { case TGSI_TYPE_UNSIGNED: case TGSI_TYPE_SIGNED: - return LLVMInt32TypeInContext(ctx); + return ctx->ac.i32; case TGSI_TYPE_UNSIGNED64: case TGSI_TYPE_SIGNED64: - return LLVMInt64TypeInContext(ctx); + return ctx->ac.i64; case TGSI_TYPE_DOUBLE: - return LLVMDoubleTypeInContext(ctx); + return ctx->ac.f64; case TGSI_TYPE_UNTYPED: case TGSI_TYPE_FLOAT: - return LLVMFloatTypeInContext(ctx); + return ctx->ac.f32; default: break; } return 0; @@ -279,11 +147,11 @@ LLVMTypeRef tgsi2llvmtype(struct lp_build_tgsi_context *bld_base, LLVMValueRef bitcast(struct lp_build_tgsi_context *bld_base, enum tgsi_opcode_type type, LLVMValueRef value) { - LLVMBuilderRef builder = bld_base->base.gallivm->builder; + struct si_shader_context *ctx = si_shader_context(bld_base); LLVMTypeRef dst_type = tgsi2llvmtype(bld_base, type); if (dst_type) - return LLVMBuildBitCast(builder, value, dst_type, ""); + return LLVMBuildBitCast(ctx->ac.builder, value, dst_type, ""); else return value; } @@ -296,12 +164,11 @@ LLVMValueRef si_llvm_bound_index(struct si_shader_context *ctx, LLVMValueRef index, unsigned num) { - struct gallivm_state *gallivm = &ctx->gallivm; - LLVMBuilderRef builder = gallivm->builder; + LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef c_max = LLVMConstInt(ctx->i32, num - 1, 0); LLVMValueRef cc; - if (util_is_power_of_two(num)) { + if (util_is_power_of_two_or_zero(num)) { index = LLVMBuildAnd(builder, index, c_max, ""); } else { /* In theory, this MAX pattern should result in code that is @@ -317,45 +184,6 @@ LLVMValueRef si_llvm_bound_index(struct si_shader_context *ctx, return index; } -static struct si_llvm_flow * -get_current_flow(struct si_shader_context *ctx) -{ - if (ctx->flow_depth > 0) - return &ctx->flow[ctx->flow_depth - 1]; - return NULL; -} - -static struct si_llvm_flow * -get_innermost_loop(struct si_shader_context *ctx) -{ - for (unsigned i = ctx->flow_depth; i > 0; --i) { - if (ctx->flow[i - 1].loop_entry_block) - return &ctx->flow[i - 1]; - } - return NULL; -} - -static struct si_llvm_flow * -push_flow(struct si_shader_context *ctx) -{ - struct si_llvm_flow *flow; - - if (ctx->flow_depth >= ctx->flow_depth_max) { - unsigned new_max = MAX2(ctx->flow_depth << 1, RADEON_LLVM_INITIAL_CF_DEPTH); - ctx->flow = REALLOC(ctx->flow, - ctx->flow_depth_max * sizeof(*ctx->flow), - new_max * sizeof(*ctx->flow)); - ctx->flow_depth_max = new_max; - } - - flow = &ctx->flow[ctx->flow_depth]; - ctx->flow_depth++; - - flow->next_block = NULL; - flow->loop_entry_block = NULL; - return flow; -} - static LLVMValueRef emit_swizzle(struct lp_build_tgsi_context *bld_base, LLVMValueRef value, unsigned swizzle_x, @@ -363,16 +191,15 @@ static LLVMValueRef emit_swizzle(struct lp_build_tgsi_context *bld_base, unsigned swizzle_z, unsigned swizzle_w) { + struct si_shader_context *ctx = si_shader_context(bld_base); LLVMValueRef swizzles[4]; - LLVMTypeRef i32t = - LLVMInt32TypeInContext(bld_base->base.gallivm->context); - swizzles[0] = LLVMConstInt(i32t, swizzle_x, 0); - swizzles[1] = LLVMConstInt(i32t, swizzle_y, 0); - swizzles[2] = LLVMConstInt(i32t, swizzle_z, 0); - swizzles[3] = LLVMConstInt(i32t, swizzle_w, 0); + swizzles[0] = LLVMConstInt(ctx->i32, swizzle_x, 0); + swizzles[1] = LLVMConstInt(ctx->i32, swizzle_y, 0); + swizzles[2] = LLVMConstInt(ctx->i32, swizzle_z, 0); + swizzles[3] = LLVMConstInt(ctx->i32, swizzle_w, 0); - return LLVMBuildShuffleVector(bld_base->base.gallivm->builder, + return LLVMBuildShuffleVector(ctx->ac.builder, value, LLVMGetUndef(LLVMTypeOf(value)), LLVMConstVector(swizzles, 4), ""); @@ -423,20 +250,6 @@ get_array_range(struct lp_build_tgsi_context *bld_base, return range; } -static LLVMValueRef -emit_array_index(struct si_shader_context *ctx, - const struct tgsi_ind_register *reg, - unsigned offset) -{ - struct gallivm_state *gallivm = &ctx->gallivm; - - if (!reg) { - return LLVMConstInt(ctx->i32, offset, 0); - } - LLVMValueRef addr = LLVMBuildLoad(gallivm->builder, ctx->addrs[reg->Index][reg->Swizzle], ""); - return LLVMBuildAdd(gallivm->builder, addr, LLVMConstInt(ctx->i32, offset, 0), ""); -} - /** * For indirect registers, construct a pointer directly to the requested * element using getelementptr if possible. @@ -453,8 +266,6 @@ get_pointer_into_array(struct si_shader_context *ctx, { unsigned array_id; struct tgsi_array_info *array; - struct gallivm_state *gallivm = &ctx->gallivm; - LLVMBuilderRef builder = gallivm->builder; LLVMValueRef idxs[2]; LLVMValueRef index; LLVMValueRef alloca; @@ -475,8 +286,8 @@ get_pointer_into_array(struct si_shader_context *ctx, if (!(array->writemask & (1 << swizzle))) return ctx->undef_alloca; - index = emit_array_index(ctx, reg_indirect, - reg_index - ctx->temp_arrays[array_id - 1].range.First); + index = si_get_indirect_index(ctx, reg_indirect, 1, + reg_index - ctx->temp_arrays[array_id - 1].range.First); /* Ensure that the index is within a valid range, to guard against * VM faults and overwriting critical data (e.g. spilled resource @@ -492,63 +303,51 @@ get_pointer_into_array(struct si_shader_context *ctx, */ index = si_llvm_bound_index(ctx, index, array->range.Last - array->range.First + 1); - index = LLVMBuildMul( - builder, index, - LLVMConstInt(ctx->i32, util_bitcount(array->writemask), 0), - ""); - index = LLVMBuildAdd( - builder, index, - LLVMConstInt(ctx->i32, - util_bitcount(array->writemask & ((1 << swizzle) - 1)), 0), - ""); + index = ac_build_imad(&ctx->ac, index, + LLVMConstInt(ctx->i32, util_bitcount(array->writemask), 0), + LLVMConstInt(ctx->i32, + util_bitcount(array->writemask & ((1 << swizzle) - 1)), 0)); idxs[0] = ctx->i32_0; idxs[1] = index; - return LLVMBuildGEP(builder, alloca, idxs, 2, ""); + return LLVMBuildGEP(ctx->ac.builder, alloca, idxs, 2, ""); } LLVMValueRef si_llvm_emit_fetch_64bit(struct lp_build_tgsi_context *bld_base, - enum tgsi_opcode_type type, + LLVMTypeRef type, LLVMValueRef ptr, LLVMValueRef ptr2) { - LLVMBuilderRef builder = bld_base->base.gallivm->builder; - LLVMValueRef result; - - result = LLVMGetUndef(LLVMVectorType(LLVMIntTypeInContext(bld_base->base.gallivm->context, 32), bld_base->base.type.length * 2)); - - result = LLVMBuildInsertElement(builder, - result, - bitcast(bld_base, TGSI_TYPE_UNSIGNED, ptr), - bld_base->int_bld.zero, ""); - result = LLVMBuildInsertElement(builder, - result, - bitcast(bld_base, TGSI_TYPE_UNSIGNED, ptr2), - bld_base->int_bld.one, ""); - return bitcast(bld_base, type, result); + struct si_shader_context *ctx = si_shader_context(bld_base); + LLVMValueRef values[2] = { + ac_to_integer(&ctx->ac, ptr), + ac_to_integer(&ctx->ac, ptr2), + }; + LLVMValueRef result = ac_build_gather_values(&ctx->ac, values, 2); + return LLVMBuildBitCast(ctx->ac.builder, result, type, ""); } static LLVMValueRef emit_array_fetch(struct lp_build_tgsi_context *bld_base, unsigned File, enum tgsi_opcode_type type, struct tgsi_declaration_range range, - unsigned swizzle) + unsigned swizzle_in) { struct si_shader_context *ctx = si_shader_context(bld_base); - - LLVMBuilderRef builder = ctx->gallivm.builder; - unsigned i, size = range.Last - range.First + 1; LLVMTypeRef vec = LLVMVectorType(tgsi2llvmtype(bld_base, type), size); LLVMValueRef result = LLVMGetUndef(vec); - + unsigned swizzle = swizzle_in; struct tgsi_full_src_register tmp_reg = {}; tmp_reg.Register.File = File; + if (tgsi_type_is_64bit(type)) + swizzle |= (swizzle_in + 1) << 16; for (i = 0; i < size; ++i) { tmp_reg.Register.Index = i + range.First; + LLVMValueRef temp = si_llvm_emit_fetch(bld_base, &tmp_reg, type, swizzle); - result = LLVMBuildInsertElement(builder, result, temp, + result = LLVMBuildInsertElement(ctx->ac.builder, result, temp, LLVMConstInt(ctx->i32, i, 0), "array_vector"); } return result; @@ -563,8 +362,7 @@ load_value_from_array(struct lp_build_tgsi_context *bld_base, const struct tgsi_ind_register *reg_indirect) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - LLVMBuilderRef builder = gallivm->builder; + LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef ptr; ptr = get_pointer_into_array(ctx, file, swizzle, reg_index, reg_indirect); @@ -574,7 +372,8 @@ load_value_from_array(struct lp_build_tgsi_context *bld_base, LLVMValueRef ptr_hi, val_hi; ptr_hi = LLVMBuildGEP(builder, ptr, &ctx->i32_1, 1, ""); val_hi = LLVMBuildLoad(builder, ptr_hi, ""); - val = si_llvm_emit_fetch_64bit(bld_base, type, val, val_hi); + val = si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type), + val, val_hi); } return val; @@ -582,7 +381,7 @@ load_value_from_array(struct lp_build_tgsi_context *bld_base, struct tgsi_declaration_range range = get_array_range(bld_base, file, reg_index, reg_indirect); LLVMValueRef index = - emit_array_index(ctx, reg_indirect, reg_index - range.First); + si_get_indirect_index(ctx, reg_indirect, 1, reg_index - range.First); LLVMValueRef array = emit_array_fetch(bld_base, file, type, range, swizzle); return LLVMBuildExtractElement(builder, array, index, ""); @@ -598,8 +397,7 @@ store_value_to_array(struct lp_build_tgsi_context *bld_base, const struct tgsi_ind_register *reg_indirect) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - LLVMBuilderRef builder = gallivm->builder; + LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef ptr; ptr = get_pointer_into_array(ctx, file, chan_index, reg_index, reg_indirect); @@ -608,7 +406,7 @@ store_value_to_array(struct lp_build_tgsi_context *bld_base, } else { unsigned i, size; struct tgsi_declaration_range range = get_array_range(bld_base, file, reg_index, reg_indirect); - LLVMValueRef index = emit_array_index(ctx, reg_indirect, reg_index - range.First); + LLVMValueRef index = si_get_indirect_index(ctx, reg_indirect, 1, reg_index - range.First); LLVMValueRef array = emit_array_fetch(bld_base, file, TGSI_TYPE_FLOAT, range, chan_index); LLVMValueRef temp_ptr; @@ -664,19 +462,20 @@ get_output_ptr(struct lp_build_tgsi_context *bld_base, unsigned index, LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base, const struct tgsi_full_src_register *reg, enum tgsi_opcode_type type, - unsigned swizzle) + unsigned swizzle_in) { struct si_shader_context *ctx = si_shader_context(bld_base); - LLVMBuilderRef builder = ctx->gallivm.builder; + LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef result = NULL, ptr, ptr2; + unsigned swizzle = swizzle_in & 0xffff; - if (swizzle == ~0) { + if (swizzle_in == ~0) { LLVMValueRef values[TGSI_NUM_CHANNELS]; unsigned chan; for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { values[chan] = si_llvm_emit_fetch(bld_base, reg, type, chan); } - return lp_build_gather_values(&ctx->gallivm, values, + return ac_build_gather_values(&ctx->ac, values, TGSI_NUM_CHANNELS); } @@ -690,12 +489,12 @@ LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base, case TGSI_FILE_IMMEDIATE: { LLVMTypeRef ctype = tgsi2llvmtype(bld_base, type); if (tgsi_type_is_64bit(type)) { - result = LLVMGetUndef(LLVMVectorType(ctx->i32, bld_base->base.type.length * 2)); + result = LLVMGetUndef(LLVMVectorType(ctx->i32, 2)); result = LLVMConstInsertElement(result, ctx->imms[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle], ctx->i32_0); result = LLVMConstInsertElement(result, - ctx->imms[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle + 1], + ctx->imms[reg->Register.Index * TGSI_NUM_CHANNELS + (swizzle_in >> 16)], ctx->i32_1); return LLVMConstBitCast(result, ctype); } else { @@ -722,8 +521,9 @@ LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base, if (tgsi_type_is_64bit(type)) { ptr = result; - ptr2 = input[swizzle + 1]; - return si_llvm_emit_fetch_64bit(bld_base, type, ptr, ptr2); + ptr2 = input[swizzle_in >> 16]; + return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type), + ptr, ptr2); } break; } @@ -733,8 +533,8 @@ LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base, return LLVMGetUndef(tgsi2llvmtype(bld_base, type)); ptr = ctx->temps[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle]; if (tgsi_type_is_64bit(type)) { - ptr2 = ctx->temps[reg->Register.Index * TGSI_NUM_CHANNELS + swizzle + 1]; - return si_llvm_emit_fetch_64bit(bld_base, type, + ptr2 = ctx->temps[reg->Register.Index * TGSI_NUM_CHANNELS + (swizzle_in >> 16)]; + return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type), LLVMBuildLoad(builder, ptr, ""), LLVMBuildLoad(builder, ptr2, "")); } @@ -744,8 +544,8 @@ LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base, case TGSI_FILE_OUTPUT: ptr = get_output_ptr(bld_base, reg->Register.Index, swizzle); if (tgsi_type_is_64bit(type)) { - ptr2 = get_output_ptr(bld_base, reg->Register.Index, swizzle + 1); - return si_llvm_emit_fetch_64bit(bld_base, type, + ptr2 = get_output_ptr(bld_base, reg->Register.Index, (swizzle_in >> 16)); + return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type), LLVMBuildLoad(builder, ptr, ""), LLVMBuildLoad(builder, ptr2, "")); } @@ -762,11 +562,12 @@ LLVMValueRef si_llvm_emit_fetch(struct lp_build_tgsi_context *bld_base, static LLVMValueRef fetch_system_value(struct lp_build_tgsi_context *bld_base, const struct tgsi_full_src_register *reg, enum tgsi_opcode_type type, - unsigned swizzle) + unsigned swizzle_in) { struct si_shader_context *ctx = si_shader_context(bld_base); - LLVMBuilderRef builder = ctx->gallivm.builder; + LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef cval = ctx->system_values[reg->Register.Index]; + unsigned swizzle = swizzle_in & 0xffff; if (tgsi_type_is_64bit(type)) { LLVMValueRef lo, hi; @@ -776,9 +577,10 @@ static LLVMValueRef fetch_system_value(struct lp_build_tgsi_context *bld_base, lo = LLVMBuildExtractElement( builder, cval, LLVMConstInt(ctx->i32, swizzle, 0), ""); hi = LLVMBuildExtractElement( - builder, cval, LLVMConstInt(ctx->i32, swizzle + 1, 0), ""); + builder, cval, LLVMConstInt(ctx->i32, (swizzle_in >> 16), 0), ""); - return si_llvm_emit_fetch_64bit(bld_base, type, lo, hi); + return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type), + lo, hi); } if (LLVMGetTypeKind(LLVMTypeOf(cval)) == LLVMVectorTypeKind) { @@ -795,7 +597,7 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, const struct tgsi_full_declaration *decl) { struct si_shader_context *ctx = si_shader_context(bld_base); - LLVMBuilderRef builder = ctx->gallivm.builder; + LLVMBuilderRef builder = ctx->ac.builder; unsigned first, last, i; switch(decl->Declaration.File) { case TGSI_FILE_ADDRESS: @@ -804,9 +606,8 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) { unsigned chan; for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - ctx->addrs[idx][chan] = lp_build_alloca_undef( - &ctx->gallivm, - ctx->i32, ""); + ctx->addrs[idx][chan] = ac_build_alloca_undef( + &ctx->ac, ctx->i32, ""); } } break; @@ -814,7 +615,7 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, case TGSI_FILE_TEMPORARY: { - char name[16] = ""; + char name[18] = ""; LLVMValueRef array_alloca = NULL; unsigned decl_size; unsigned writemask = decl->Declaration.UsageMask; @@ -848,13 +649,10 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, * FIXME: We shouldn't need to have the non-alloca * code path for arrays. LLVM should be smart enough to * promote allocas into registers when profitable. - * - * LLVM 3.8 crashes with this. */ - if ((HAVE_LLVM >= 0x0309 && array_size > 16) || - /* TODO: VGPR indexing is buggy on GFX9. */ - ctx->screen->b.chip_class == GFX9) { - array_alloca = LLVMBuildAlloca(builder, + if (array_size > 16 || + !ctx->screen->llvm_has_working_vgpr_indexing) { + array_alloca = ac_build_alloca_undef(&ctx->ac, LLVMArrayType(ctx->f32, array_size), "array"); ctx->temp_array_allocas[id] = array_alloca; @@ -867,12 +665,12 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, } if (!array_alloca) { for (i = 0; i < decl_size; ++i) { -#ifdef DEBUG +#ifndef NDEBUG snprintf(name, sizeof(name), "TEMP%d.%c", first + i / 4, "xyzw"[i % 4]); #endif ctx->temps[first * TGSI_NUM_CHANNELS + i] = - lp_build_alloca_undef(&ctx->gallivm, + ac_build_alloca_undef(&ctx->ac, ctx->f32, name); } @@ -890,15 +688,14 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, * a shader ever reads from a channel that * it never writes to. */ - ctx->undef_alloca = lp_build_alloca_undef( - &ctx->gallivm, - ctx->f32, "undef"); + ctx->undef_alloca = ac_build_alloca_undef( + &ctx->ac, ctx->f32, "undef"); } for (i = 0; i < decl_size; ++i) { LLVMValueRef ptr; if (writemask & (1 << (i % 4))) { -#ifdef DEBUG +#ifndef NDEBUG snprintf(name, sizeof(name), "TEMP%d.%c", first + i / 4, "xyzw"[i % 4]); #endif @@ -937,7 +734,7 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, { unsigned idx; for (idx = decl->Range.First; idx <= decl->Range.Last; idx++) { - ctx->load_system_value(ctx, idx, decl); + si_load_system_value(ctx, idx, decl); } } break; @@ -952,20 +749,19 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, if (ctx->outputs[idx][0]) continue; for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { -#ifdef DEBUG +#ifndef NDEBUG snprintf(name, sizeof(name), "OUT%d.%c", idx, "xyzw"[chan % 4]); #endif - ctx->outputs[idx][chan] = lp_build_alloca_undef( - &ctx->gallivm, - ctx->f32, name); + ctx->outputs[idx][chan] = ac_build_alloca_undef( + &ctx->ac, ctx->f32, name); } } break; } case TGSI_FILE_MEMORY: - ctx->declare_memory_region(ctx, decl); + si_tgsi_declare_compute_memory(ctx, decl); break; default: @@ -976,16 +772,15 @@ static void emit_declaration(struct lp_build_tgsi_context *bld_base, void si_llvm_emit_store(struct lp_build_tgsi_context *bld_base, const struct tgsi_full_instruction *inst, const struct tgsi_opcode_info *info, + unsigned index, LLVMValueRef dst[4]) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - const struct tgsi_full_dst_register *reg = &inst->Dst[0]; - LLVMBuilderRef builder = ctx->gallivm.builder; + const struct tgsi_full_dst_register *reg = &inst->Dst[index]; + LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef temp_ptr, temp_ptr2 = NULL; - unsigned chan, chan_index; bool is_vec_store = false; - enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode); + enum tgsi_opcode_type dtype = tgsi_opcode_infer_dst_type(inst->Instruction.Opcode, index); if (dst[0]) { LLVMTypeKind k = LLVMGetTypeKind(LLVMTypeOf(dst[0])); @@ -994,16 +789,20 @@ void si_llvm_emit_store(struct lp_build_tgsi_context *bld_base, if (is_vec_store) { LLVMValueRef values[4] = {}; - TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan) { + uint32_t writemask = reg->Register.WriteMask; + while (writemask) { + unsigned chan = u_bit_scan(&writemask); LLVMValueRef index = LLVMConstInt(ctx->i32, chan, 0); - values[chan] = LLVMBuildExtractElement(gallivm->builder, + values[chan] = LLVMBuildExtractElement(ctx->ac.builder, dst[0], index, ""); } - bld_base->emit_store(bld_base, inst, info, values); + bld_base->emit_store(bld_base, inst, info, index, values); return; } - TGSI_FOR_EACH_DST0_ENABLED_CHANNEL( inst, chan_index ) { + uint32_t writemask = reg->Register.WriteMask; + while (writemask) { + unsigned chan_index = u_bit_scan(&writemask); LLVMValueRef value = dst[chan_index]; if (tgsi_type_is_64bit(dtype) && (chan_index == 1 || chan_index == 3)) @@ -1018,7 +817,7 @@ void si_llvm_emit_store(struct lp_build_tgsi_context *bld_base, } if (!tgsi_type_is_64bit(dtype)) - value = bitcast(bld_base, TGSI_TYPE_FLOAT, value); + value = ac_to_float(&ctx->ac, value); if (reg->Register.Indirect) { unsigned file = reg->Register.File; @@ -1058,51 +857,20 @@ void si_llvm_emit_store(struct lp_build_tgsi_context *bld_base, val2 = LLVMBuildExtractElement(builder, ptr, ctx->i32_1, ""); - LLVMBuildStore(builder, bitcast(bld_base, TGSI_TYPE_FLOAT, value), temp_ptr); - LLVMBuildStore(builder, bitcast(bld_base, TGSI_TYPE_FLOAT, val2), temp_ptr2); + LLVMBuildStore(builder, ac_to_float(&ctx->ac, value), temp_ptr); + LLVMBuildStore(builder, ac_to_float(&ctx->ac, val2), temp_ptr2); } } } } -static void set_basicblock_name(LLVMBasicBlockRef bb, const char *base, int pc) +static int get_line(int pc) { - char buf[32]; /* Subtract 1 so that the number shown is that of the corresponding * opcode in the TGSI dump, e.g. an if block has the same suffix as * the instruction number of the corresponding TGSI IF. */ - snprintf(buf, sizeof(buf), "%s%d", base, pc - 1); - LLVMSetValueName(LLVMBasicBlockAsValue(bb), buf); -} - -/* Append a basic block at the level of the parent flow. - */ -static LLVMBasicBlockRef append_basic_block(struct si_shader_context *ctx, - const char *name) -{ - struct gallivm_state *gallivm = &ctx->gallivm; - - assert(ctx->flow_depth >= 1); - - if (ctx->flow_depth >= 2) { - struct si_llvm_flow *flow = &ctx->flow[ctx->flow_depth - 2]; - - return LLVMInsertBasicBlockInContext(gallivm->context, - flow->next_block, name); - } - - return LLVMAppendBasicBlockInContext(gallivm->context, ctx->main_fn, name); -} - -/* Emit a branch to the given default target for the current block if - * applicable -- that is, if the current block does not already contain a - * branch from a break or continue. - */ -static void emit_default_branch(LLVMBuilderRef builder, LLVMBasicBlockRef target) -{ - if (!LLVMGetBasicBlockTerminator(LLVMGetInsertBlock(builder))) - LLVMBuildBr(builder, target); + return pc - 1; } static void bgnloop_emit(const struct lp_build_tgsi_action *action, @@ -1110,13 +878,7 @@ static void bgnloop_emit(const struct lp_build_tgsi_action *action, struct lp_build_emit_data *emit_data) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *flow = push_flow(ctx); - flow->loop_entry_block = append_basic_block(ctx, "LOOP"); - flow->next_block = append_basic_block(ctx, "ENDLOOP"); - set_basicblock_name(flow->loop_entry_block, "loop", bld_base->pc); - LLVMBuildBr(gallivm->builder, flow->loop_entry_block); - LLVMPositionBuilderAtEnd(gallivm->builder, flow->loop_entry_block); + ac_build_bgnloop(&ctx->ac, get_line(bld_base->pc)); } static void brk_emit(const struct lp_build_tgsi_action *action, @@ -1124,10 +886,7 @@ static void brk_emit(const struct lp_build_tgsi_action *action, struct lp_build_emit_data *emit_data) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *flow = get_innermost_loop(ctx); - - LLVMBuildBr(gallivm->builder, flow->next_block); + ac_build_break(&ctx->ac); } static void cont_emit(const struct lp_build_tgsi_action *action, @@ -1135,10 +894,7 @@ static void cont_emit(const struct lp_build_tgsi_action *action, struct lp_build_emit_data *emit_data) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *flow = get_innermost_loop(ctx); - - LLVMBuildBr(gallivm->builder, flow->loop_entry_block); + ac_build_continue(&ctx->ac); } static void else_emit(const struct lp_build_tgsi_action *action, @@ -1146,19 +902,7 @@ static void else_emit(const struct lp_build_tgsi_action *action, struct lp_build_emit_data *emit_data) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *current_branch = get_current_flow(ctx); - LLVMBasicBlockRef endif_block; - - assert(!current_branch->loop_entry_block); - - endif_block = append_basic_block(ctx, "ENDIF"); - emit_default_branch(gallivm->builder, endif_block); - - LLVMPositionBuilderAtEnd(gallivm->builder, current_branch->next_block); - set_basicblock_name(current_branch->next_block, "else", bld_base->pc); - - current_branch->next_block = endif_block; + ac_build_else(&ctx->ac, get_line(bld_base->pc)); } static void endif_emit(const struct lp_build_tgsi_action *action, @@ -1166,16 +910,7 @@ static void endif_emit(const struct lp_build_tgsi_action *action, struct lp_build_emit_data *emit_data) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *current_branch = get_current_flow(ctx); - - assert(!current_branch->loop_entry_block); - - emit_default_branch(gallivm->builder, current_branch->next_block); - LLVMPositionBuilderAtEnd(gallivm->builder, current_branch->next_block); - set_basicblock_name(current_branch->next_block, "endif", bld_base->pc); - - ctx->flow_depth--; + ac_build_endif(&ctx->ac, get_line(bld_base->pc)); } static void endloop_emit(const struct lp_build_tgsi_action *action, @@ -1183,61 +918,23 @@ static void endloop_emit(const struct lp_build_tgsi_action *action, struct lp_build_emit_data *emit_data) { struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *current_loop = get_current_flow(ctx); - - assert(current_loop->loop_entry_block); - - emit_default_branch(gallivm->builder, current_loop->loop_entry_block); - - LLVMPositionBuilderAtEnd(gallivm->builder, current_loop->next_block); - set_basicblock_name(current_loop->next_block, "endloop", bld_base->pc); - ctx->flow_depth--; -} - -static void if_cond_emit(const struct lp_build_tgsi_action *action, - struct lp_build_tgsi_context *bld_base, - struct lp_build_emit_data *emit_data, - LLVMValueRef cond) -{ - struct si_shader_context *ctx = si_shader_context(bld_base); - struct gallivm_state *gallivm = &ctx->gallivm; - struct si_llvm_flow *flow = push_flow(ctx); - LLVMBasicBlockRef if_block; - - if_block = append_basic_block(ctx, "IF"); - flow->next_block = append_basic_block(ctx, "ELSE"); - set_basicblock_name(if_block, "if", bld_base->pc); - LLVMBuildCondBr(gallivm->builder, cond, if_block, flow->next_block); - LLVMPositionBuilderAtEnd(gallivm->builder, if_block); + ac_build_endloop(&ctx->ac, get_line(bld_base->pc)); } static void if_emit(const struct lp_build_tgsi_action *action, struct lp_build_tgsi_context *bld_base, struct lp_build_emit_data *emit_data) { - struct gallivm_state *gallivm = bld_base->base.gallivm; - LLVMValueRef cond; - - cond = LLVMBuildFCmp(gallivm->builder, LLVMRealUNE, - emit_data->args[0], - bld_base->base.zero, ""); - - if_cond_emit(action, bld_base, emit_data, cond); + struct si_shader_context *ctx = si_shader_context(bld_base); + ac_build_if(&ctx->ac, emit_data->args[0], get_line(bld_base->pc)); } static void uif_emit(const struct lp_build_tgsi_action *action, struct lp_build_tgsi_context *bld_base, struct lp_build_emit_data *emit_data) { - struct gallivm_state *gallivm = bld_base->base.gallivm; - LLVMValueRef cond; - - cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, - bitcast(bld_base, TGSI_TYPE_UNSIGNED, emit_data->args[0]), - bld_base->int_bld.zero, ""); - - if_cond_emit(action, bld_base, emit_data, cond); + struct si_shader_context *ctx = si_shader_context(bld_base); + ac_build_uif(&ctx->ac, emit_data->args[0], get_line(bld_base->pc)); } static void emit_immediate(struct lp_build_tgsi_context *bld_base, @@ -1256,7 +953,9 @@ static void emit_immediate(struct lp_build_tgsi_context *bld_base, void si_llvm_context_init(struct si_shader_context *ctx, struct si_screen *sscreen, - LLVMTargetMachineRef tm) + struct ac_llvm_compiler *compiler, + unsigned wave_size, + unsigned ballot_mask_bits) { struct lp_type type; @@ -1267,32 +966,16 @@ void si_llvm_context_init(struct si_shader_context *ctx, */ memset(ctx, 0, sizeof(*ctx)); ctx->screen = sscreen; - ctx->tm = tm; - - ctx->gallivm.context = LLVMContextCreate(); - ctx->gallivm.module = LLVMModuleCreateWithNameInContext("tgsi", - ctx->gallivm.context); - LLVMSetTarget(ctx->gallivm.module, "amdgcn--"); - -#if HAVE_LLVM >= 0x0309 - LLVMTargetDataRef data_layout = LLVMCreateTargetDataLayout(tm); - char *data_layout_str = LLVMCopyStringRepOfTargetData(data_layout); - LLVMSetDataLayout(ctx->gallivm.module, data_layout_str); - LLVMDisposeTargetData(data_layout); - LLVMDisposeMessage(data_layout_str); -#endif + ctx->compiler = compiler; - bool unsafe_fpmath = (sscreen->b.debug_flags & DBG_UNSAFE_MATH) != 0; - enum lp_float_mode float_mode = - unsafe_fpmath ? LP_FLOAT_MODE_UNSAFE_FP_MATH : - LP_FLOAT_MODE_NO_SIGNED_ZEROS_FP_MATH; + ac_llvm_context_init(&ctx->ac, compiler, sscreen->info.chip_class, + sscreen->info.family, + AC_FLOAT_MODE_NO_SIGNED_ZEROS_FP_MATH, + wave_size, ballot_mask_bits); - ctx->gallivm.builder = lp_create_builder(ctx->gallivm.context, - float_mode); - - ac_llvm_context_init(&ctx->ac, ctx->gallivm.context); - ctx->ac.module = ctx->gallivm.module; - ctx->ac.builder = ctx->gallivm.builder; + ctx->gallivm.context = ctx->ac.context; + ctx->gallivm.module = ctx->ac.module; + ctx->gallivm.builder = ctx->ac.builder; struct lp_build_tgsi_context *bld_base = &ctx->bld_base; @@ -1316,13 +999,6 @@ void si_llvm_context_init(struct si_shader_context *ctx, bld_base->emit_declaration = emit_declaration; bld_base->emit_immediate = emit_immediate; - /* metadata allowing 2.5 ULP */ - ctx->fpmath_md_kind = LLVMGetMDKindIDInContext(ctx->gallivm.context, - "fpmath", 6); - LLVMValueRef arg = lp_build_const_float(&ctx->gallivm, 2.5); - ctx->fpmath_md_2p5_ulp = LLVMMDNodeInContext(ctx->gallivm.context, - &arg, 1); - bld_base->op_actions[TGSI_OPCODE_BGNLOOP].emit = bgnloop_emit; bld_base->op_actions[TGSI_OPCODE_BRK].emit = brk_emit; bld_base->op_actions[TGSI_OPCODE_CONT].emit = cont_emit; @@ -1333,15 +1009,15 @@ void si_llvm_context_init(struct si_shader_context *ctx, bld_base->op_actions[TGSI_OPCODE_ENDLOOP].emit = endloop_emit; si_shader_context_init_alu(&ctx->bld_base); - - ctx->voidt = LLVMVoidTypeInContext(ctx->gallivm.context); - ctx->i1 = LLVMInt1TypeInContext(ctx->gallivm.context); - ctx->i8 = LLVMInt8TypeInContext(ctx->gallivm.context); - ctx->i32 = LLVMInt32TypeInContext(ctx->gallivm.context); - ctx->i64 = LLVMInt64TypeInContext(ctx->gallivm.context); - ctx->i128 = LLVMIntTypeInContext(ctx->gallivm.context, 128); - ctx->f32 = LLVMFloatTypeInContext(ctx->gallivm.context); - ctx->v16i8 = LLVMVectorType(ctx->i8, 16); + si_shader_context_init_mem(ctx); + + ctx->voidt = LLVMVoidTypeInContext(ctx->ac.context); + ctx->i1 = LLVMInt1TypeInContext(ctx->ac.context); + ctx->i8 = LLVMInt8TypeInContext(ctx->ac.context); + ctx->i32 = LLVMInt32TypeInContext(ctx->ac.context); + ctx->i64 = LLVMInt64TypeInContext(ctx->ac.context); + ctx->i128 = LLVMIntTypeInContext(ctx->ac.context, 128); + ctx->f32 = LLVMFloatTypeInContext(ctx->ac.context); ctx->v2i32 = LLVMVectorType(ctx->i32, 2); ctx->v4i32 = LLVMVectorType(ctx->i32, 4); ctx->v4f32 = LLVMVectorType(ctx->f32, 4); @@ -1349,23 +1025,20 @@ void si_llvm_context_init(struct si_shader_context *ctx, ctx->i32_0 = LLVMConstInt(ctx->i32, 0, 0); ctx->i32_1 = LLVMConstInt(ctx->i32, 1, 0); + ctx->i1false = LLVMConstInt(ctx->i1, 0, 0); + ctx->i1true = LLVMConstInt(ctx->i1, 1, 0); } /* Set the context to a certain TGSI shader. Can be called repeatedly * to change the shader. */ -void si_llvm_context_set_tgsi(struct si_shader_context *ctx, - struct si_shader *shader) +void si_llvm_context_set_ir(struct si_shader_context *ctx, + struct si_shader *shader) { - const struct tgsi_shader_info *info = NULL; - const struct tgsi_token *tokens = NULL; - - if (shader && shader->selector) { - info = &shader->selector->info; - tokens = shader->selector->tokens; - } + struct si_shader_selector *sel = shader->selector; + const struct tgsi_shader_info *info = &sel->info; ctx->shader = shader; - ctx->type = info ? info->processor : -1; + ctx->type = sel->type; ctx->bld_base.info = info; /* Clean up the old contents. */ @@ -1382,7 +1055,13 @@ void si_llvm_context_set_tgsi(struct si_shader_context *ctx, ctx->temps = NULL; ctx->temps_count = 0; - if (!info || !tokens) + ctx->num_const_buffers = util_last_bit(info->const_buffers_declared); + ctx->num_shader_buffers = util_last_bit(info->shader_buffers_declared); + + ctx->num_samplers = util_last_bit(info->samplers_declared); + ctx->num_images = util_last_bit(info->images_declared); + + if (sel->nir) return; if (info->array_max[TGSI_FILE_TEMPORARY] > 0) { @@ -1391,7 +1070,7 @@ void si_llvm_context_set_tgsi(struct si_shader_context *ctx, ctx->temp_arrays = CALLOC(size, sizeof(ctx->temp_arrays[0])); ctx->temp_array_allocas = CALLOC(size, sizeof(ctx->temp_array_allocas[0])); - tgsi_scan_arrays(tokens, TGSI_FILE_TEMPORARY, size, + tgsi_scan_arrays(sel->tokens, TGSI_FILE_TEMPORARY, size, ctx->temp_arrays); } if (info->file_max[TGSI_FILE_IMMEDIATE] >= 0) { @@ -1419,57 +1098,68 @@ void si_llvm_create_func(struct si_shader_context *ctx, { LLVMTypeRef main_fn_type, ret_type; LLVMBasicBlockRef main_fn_body; + enum si_llvm_calling_convention call_conv; + enum pipe_shader_type real_shader_type; if (num_return_elems) - ret_type = LLVMStructTypeInContext(ctx->gallivm.context, + ret_type = LLVMStructTypeInContext(ctx->ac.context, return_types, num_return_elems, true); else - ret_type = LLVMVoidTypeInContext(ctx->gallivm.context); + ret_type = ctx->voidt; /* Setup the function */ ctx->return_type = ret_type; main_fn_type = LLVMFunctionType(ret_type, ParamTypes, ParamCount, 0); ctx->main_fn = LLVMAddFunction(ctx->gallivm.module, name, main_fn_type); - main_fn_body = LLVMAppendBasicBlockInContext(ctx->gallivm.context, + main_fn_body = LLVMAppendBasicBlockInContext(ctx->ac.context, ctx->main_fn, "main_body"); - LLVMPositionBuilderAtEnd(ctx->gallivm.builder, main_fn_body); -} - -void si_llvm_finalize_module(struct si_shader_context *ctx, - bool run_verifier) -{ - struct gallivm_state *gallivm = &ctx->gallivm; - const char *triple = LLVMGetTarget(gallivm->module); - LLVMTargetLibraryInfoRef target_library_info; + LLVMPositionBuilderAtEnd(ctx->ac.builder, main_fn_body); - /* Create the pass manager */ - gallivm->passmgr = LLVMCreatePassManager(); + real_shader_type = ctx->type; - target_library_info = gallivm_create_target_library_info(triple); - LLVMAddTargetLibraryInfo(target_library_info, gallivm->passmgr); - - if (run_verifier) - LLVMAddVerifierPass(gallivm->passmgr); + /* LS is merged into HS (TCS), and ES is merged into GS. */ + if (ctx->screen->info.chip_class >= GFX9) { + if (ctx->shader->key.as_ls) + real_shader_type = PIPE_SHADER_TESS_CTRL; + else if (ctx->shader->key.as_es || ctx->shader->key.as_ngg) + real_shader_type = PIPE_SHADER_GEOMETRY; + } - LLVMAddAlwaysInlinerPass(gallivm->passmgr); + switch (real_shader_type) { + case PIPE_SHADER_VERTEX: + case PIPE_SHADER_TESS_EVAL: + call_conv = RADEON_LLVM_AMDGPU_VS; + break; + case PIPE_SHADER_TESS_CTRL: + call_conv = RADEON_LLVM_AMDGPU_HS; + break; + case PIPE_SHADER_GEOMETRY: + call_conv = RADEON_LLVM_AMDGPU_GS; + break; + case PIPE_SHADER_FRAGMENT: + call_conv = RADEON_LLVM_AMDGPU_PS; + break; + case PIPE_SHADER_COMPUTE: + call_conv = RADEON_LLVM_AMDGPU_CS; + break; + default: + unreachable("Unhandle shader type"); + } - /* This pass should eliminate all the load and store instructions */ - LLVMAddPromoteMemoryToRegisterPass(gallivm->passmgr); + LLVMSetFunctionCallConv(ctx->main_fn, call_conv); +} - /* Add some optimization passes */ - LLVMAddScalarReplAggregatesPass(gallivm->passmgr); - LLVMAddLICMPass(gallivm->passmgr); - LLVMAddAggressiveDCEPass(gallivm->passmgr); - LLVMAddCFGSimplificationPass(gallivm->passmgr); - LLVMAddInstructionCombiningPass(gallivm->passmgr); +void si_llvm_optimize_module(struct si_shader_context *ctx) +{ + /* Dump LLVM IR before any optimization passes */ + if (ctx->screen->debug_flags & DBG(PREOPT_IR) && + si_can_dump_shader(ctx->screen, ctx->type)) + LLVMDumpModule(ctx->gallivm.module); /* Run the pass */ - LLVMRunPassManager(gallivm->passmgr, ctx->gallivm.module); - - LLVMDisposeBuilder(gallivm->builder); - LLVMDisposePassManager(gallivm->passmgr); - gallivm_dispose_target_library_info(target_library_info); + LLVMRunPassManager(ctx->compiler->passmgr, ctx->gallivm.module); + LLVMDisposeBuilder(ctx->ac.builder); } void si_llvm_dispose(struct si_shader_context *ctx) @@ -1486,7 +1176,5 @@ void si_llvm_dispose(struct si_shader_context *ctx) FREE(ctx->imms); ctx->imms = NULL; ctx->imms_num = 0; - FREE(ctx->flow); - ctx->flow = NULL; - ctx->flow_depth_max = 0; + ac_llvm_context_dispose(&ctx->ac); }