From c3464a6bece80703ed0f1968f46505a8d3ed5082 Mon Sep 17 00:00:00 2001 From: Vivek Pandya Date: Mon, 21 Dec 2020 16:38:07 +0530 Subject: [PATCH] Fixing load input and store output related issues. --- src/libre-soc/vulkan/libresoc_llvm.c | 250 ++++++++++++++++++--------- src/libre-soc/vulkan/libresoc_llvm.h | 46 +++++ 2 files changed, 212 insertions(+), 84 deletions(-) diff --git a/src/libre-soc/vulkan/libresoc_llvm.c b/src/libre-soc/vulkan/libresoc_llvm.c index 234091292c0..100fc1ea234 100644 --- a/src/libre-soc/vulkan/libresoc_llvm.c +++ b/src/libre-soc/vulkan/libresoc_llvm.c @@ -1,36 +1,5 @@ #include "libresoc_llvm.h" -#include "libresoc_shader_args.h" #include "libresoc_llvm_build.h" -#include -#include -#include -#include "nir/nir.h" -#include "nir/nir_deref.h" -#include - -struct libresoc_nir_tran_ctx { - struct libresoc_llvm_context lc; - gl_shader_stage stage; - shader_info *info; - - struct shader_args args; - LLVMValueRef *ssa_defs; - - LLVMValueRef scratch; - LLVMValueRef constant_data; - - struct hash_table *defs; - struct hash_table *phis; - struct hash_table *vars; - struct hash_table *verified_interp; - - LLVMValueRef main_function; - LLVMBasicBlockRef continue_block; - LLVMBasicBlockRef break_block; - - int num_locals; - LLVMValueRef *locals; -}; void InitLLVM(struct libresoc_llvm *llvm_ref) { @@ -118,6 +87,55 @@ static uint64_t orc_sym_resolver(const char *name, void *ctx) return (uint64_t)address; } +void handle_shader_output_decl(struct libresoc_nir_tran_ctx *ctx, + struct nir_shader *nir, struct nir_variable *variable, + gl_shader_stage stage) +{ + unsigned output_loc = variable->data.driver_location / 4; + unsigned attrib_count = glsl_count_attribute_slots(variable->type, false); + + /* tess ctrl has it's own load/store paths for outputs */ + if (stage == MESA_SHADER_TESS_CTRL) + return; + + if (stage == MESA_SHADER_VERTEX || stage == MESA_SHADER_TESS_EVAL || + stage == MESA_SHADER_GEOMETRY) { + int idx = variable->data.location + variable->data.index; + if (idx == VARYING_SLOT_CLIP_DIST0) { + int length = nir->info.clip_distance_array_size + nir->info.cull_distance_array_size; + + if (length > 4) + attrib_count = 2; + else + attrib_count = 1; + } + } + + bool is_16bit = glsl_type_is_16bit(glsl_without_array(variable->type)); + LLVMTypeRef type = is_16bit ? ctx->lc.f16 : ctx->lc.f32; + for (unsigned i = 0; i < attrib_count; ++i) { + for (unsigned chan = 0; chan < 4; chan++) { + ctx->outputs[llvm_reg_index_soa(output_loc + i, chan)] = + build_alloca_undef(&ctx->lc, type, ""); + } + } +} + +static void build_store_values_extended(struct libresoc_llvm_context *lc, LLVMValueRef *values, + unsigned value_count, unsigned value_stride, + LLVMValueRef vec) +{ + LLVMBuilderRef builder = lc->builder; + unsigned i; + + for (i = 0; i < value_count; i++) { + LLVMValueRef ptr = values[i * value_stride]; + LLVMValueRef index = LLVMConstInt(lc->i32, i, false); + LLVMValueRef value = LLVMBuildExtractElement(builder, vec, index, ""); + LLVMBuildStore(builder, value, ptr); + } +} + static LLVMTypeRef arg_llvm_type(enum arg_type type, unsigned size, struct libresoc_llvm_context *ctx) { if (type == ARG_FLOAT) { @@ -697,6 +715,64 @@ static void visit_load_const(struct libresoc_nir_tran_ctx *ctx, const nir_load_c ctx->ssa_defs[instr->def.index] = value; } +static void visit_store_output(struct libresoc_nir_tran_ctx *ctx, nir_intrinsic_instr *instr) +{ + // if (ctx->ac.postponed_kill) { + // LLVMValueRef cond = LLVMBuildLoad(ctx->ac.builder, ctx->ac.postponed_kill, ""); + // ac_build_ifcc(&ctx->ac, cond, 7002); + // } + + unsigned base = nir_intrinsic_base(instr); + unsigned writemask = nir_intrinsic_write_mask(instr); + unsigned component = nir_intrinsic_component(instr); + LLVMValueRef src = to_float(&ctx->lc, get_src(ctx, instr->src[0])); + nir_src offset = *nir_get_io_offset_src(instr); + LLVMValueRef indir_index = NULL; + + if (nir_src_is_const(offset)) + assert(nir_src_as_uint(offset) == 0); + else + indir_index = get_src(ctx, offset); + + switch (get_elem_bits(&ctx->lc, LLVMTypeOf(src))) { + case 32: + break; + case 64: + writemask = widen_mask(writemask, 2); + src = LLVMBuildBitCast(ctx->lc.builder, src, + LLVMVectorType(ctx->lc.f32, get_llvm_num_components(src) * 2), ""); + break; + default: + unreachable("unhandled store_output bit size"); + return; + } + + writemask <<= component; + + // if (ctx->stage == MESA_SHADER_TESS_CTRL) { + // nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr); + // LLVMValueRef vertex_index = vertex_index_src ? get_src(ctx, *vertex_index_src) : NULL; + + // ctx->abi->store_tcs_outputs(ctx->abi, NULL, vertex_index, indir_index, 0, src, writemask, + // component, base * 4); + // return; + // } + + /* No indirect indexing is allowed after this point. */ + assert(!indir_index); + + for (unsigned chan = 0; chan < 8; chan++) { + if (!(writemask & (1 << chan))) + continue; + + LLVMValueRef value = llvm_extract_elem(&ctx->lc, src, chan - component); + LLVMBuildStore(ctx->lc.builder, value, ctx->outputs[base * 4 + chan]); + } + + // if (ctx->ac.postponed_kill) + // ac_build_endif(&ctx->ac, 7002); +} + static void visit_deref(struct libresoc_nir_tran_ctx *ctx, nir_deref_instr *instr) { if (instr->mode != nir_var_mem_shared && instr->mode != nir_var_mem_global) @@ -1524,18 +1600,18 @@ static LLVMValueRef visit_load_var(struct libresoc_nir_tran_ctx *ctx, nir_intrin // const_index, type); // } - // for (unsigned chan = comp; chan < ve + comp; chan++) { - // if (indir_index) { - // unsigned count = - // glsl_count_attribute_slots(var->type, ctx->stage == MESA_SHADER_VERTEX); - // count -= chan / 4; - // LLVMValueRef tmp_vec = build_gather_values_extended( - // &ctx->lc, ctx->abi->inputs + idx + chan, count, stride, false, true); - - // values[chan] = LLVMBuildExtractElement(ctx->lc.builder, tmp_vec, indir_index, ""); - // } else - // values[chan] = ctx->abi->inputs[idx + chan + const_index * stride]; - // } + for (unsigned chan = comp; chan < ve + comp; chan++) { + if (indir_index) { + unsigned count = + glsl_count_attribute_slots(var->type, ctx->stage == MESA_SHADER_VERTEX); + count -= chan / 4; + LLVMValueRef tmp_vec = build_gather_values_extended( + &ctx->lc, ctx->inputs + idx + chan, count, stride, false, true); + + values[chan] = LLVMBuildExtractElement(ctx->lc.builder, tmp_vec, indir_index, ""); + } else + values[chan] = ctx->inputs[idx + chan + const_index * stride]; + } break; case nir_var_function_temp: for (unsigned chan = 0; chan < ve; chan++) { @@ -1561,19 +1637,19 @@ static LLVMValueRef visit_load_var(struct libresoc_nir_tran_ctx *ctx, nir_intrin // if (ctx->stage == MESA_SHADER_FRAGMENT && var->data.fb_fetch_output && ctx->abi->emit_fbfetch) // return ctx->abi->emit_fbfetch(ctx->abi); - // for (unsigned chan = comp; chan < ve + comp; chan++) { - // if (indir_index) { - // unsigned count = glsl_count_attribute_slots(var->type, false); - // count -= chan / 4; - // LLVMValueRef tmp_vec = build_gather_values_extended( - // &ctx->lc, ctx->abi->outputs + idx + chan, count, stride, true, true); - - // values[chan] = LLVMBuildExtractElement(ctx->lc.builder, tmp_vec, indir_index, ""); - // } else { - // values[chan] = LLVMBuildLoad(ctx->lc.builder, - // ctx->abi->outputs[idx + chan + const_index * stride], ""); - // } - // } + for (unsigned chan = comp; chan < ve + comp; chan++) { + if (indir_index) { + unsigned count = glsl_count_attribute_slots(var->type, false); + count -= chan / 4; + LLVMValueRef tmp_vec = build_gather_values_extended( + &ctx->lc, ctx->outputs + idx + chan, count, stride, true, true); + + values[chan] = LLVMBuildExtractElement(ctx->lc.builder, tmp_vec, indir_index, ""); + } else { + values[chan] = LLVMBuildLoad(ctx->lc.builder, + ctx->outputs[idx + chan + const_index * stride], ""); + } + } break; case nir_var_mem_global: { LLVMValueRef address = get_src(ctx, instr->src[0]); @@ -1685,23 +1761,23 @@ static void visit_store_var(struct libresoc_nir_tran_ctx *ctx, nir_intrinsic_ins value = llvm_extract_elem(&ctx->lc, src, chan - comp); - // if (var->data.compact) - // stride = 1; - // if (indir_index) { - // unsigned count = glsl_count_attribute_slots(var->type, false); - // count -= chan / 4; - // LLVMValueRef tmp_vec = build_gather_values_extended( - // &ctx->lc, ctx->abi->outputs + idx + chan, count, stride, true, true); + if (var->data.compact) + stride = 1; + if (indir_index) { + unsigned count = glsl_count_attribute_slots(var->type, false); + count -= chan / 4; + LLVMValueRef tmp_vec = build_gather_values_extended( + &ctx->lc, ctx->outputs + idx + chan, count, stride, true, true); - // tmp_vec = LLVMBuildInsertElement(ctx->lc.builder, tmp_vec, value, indir_index, ""); - // build_store_values_extended(&ctx->lc, ctx->abi->outputs + idx + chan, count, stride, - // tmp_vec); + tmp_vec = LLVMBuildInsertElement(ctx->lc.builder, tmp_vec, value, indir_index, ""); + build_store_values_extended(&ctx->lc, ctx->outputs + idx + chan, count, stride, + tmp_vec); - // } else { - // temp_ptr = ctx->abi->outputs[idx + chan + const_index * stride]; + } else { + temp_ptr = ctx->outputs[idx + chan + const_index * stride]; - // LLVMBuildStore(ctx->lc.builder, value, temp_ptr); - // } + LLVMBuildStore(ctx->lc.builder, value, temp_ptr); + } } break; case nir_var_function_temp: @@ -1710,19 +1786,19 @@ static void visit_store_var(struct libresoc_nir_tran_ctx *ctx, nir_intrinsic_ins continue; value = llvm_extract_elem(&ctx->lc, src, chan); - // if (indir_index) { - // unsigned count = glsl_count_attribute_slots(var->type, false); - // count -= chan / 4; - // LLVMValueRef tmp_vec = build_gather_values_extended( - // &ctx->lc, ctx->locals + idx + chan, count, 4, true, true); - - // tmp_vec = LLVMBuildInsertElement(ctx->lc.builder, tmp_vec, value, indir_index, ""); - // build_store_values_extended(&ctx->lc, ctx->locals + idx + chan, count, 4, tmp_vec); - // } else { - // temp_ptr = ctx->locals[idx + chan + const_index * 4]; - - // LLVMBuildStore(ctx->lc.builder, value, temp_ptr); - // } + if (indir_index) { + unsigned count = glsl_count_attribute_slots(var->type, false); + count -= chan / 4; + LLVMValueRef tmp_vec = build_gather_values_extended( + &ctx->lc, ctx->locals + idx + chan, count, 4, true, true); + + tmp_vec = LLVMBuildInsertElement(ctx->lc.builder, tmp_vec, value, indir_index, ""); + build_store_values_extended(&ctx->lc, ctx->locals + idx + chan, count, 4, tmp_vec); + } else { + temp_ptr = ctx->locals[idx + chan + const_index * 4]; + + LLVMBuildStore(ctx->lc.builder, value, temp_ptr); + } } break; @@ -1983,7 +2059,7 @@ static void visit_intrinsic(struct libresoc_nir_tran_ctx *ctx, nir_intrinsic_ins break; case nir_intrinsic_store_output: case nir_intrinsic_store_per_vertex_output: - // visit_store_output(ctx, instr); + visit_store_output(ctx, instr); break; case nir_intrinsic_load_shared: // result = visit_load_shared(ctx, instr); @@ -2452,6 +2528,12 @@ LLVMModuleRef libresoc_nir_translate(struct libresoc_llvm *llvm_ref, struct nir_ LLVMPositionBuilderAtEnd(ctx.lc.builder, main_function_body); ctx.main_function = main_function; + if (!nir->info.io_lowered) { + nir_foreach_shader_out_variable(variable, nir) + { + handle_shader_output_decl(&ctx, nir, variable, ctx.stage); + } + } ctx.defs = _mesa_hash_table_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); ctx.phis = _mesa_hash_table_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); ctx.vars = _mesa_hash_table_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); diff --git a/src/libre-soc/vulkan/libresoc_llvm.h b/src/libre-soc/vulkan/libresoc_llvm.h index 8d3f671a029..4668e47c15b 100644 --- a/src/libre-soc/vulkan/libresoc_llvm.h +++ b/src/libre-soc/vulkan/libresoc_llvm.h @@ -1,7 +1,15 @@ #ifndef LIBRESOC_LLVM_H #define LIBRESOC_LLVM_H +#include "compiler/shader_enums.h" #include +#include "libresoc_shader_args.h" +#include +#include +#include +#include "nir/nir.h" +#include "nir/nir_deref.h" +#include enum { @@ -117,9 +125,47 @@ struct libresoc_llvm { libresoc_llvm_context lc; }; +struct libresoc_nir_tran_ctx { + struct libresoc_llvm_context lc; + gl_shader_stage stage; + shader_info *info; + + struct shader_args args; + LLVMValueRef *ssa_defs; + + LLVMValueRef scratch; + LLVMValueRef constant_data; + + struct hash_table *defs; + struct hash_table *phis; + struct hash_table *vars; + struct hash_table *verified_interp; + + LLVMValueRef main_function; + LLVMBasicBlockRef continue_block; + LLVMBasicBlockRef break_block; + + int num_locals; + LLVMValueRef *locals; + /* For VS and PS: pre-loaded shader inputs. + * + * Currently only used for NIR shaders; indexed by variables' + * driver_location. + */ + LLVMValueRef *inputs; + LLVMValueRef outputs[256]; +}; void InitLLVM(struct libresoc_llvm *llvm_ref); void DestroyLLVM(struct libresoc_llvm *llvm_ref); +static inline unsigned llvm_reg_index_soa(unsigned index, unsigned chan) +{ + return (index * 4) + chan; +} + +void handle_shader_output_decl(struct libresoc_nir_tran_ctx *ctx, + struct nir_shader *nir, struct nir_variable *variable, + gl_shader_stage stage); LLVMModuleRef libresoc_nir_translate(struct libresoc_llvm *llvm_ref, struct nir_shader *nir); #endif -- 2.30.2