X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fspirv%2Fspirv_to_nir.c;h=270f263d0471c5017d9cb75ca330668d1b3c6f72;hb=12301766def1a5eb159644d9afa402ac187d9a01;hp=b719ea831ffc9048b73de5c0f0cc27816488ac62;hpb=8256ee3fa363064ac3bd824d436aced81c61d23f;p=mesa.git diff --git a/src/compiler/spirv/spirv_to_nir.c b/src/compiler/spirv/spirv_to_nir.c index b719ea831ff..270f263d047 100644 --- a/src/compiler/spirv/spirv_to_nir.c +++ b/src/compiler/spirv/spirv_to_nir.c @@ -29,6 +29,7 @@ #include "nir/nir_vla.h" #include "nir/nir_control_flow.h" #include "nir/nir_constant_expressions.h" +#include "nir/nir_deref.h" #include "spirv_info.h" #include @@ -129,6 +130,18 @@ _vtn_warn(struct vtn_builder *b, const char *file, unsigned line, va_end(args); } +void +_vtn_err(struct vtn_builder *b, const char *file, unsigned line, + const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + vtn_log_err(b, NIR_SPIRV_DEBUG_LEVEL_ERROR, "SPIR-V ERROR:\n", + file, line, fmt, args); + va_end(args); +} + void _vtn_fail(struct vtn_builder *b, const char *file, unsigned line, const char *fmt, ...) @@ -207,6 +220,8 @@ vtn_const_ssa_value(struct vtn_builder *b, nir_constant *constant, case GLSL_TYPE_UINT: case GLSL_TYPE_INT16: case GLSL_TYPE_UINT16: + case GLSL_TYPE_UINT8: + case GLSL_TYPE_INT8: case GLSL_TYPE_INT64: case GLSL_TYPE_UINT64: case GLSL_TYPE_BOOL: @@ -373,6 +388,12 @@ vtn_handle_extension(struct vtn_builder *b, SpvOp opcode, struct vtn_value *val = vtn_push_value(b, w[1], vtn_value_type_extension); if (strcmp((const char *)&w[2], "GLSL.std.450") == 0) { val->ext_handler = vtn_handle_glsl450_instruction; + } else if ((strcmp((const char *)&w[2], "SPV_AMD_gcn_shader") == 0) + && (b->options && b->options->caps.gcn_shader)) { + val->ext_handler = vtn_handle_amd_gcn_shader_instruction; + } else if ((strcmp((const char *)&w[2], "SPV_AMD_shader_trinary_minmax") == 0) + && (b->options && b->options->caps.trinary_minmax)) { + val->ext_handler = vtn_handle_amd_shader_trinary_minmax_instruction; } else { vtn_fail("Unsupported extension"); } @@ -458,7 +479,7 @@ vtn_foreach_execution_mode(struct vtn_builder *b, struct vtn_value *value, } } -static void +void vtn_handle_decoration(struct vtn_builder *b, SpvOp opcode, const uint32_t *w, unsigned count) { @@ -929,7 +950,6 @@ vtn_type_layout_std430(struct vtn_builder *b, struct vtn_type *type, case vtn_base_type_vector: { uint32_t comp_size = glsl_get_bit_size(type->type) / 8; - assert(type->length > 0 && type->length <= 4); unsigned align_comps = type->length == 3 ? 4 : type->length; *size_out = comp_size * type->length, *align_out = comp_size * align_comps; @@ -1006,6 +1026,9 @@ vtn_handle_type(struct vtn_builder *b, SpvOp opcode, case 16: val->type->type = (signedness ? glsl_int16_t_type() : glsl_uint16_t_type()); break; + case 8: + val->type->type = (signedness ? glsl_int8_t_type() : glsl_uint8_t_type()); + break; default: vtn_fail("Invalid int bit size"); } @@ -1039,7 +1062,7 @@ vtn_handle_type(struct vtn_builder *b, SpvOp opcode, vtn_fail_if(base->base_type != vtn_base_type_scalar, "Base type for OpTypeVector must be a scalar"); - vtn_fail_if(elems < 2 || elems > 4, + vtn_fail_if((elems < 2 || elems > 4) && (elems != 8) && (elems != 16), "Invalid component count for OpTypeVector"); val->type->base_type = vtn_base_type_vector; @@ -1159,6 +1182,13 @@ vtn_handle_type(struct vtn_builder *b, SpvOp opcode, val->type->type = glsl_vector_type(GLSL_TYPE_UINT, 2); } + if (storage_class == SpvStorageClassPushConstant) { + /* These can actually be stored to nir_variables and used as SSA + * values so they need a real glsl_type. + */ + val->type->type = glsl_uint_type(); + } + if (storage_class == SpvStorageClassWorkgroup && b->options->lower_workgroup_access_to_offsets) { uint32_t size, align; @@ -1280,6 +1310,8 @@ vtn_null_constant(struct vtn_builder *b, const struct glsl_type *type) case GLSL_TYPE_UINT: case GLSL_TYPE_INT16: case GLSL_TYPE_UINT16: + case GLSL_TYPE_UINT8: + case GLSL_TYPE_INT8: case GLSL_TYPE_INT64: case GLSL_TYPE_UINT64: case GLSL_TYPE_BOOL: @@ -1419,6 +1451,9 @@ vtn_handle_constant(struct vtn_builder *b, SpvOp opcode, case 16: val->constant->values->u16[0] = w[3]; break; + case 8: + val->constant->values->u8[0] = w[3]; + break; default: vtn_fail("Unsupported SpvOpConstant bit size"); } @@ -1441,6 +1476,9 @@ vtn_handle_constant(struct vtn_builder *b, SpvOp opcode, case 16: val->constant->values[0].u16[0] = get_specialization(b, val, w[3]); break; + case 8: + val->constant->values[0].u8[0] = get_specialization(b, val, w[3]); + break; default: vtn_fail("Unsupported SpvOpSpecConstant bit size"); } @@ -1473,6 +1511,9 @@ vtn_handle_constant(struct vtn_builder *b, SpvOp opcode, case 16: val->constant->values[0].u16[i] = elems[i]->values[0].u16[0]; break; + case 8: + val->constant->values[0].u8[i] = elems[i]->values[0].u8[0]; + break; default: vtn_fail("Invalid SpvOpConstantComposite bit size"); } @@ -1643,6 +1684,9 @@ vtn_handle_constant(struct vtn_builder *b, SpvOp opcode, case 16: val->constant->values[0].u16[i] = (*c)->values[col].u16[elem + i]; break; + case 8: + val->constant->values[0].u8[i] = (*c)->values[col].u8[elem + i]; + break; default: vtn_fail("Invalid SpvOpCompositeExtract bit size"); } @@ -1667,6 +1711,9 @@ vtn_handle_constant(struct vtn_builder *b, SpvOp opcode, case 16: (*c)->values[col].u16[elem + i] = insert->constant->values[0].u16[i]; break; + case 8: + (*c)->values[col].u8[elem + i] = insert->constant->values[0].u8[i]; + break; default: vtn_fail("Invalid SpvOpCompositeInsert bit size"); } @@ -1700,8 +1747,8 @@ vtn_handle_constant(struct vtn_builder *b, SpvOp opcode, }; nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap, - src_alu_type, - dst_alu_type); + nir_alu_type_get_type_size(src_alu_type), + nir_alu_type_get_type_size(dst_alu_type)); nir_const_value src[4]; for (unsigned i = 0; i < count - 4; i++) { @@ -1748,39 +1795,54 @@ vtn_handle_function_call(struct vtn_builder *b, SpvOp opcode, vtn_callee->referenced = true; nir_call_instr *call = nir_call_instr_create(b->nb.shader, callee); - for (unsigned i = 0; i < call->num_params; i++) { - unsigned arg_id = w[4 + i]; - struct vtn_value *arg = vtn_untyped_value(b, arg_id); - if (arg->value_type == vtn_value_type_pointer && - arg->pointer->ptr_type->type == NULL) { - nir_deref_var *d = vtn_pointer_to_deref(b, arg->pointer); - call->params[i] = nir_deref_var_clone(d, call); - } else { - struct vtn_ssa_value *arg_ssa = vtn_ssa_value(b, arg_id); - /* Make a temporary to store the argument in */ - nir_variable *tmp = - nir_local_variable_create(b->nb.impl, arg_ssa->type, "arg_tmp"); - call->params[i] = nir_deref_var_create(call, tmp); + unsigned param_idx = 0; - vtn_local_store(b, arg_ssa, call->params[i]); - } + nir_deref_instr *ret_deref = NULL; + struct vtn_type *ret_type = vtn_callee->type->return_type; + if (ret_type->base_type != vtn_base_type_void) { + nir_variable *ret_tmp = + nir_local_variable_create(b->nb.impl, ret_type->type, "return_tmp"); + ret_deref = nir_build_deref_var(&b->nb, ret_tmp); + call->params[param_idx++] = nir_src_for_ssa(&ret_deref->dest.ssa); } - nir_variable *out_tmp = NULL; - vtn_assert(res_type->type == callee->return_type); - if (!glsl_type_is_void(callee->return_type)) { - out_tmp = nir_local_variable_create(b->nb.impl, callee->return_type, - "out_tmp"); - call->return_deref = nir_deref_var_create(call, out_tmp); + for (unsigned i = 0; i < vtn_callee->type->length; i++) { + struct vtn_type *arg_type = vtn_callee->type->params[i]; + unsigned arg_id = w[4 + i]; + + if (arg_type->base_type == vtn_base_type_sampled_image) { + struct vtn_sampled_image *sampled_image = + vtn_value(b, arg_id, vtn_value_type_sampled_image)->sampled_image; + + call->params[param_idx++] = + nir_src_for_ssa(&sampled_image->image->deref->dest.ssa); + call->params[param_idx++] = + nir_src_for_ssa(&sampled_image->sampler->deref->dest.ssa); + } else if (arg_type->base_type == vtn_base_type_pointer || + arg_type->base_type == vtn_base_type_image || + arg_type->base_type == vtn_base_type_sampler) { + struct vtn_pointer *pointer = + vtn_value(b, arg_id, vtn_value_type_pointer)->pointer; + call->params[param_idx++] = + nir_src_for_ssa(vtn_pointer_to_ssa(b, pointer)); + } else { + /* This is a regular SSA value and we need a temporary */ + nir_variable *tmp = + nir_local_variable_create(b->nb.impl, arg_type->type, "arg_tmp"); + nir_deref_instr *tmp_deref = nir_build_deref_var(&b->nb, tmp); + vtn_local_store(b, vtn_ssa_value(b, arg_id), tmp_deref); + call->params[param_idx++] = nir_src_for_ssa(&tmp_deref->dest.ssa); + } } + assert(param_idx == call->num_params); nir_builder_instr_insert(&b->nb, &call->instr); - if (glsl_type_is_void(callee->return_type)) { + if (ret_type->base_type == vtn_base_type_void) { vtn_push_value(b, w[2], vtn_value_type_undef); } else { - vtn_push_ssa(b, w[2], res_type, vtn_local_load(b, call->return_deref)); + vtn_push_ssa(b, w[2], res_type, vtn_local_load(b, ret_deref)); } } @@ -1801,6 +1863,8 @@ vtn_create_ssa_value(struct vtn_builder *b, const struct glsl_type *type) case GLSL_TYPE_UINT: case GLSL_TYPE_INT16: case GLSL_TYPE_UINT16: + case GLSL_TYPE_UINT8: + case GLSL_TYPE_INT8: case GLSL_TYPE_INT64: case GLSL_TYPE_UINT64: case GLSL_TYPE_BOOL: @@ -1879,7 +1943,6 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, const struct glsl_type *image_type = sampled.type->type; const enum glsl_sampler_dim sampler_dim = glsl_get_sampler_dim(image_type); const bool is_array = glsl_sampler_type_is_array(image_type); - const bool is_shadow = glsl_sampler_type_is_shadow(image_type); /* Figure out the base texture operation */ nir_texop texop; @@ -1932,9 +1995,41 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, vtn_fail("Unhandled opcode"); } - nir_tex_src srcs[8]; /* 8 should be enough */ + nir_tex_src srcs[10]; /* 10 should be enough */ nir_tex_src *p = srcs; + nir_deref_instr *sampler = vtn_pointer_to_deref(b, sampled.sampler); + nir_deref_instr *texture = + sampled.image ? vtn_pointer_to_deref(b, sampled.image) : sampler; + + p->src = nir_src_for_ssa(&texture->dest.ssa); + p->src_type = nir_tex_src_texture_deref; + p++; + + switch (texop) { + case nir_texop_tex: + case nir_texop_txb: + case nir_texop_txl: + case nir_texop_txd: + case nir_texop_tg4: + /* These operations require a sampler */ + p->src = nir_src_for_ssa(&sampler->dest.ssa); + p->src_type = nir_tex_src_sampler_deref; + p++; + break; + case nir_texop_txf: + case nir_texop_txf_ms: + case nir_texop_txs: + case nir_texop_lod: + case nir_texop_query_levels: + case nir_texop_texture_samples: + case nir_texop_samples_identical: + /* These don't */ + break; + case nir_texop_txf_ms_mcs: + vtn_fail("unexpected nir_texop_txf_ms_mcs"); + } + unsigned idx = 4; struct nir_ssa_def *coord; @@ -2003,6 +2098,7 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, break; } + bool is_shadow = false; unsigned gather_component = 0; switch (opcode) { case SpvOpImageSampleDrefImplicitLod: @@ -2011,6 +2107,7 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, case SpvOpImageSampleProjDrefExplicitLod: case SpvOpImageDrefGather: /* These all have an explicit depth value as their next source */ + is_shadow = true; (*p++) = vtn_tex_src(b, w[idx++], nir_tex_src_comparator); break; @@ -2057,8 +2154,9 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, (*p++) = vtn_tex_src(b, w[idx++], nir_tex_src_offset); if (operands & SpvImageOperandsConstOffsetsMask) { + nir_tex_src none = {0}; gather_offsets = vtn_ssa_value(b, w[idx++]); - (*p++) = (nir_tex_src){}; + (*p++) = none; } if (operands & SpvImageOperandsSampleMask) { @@ -2092,40 +2190,6 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, vtn_fail("Invalid base type for sampler result"); } - nir_deref_var *sampler = vtn_pointer_to_deref(b, sampled.sampler); - nir_deref_var *texture; - if (sampled.image) { - nir_deref_var *image = vtn_pointer_to_deref(b, sampled.image); - texture = image; - } else { - texture = sampler; - } - - instr->texture = nir_deref_var_clone(texture, instr); - - switch (instr->op) { - case nir_texop_tex: - case nir_texop_txb: - case nir_texop_txl: - case nir_texop_txd: - case nir_texop_tg4: - /* These operations require a sampler */ - instr->sampler = nir_deref_var_clone(sampler, instr); - break; - case nir_texop_txf: - case nir_texop_txf_ms: - case nir_texop_txs: - case nir_texop_lod: - case nir_texop_query_levels: - case nir_texop_texture_samples: - case nir_texop_samples_identical: - /* These don't */ - instr->sampler = NULL; - break; - case nir_texop_txf_ms_mcs: - vtn_fail("unexpected nir_texop_txf_ms_mcs"); - } - nir_ssa_dest_init(&instr->instr, &instr->dest, nir_tex_instr_dest_size(instr), 32, NULL); @@ -2150,8 +2214,6 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, instrs[i]->is_new_style_shadow = instr->is_new_style_shadow; instrs[i]->component = instr->component; instrs[i]->dest_type = instr->dest_type; - instrs[i]->texture = nir_deref_var_clone(texture, instrs[i]); - instrs[i]->sampler = NULL; memcpy(instrs[i]->src, srcs, instr->num_srcs * sizeof(*instr->src)); @@ -2326,7 +2388,7 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, nir_intrinsic_op op; switch (opcode) { -#define OP(S, N) case SpvOp##S: op = nir_intrinsic_image_##N; break; +#define OP(S, N) case SpvOp##S: op = nir_intrinsic_image_deref_##N; break; OP(ImageQuerySize, size) OP(ImageRead, load) OP(ImageWrite, store) @@ -2352,8 +2414,8 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op); - nir_deref_var *image_deref = vtn_pointer_to_deref(b, image.image); - intrin->variables[0] = nir_deref_var_clone(image_deref, intrin); + nir_deref_instr *image_deref = vtn_pointer_to_deref(b, image.image); + intrin->src[0] = nir_src_for_ssa(&image_deref->dest.ssa); /* ImageQuerySize doesn't take any extra parameters */ if (opcode != SpvOpImageQuerySize) { @@ -2363,9 +2425,9 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, unsigned swiz[4]; for (unsigned i = 0; i < 4; i++) swiz[i] = i < image.coord->num_components ? i : 0; - intrin->src[0] = nir_src_for_ssa(nir_swizzle(&b->nb, image.coord, + intrin->src[1] = nir_src_for_ssa(nir_swizzle(&b->nb, image.coord, swiz, 4, false)); - intrin->src[1] = nir_src_for_ssa(image.sample); + intrin->src[2] = nir_src_for_ssa(image.sample); } switch (opcode) { @@ -2374,10 +2436,10 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, case SpvOpImageRead: break; case SpvOpAtomicStore: - intrin->src[2] = nir_src_for_ssa(vtn_ssa_value(b, w[4])->def); + intrin->src[3] = nir_src_for_ssa(vtn_ssa_value(b, w[4])->def); break; case SpvOpImageWrite: - intrin->src[2] = nir_src_for_ssa(vtn_ssa_value(b, w[3])->def); + intrin->src[3] = nir_src_for_ssa(vtn_ssa_value(b, w[3])->def); break; case SpvOpAtomicCompareExchange: @@ -2393,7 +2455,7 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, case SpvOpAtomicAnd: case SpvOpAtomicOr: case SpvOpAtomicXor: - fill_common_atomic_sources(b, opcode, w, &intrin->src[2]); + fill_common_atomic_sources(b, opcode, w, &intrin->src[3]); break; default: @@ -2404,9 +2466,8 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); struct vtn_type *type = vtn_value(b, w[1], vtn_value_type_type)->type; - unsigned dest_components = - nir_intrinsic_infos[intrin->intrinsic].dest_components; - if (intrin->intrinsic == nir_intrinsic_image_size) { + unsigned dest_components = nir_intrinsic_dest_components(intrin); + if (intrin->intrinsic == nir_intrinsic_image_deref_size) { dest_components = intrin->num_components = glsl_get_vector_elements(type->type); } @@ -2449,6 +2510,35 @@ get_ssbo_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) } } +static nir_intrinsic_op +get_uniform_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) +{ + switch (opcode) { +#define OP(S, N) case SpvOp##S: return nir_intrinsic_atomic_counter_ ##N; + OP(AtomicLoad, read_deref) + OP(AtomicExchange, exchange) + OP(AtomicCompareExchange, comp_swap) + OP(AtomicIIncrement, inc_deref) + OP(AtomicIDecrement, dec_deref) + OP(AtomicIAdd, add_deref) + OP(AtomicISub, add_deref) + OP(AtomicUMin, min_deref) + OP(AtomicUMax, max_deref) + OP(AtomicAnd, and_deref) + OP(AtomicOr, or_deref) + OP(AtomicXor, xor_deref) +#undef OP + default: + /* We left the following out: AtomicStore, AtomicSMin and + * AtomicSmax. Right now there are not nir intrinsics for them. At this + * moment Atomic Counter support is needed for ARB_spirv support, so is + * only need to support GLSL Atomic Counters that are uints and don't + * allow direct storage. + */ + unreachable("Invalid uniform atomic"); + } +} + static nir_intrinsic_op get_shared_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) { @@ -2476,12 +2566,12 @@ get_shared_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) } static nir_intrinsic_op -get_var_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) +get_deref_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) { switch (opcode) { - case SpvOpAtomicLoad: return nir_intrinsic_load_var; - case SpvOpAtomicStore: return nir_intrinsic_store_var; -#define OP(S, N) case SpvOp##S: return nir_intrinsic_var_##N; + case SpvOpAtomicLoad: return nir_intrinsic_load_deref; + case SpvOpAtomicStore: return nir_intrinsic_store_deref; +#define OP(S, N) case SpvOp##S: return nir_intrinsic_deref_##N; OP(AtomicExchange, atomic_exchange) OP(AtomicCompareExchange, atomic_comp_swap) OP(AtomicIIncrement, atomic_add) @@ -2501,9 +2591,12 @@ get_var_nir_atomic_op(struct vtn_builder *b, SpvOp opcode) } } +/* + * Handles shared atomics, ssbo atomics and atomic counters. + */ static void -vtn_handle_ssbo_or_shared_atomic(struct vtn_builder *b, SpvOp opcode, - const uint32_t *w, unsigned count) +vtn_handle_atomics(struct vtn_builder *b, SpvOp opcode, + const uint32_t *w, unsigned count) { struct vtn_pointer *ptr; nir_intrinsic_instr *atomic; @@ -2540,13 +2633,18 @@ vtn_handle_ssbo_or_shared_atomic(struct vtn_builder *b, SpvOp opcode, SpvMemorySemanticsMask semantics = w[5]; */ - if (ptr->mode == vtn_variable_mode_workgroup && - !b->options->lower_workgroup_access_to_offsets) { - nir_deref_var *deref = vtn_pointer_to_deref(b, ptr); - const struct glsl_type *deref_type = nir_deref_tail(&deref->deref)->type; - nir_intrinsic_op op = get_var_nir_atomic_op(b, opcode); + /* uniform as "atomic counter uniform" */ + if (ptr->mode == vtn_variable_mode_uniform) { + nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr); + const struct glsl_type *deref_type = deref->type; + nir_intrinsic_op op = get_uniform_nir_atomic_op(b, opcode); atomic = nir_intrinsic_instr_create(b->nb.shader, op); - atomic->variables[0] = nir_deref_var_clone(deref, atomic); + atomic->src[0] = nir_src_for_ssa(&deref->dest.ssa); + + /* SSBO needs to initialize index/offset. In this case we don't need to, + * as that info is already stored on the ptr->var->var nir_variable (see + * vtn_create_variable) + */ switch (opcode) { case SpvOpAtomicLoad: @@ -2556,7 +2654,6 @@ vtn_handle_ssbo_or_shared_atomic(struct vtn_builder *b, SpvOp opcode, case SpvOpAtomicStore: atomic->num_components = glsl_get_vector_elements(deref_type); nir_intrinsic_set_write_mask(atomic, (1 << atomic->num_components) - 1); - atomic->src[0] = nir_src_for_ssa(vtn_ssa_value(b, w[4])->def); break; case SpvOpAtomicExchange: @@ -2573,7 +2670,49 @@ vtn_handle_ssbo_or_shared_atomic(struct vtn_builder *b, SpvOp opcode, case SpvOpAtomicAnd: case SpvOpAtomicOr: case SpvOpAtomicXor: - fill_common_atomic_sources(b, opcode, w, &atomic->src[0]); + /* Nothing: we don't need to call fill_common_atomic_sources here, as + * atomic counter uniforms doesn't have sources + */ + break; + + default: + unreachable("Invalid SPIR-V atomic"); + + } + } else if (ptr->mode == vtn_variable_mode_workgroup && + !b->options->lower_workgroup_access_to_offsets) { + nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr); + const struct glsl_type *deref_type = deref->type; + nir_intrinsic_op op = get_deref_nir_atomic_op(b, opcode); + atomic = nir_intrinsic_instr_create(b->nb.shader, op); + atomic->src[0] = nir_src_for_ssa(&deref->dest.ssa); + + switch (opcode) { + case SpvOpAtomicLoad: + atomic->num_components = glsl_get_vector_elements(deref_type); + break; + + case SpvOpAtomicStore: + atomic->num_components = glsl_get_vector_elements(deref_type); + nir_intrinsic_set_write_mask(atomic, (1 << atomic->num_components) - 1); + atomic->src[1] = nir_src_for_ssa(vtn_ssa_value(b, w[4])->def); + break; + + case SpvOpAtomicExchange: + case SpvOpAtomicCompareExchange: + case SpvOpAtomicCompareExchangeWeak: + case SpvOpAtomicIIncrement: + case SpvOpAtomicIDecrement: + case SpvOpAtomicIAdd: + case SpvOpAtomicISub: + case SpvOpAtomicSMin: + case SpvOpAtomicUMin: + case SpvOpAtomicSMax: + case SpvOpAtomicUMax: + case SpvOpAtomicAnd: + case SpvOpAtomicOr: + case SpvOpAtomicXor: + fill_common_atomic_sources(b, opcode, w, &atomic->src[1]); break; default: @@ -2582,7 +2721,7 @@ vtn_handle_ssbo_or_shared_atomic(struct vtn_builder *b, SpvOp opcode, } } else { nir_ssa_def *offset, *index; - offset = vtn_pointer_to_offset(b, ptr, &index, NULL); + offset = vtn_pointer_to_offset(b, ptr, &index); nir_intrinsic_op op; if (ptr->mode == vtn_variable_mode_ssbo) { @@ -2922,6 +3061,7 @@ vtn_handle_composite(struct vtn_builder *b, SpvOp opcode, case SpvOpCompositeConstruct: { unsigned elems = count - 3; + assume(elems >= 1); if (glsl_type_is_vector_or_scalar(type)) { nir_ssa_def *srcs[4]; for (unsigned i = 0; i < elems; i++) @@ -3161,6 +3301,24 @@ stage_for_execution_model(struct vtn_builder *b, SpvExecutionModel model) spirv_capability_to_string(cap)); \ } while(0) + +void +vtn_handle_entry_point(struct vtn_builder *b, const uint32_t *w, + unsigned count) +{ + struct vtn_value *entry_point = &b->values[w[2]]; + /* Let this be a name label regardless */ + unsigned name_words; + entry_point->name = vtn_string_literal(b, &w[3], count - 3, &name_words); + + if (strcmp(entry_point->name, b->entry_point_name) != 0 || + stage_for_execution_model(b, w[1]) != b->entry_point_stage) + return; + + vtn_assert(b->entry_point == NULL); + b->entry_point = entry_point; +} + static bool vtn_handle_preamble_instruction(struct vtn_builder *b, SpvOp opcode, const uint32_t *w, unsigned count) @@ -3232,7 +3390,6 @@ vtn_handle_preamble_instruction(struct vtn_builder *b, SpvOp opcode, case SpvCapabilityFloat16: case SpvCapabilityInt64Atomics: case SpvCapabilityAtomicStorage: - case SpvCapabilityInt16: case SpvCapabilityStorageImageMultisample: case SpvCapabilityInt8: case SpvCapabilitySparseResidency: @@ -3248,6 +3405,9 @@ vtn_handle_preamble_instruction(struct vtn_builder *b, SpvOp opcode, case SpvCapabilityInt64: spv_check_supported(int64, cap); break; + case SpvCapabilityInt16: + spv_check_supported(int16, cap); + break; case SpvCapabilityAddresses: case SpvCapabilityKernel: @@ -3310,6 +3470,15 @@ vtn_handle_preamble_instruction(struct vtn_builder *b, SpvOp opcode, spv_check_supported(subgroup_shuffle, cap); break; + case SpvCapabilityGroupNonUniformQuad: + spv_check_supported(subgroup_quad, cap); + break; + + case SpvCapabilityGroupNonUniformArithmetic: + case SpvCapabilityGroupNonUniformClustered: + spv_check_supported(subgroup_arithmetic, cap); + break; + case SpvCapabilityVariablePointersStorageBuffer: case SpvCapabilityVariablePointers: spv_check_supported(variable_pointers, cap); @@ -3326,6 +3495,20 @@ vtn_handle_preamble_instruction(struct vtn_builder *b, SpvOp opcode, spv_check_supported(shader_viewport_index_layer, cap); break; + case SpvCapabilityInputAttachmentArrayDynamicIndexingEXT: + case SpvCapabilityUniformTexelBufferArrayDynamicIndexingEXT: + case SpvCapabilityStorageTexelBufferArrayDynamicIndexingEXT: + spv_check_supported(descriptor_array_dynamic_indexing, cap); + break; + + case SpvCapabilityRuntimeDescriptorArrayEXT: + spv_check_supported(runtime_descriptor_array, cap); + break; + + case SpvCapabilityStencilExportEXT: + spv_check_supported(stencil_export, cap); + break; + default: vtn_fail("Unhandled capability"); } @@ -3342,20 +3525,9 @@ vtn_handle_preamble_instruction(struct vtn_builder *b, SpvOp opcode, w[2] == SpvMemoryModelGLSL450); break; - case SpvOpEntryPoint: { - struct vtn_value *entry_point = &b->values[w[2]]; - /* Let this be a name label regardless */ - unsigned name_words; - entry_point->name = vtn_string_literal(b, &w[3], count - 3, &name_words); - - if (strcmp(entry_point->name, b->entry_point_name) != 0 || - stage_for_execution_model(b, w[1]) != b->entry_point_stage) - break; - - vtn_assert(b->entry_point == NULL); - b->entry_point = entry_point; + case SpvOpEntryPoint: + vtn_handle_entry_point(b, w, count); break; - } case SpvOpString: vtn_push_value(b, w[1], vtn_value_type_string)->str = @@ -3514,6 +3686,10 @@ vtn_handle_execution_mode(struct vtn_builder *b, struct vtn_value *entry_point, case SpvExecutionModeContractionOff: break; /* OpenCL */ + case SpvExecutionModeStencilRefReplacingEXT: + vtn_assert(b->shader->info.stage == MESA_SHADER_FRAGMENT); + break; + default: vtn_fail("Unhandled execution mode"); } @@ -3663,10 +3839,10 @@ vtn_handle_body_instruction(struct vtn_builder *b, SpvOp opcode, case SpvOpImageQuerySize: { struct vtn_pointer *image = vtn_value(b, w[3], vtn_value_type_pointer)->pointer; - if (image->mode == vtn_variable_mode_image) { + if (glsl_type_is_image(image->type->type)) { vtn_handle_image(b, opcode, w, count); } else { - vtn_assert(image->mode == vtn_variable_mode_sampler); + vtn_assert(glsl_type_is_sampler(image->type->type)); vtn_handle_texture(b, opcode, w, count); } break; @@ -3692,7 +3868,7 @@ vtn_handle_body_instruction(struct vtn_builder *b, SpvOp opcode, vtn_handle_image(b, opcode, w, count); } else { vtn_assert(pointer->value_type == vtn_value_type_pointer); - vtn_handle_ssbo_or_shared_atomic(b, opcode, w, count); + vtn_handle_atomics(b, opcode, w, count); } break; } @@ -3703,7 +3879,7 @@ vtn_handle_body_instruction(struct vtn_builder *b, SpvOp opcode, vtn_handle_image(b, opcode, w, count); } else { vtn_assert(pointer->value_type == vtn_value_type_pointer); - vtn_handle_ssbo_or_shared_atomic(b, opcode, w, count); + vtn_handle_atomics(b, opcode, w, count); } break; } @@ -3935,14 +4111,12 @@ vtn_handle_body_instruction(struct vtn_builder *b, SpvOp opcode, return true; } -nir_function * -spirv_to_nir(const uint32_t *words, size_t word_count, - struct nir_spirv_specialization *spec, unsigned num_spec, - gl_shader_stage stage, const char *entry_point_name, - const struct spirv_to_nir_options *options, - const nir_shader_compiler_options *nir_options) +struct vtn_builder* +vtn_create_builder(const uint32_t *words, size_t word_count, + gl_shader_stage stage, const char *entry_point_name, + const struct spirv_to_nir_options *options) { - /* Initialize the stn_builder object */ + /* Initialize the vtn_builder object */ struct vtn_builder *b = rzalloc(NULL, struct vtn_builder); b->spirv = words; b->spirv_word_count = word_count; @@ -3954,28 +4128,64 @@ spirv_to_nir(const uint32_t *words, size_t word_count, b->entry_point_name = entry_point_name; b->options = options; - /* See also _vtn_fail() */ - if (setjmp(b->fail_jump)) { - ralloc_free(b); - return NULL; - } - - const uint32_t *word_end = words + word_count; + /* + * Handle the SPIR-V header (first 5 dwords). + * Can't use vtx_assert() as the setjmp(3) target isn't initialized yet. + */ + if (word_count <= 5) + goto fail; - /* Handle the SPIR-V header (first 4 dwords) */ - vtn_assert(word_count > 5); + if (words[0] != SpvMagicNumber) { + vtn_err("words[0] was 0x%x, want 0x%x", words[0], SpvMagicNumber); + goto fail; + } + if (words[1] < 0x10000) { + vtn_err("words[1] was 0x%x, want >= 0x10000", words[1]); + goto fail; + } - vtn_assert(words[0] == SpvMagicNumber); - vtn_assert(words[1] >= 0x10000); /* words[2] == generator magic */ unsigned value_id_bound = words[3]; - vtn_assert(words[4] == 0); - - words+= 5; + if (words[4] != 0) { + vtn_err("words[4] was %u, want 0", words[4]); + goto fail; + } b->value_id_bound = value_id_bound; b->values = rzalloc_array(b, struct vtn_value, value_id_bound); + return b; + fail: + ralloc_free(b); + return NULL; +} + +nir_function * +spirv_to_nir(const uint32_t *words, size_t word_count, + struct nir_spirv_specialization *spec, unsigned num_spec, + gl_shader_stage stage, const char *entry_point_name, + const struct spirv_to_nir_options *options, + const nir_shader_compiler_options *nir_options) + +{ + const uint32_t *word_end = words + word_count; + + struct vtn_builder *b = vtn_create_builder(words, word_count, + stage, entry_point_name, + options); + + if (b == NULL) + return NULL; + + /* See also _vtn_fail() */ + if (setjmp(b->fail_jump)) { + ralloc_free(b); + return NULL; + } + + /* Skip the SPIR-V header, handled at vtn_create_builder */ + words+= 5; + /* Handle all the preamble instructions */ words = vtn_foreach_instruction(b, words, word_end, vtn_handle_preamble_instruction); @@ -4024,6 +4234,11 @@ spirv_to_nir(const uint32_t *words, size_t word_count, } } while (progress); + /* We sometimes generate bogus derefs that, while never used, give the + * validator a bit of heartburn. Run dead code to get rid of them. + */ + nir_opt_dce(b->shader); + vtn_assert(b->entry_point->value_type == vtn_value_type_function); nir_function *entry_point = b->entry_point->func->impl->function; vtn_assert(entry_point);