X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fspirv%2Fvtn_variables.c;h=6de4ceffb82ae2e63053f77490e3a02ec4dbe8b7;hb=23bfba8663cadeb5167c0b30d9b64cf01a913911;hp=ee76230768c40ae241aa9b6a4cef29eaccbed4e2;hpb=5441d562433a6315ca00c0c69160ff848e5ec34a;p=mesa.git diff --git a/src/compiler/spirv/vtn_variables.c b/src/compiler/spirv/vtn_variables.c index ee76230768c..6de4ceffb82 100644 --- a/src/compiler/spirv/vtn_variables.c +++ b/src/compiler/spirv/vtn_variables.c @@ -30,6 +30,76 @@ #include "nir_deref.h" #include +static void +ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, + const struct vtn_decoration *dec, void *void_ptr) +{ + struct vtn_pointer *ptr = void_ptr; + + switch (dec->decoration) { + case SpvDecorationNonUniformEXT: + ptr->access |= ACCESS_NON_UNIFORM; + break; + + default: + break; + } +} + +static struct vtn_pointer* +vtn_decorate_pointer(struct vtn_builder *b, struct vtn_value *val, + struct vtn_pointer *ptr) +{ + struct vtn_pointer dummy = { .access = 0 }; + vtn_foreach_decoration(b, val, ptr_decoration_cb, &dummy); + + /* If we're adding access flags, make a copy of the pointer. We could + * probably just OR them in without doing so but this prevents us from + * leaking them any further than actually specified in the SPIR-V. + */ + if (dummy.access & ~ptr->access) { + struct vtn_pointer *copy = ralloc(b, struct vtn_pointer); + *copy = *ptr; + copy->access |= dummy.access; + return copy; + } + + return ptr; +} + +struct vtn_value * +vtn_push_pointer(struct vtn_builder *b, uint32_t value_id, + struct vtn_pointer *ptr) +{ + struct vtn_value *val = vtn_push_value(b, value_id, vtn_value_type_pointer); + val->pointer = vtn_decorate_pointer(b, val, ptr); + return val; +} + +void +vtn_copy_value(struct vtn_builder *b, uint32_t src_value_id, + uint32_t dst_value_id) +{ + struct vtn_value *src = vtn_untyped_value(b, src_value_id); + struct vtn_value *dst = vtn_untyped_value(b, dst_value_id); + struct vtn_value src_copy = *src; + + vtn_fail_if(dst->value_type != vtn_value_type_invalid, + "SPIR-V id %u has already been written by another instruction", + dst_value_id); + + vtn_fail_if(dst->type->id != src->type->id, + "Result Type must equal Operand type"); + + src_copy.name = dst->name; + src_copy.decoration = dst->decoration; + src_copy.type = dst->type; + *dst = src_copy; + + if (dst->value_type == vtn_value_type_pointer) + dst->pointer = vtn_decorate_pointer(b, dst, dst->pointer); +} + static struct vtn_access_chain * vtn_access_chain_create(struct vtn_builder *b, unsigned length) { @@ -51,9 +121,19 @@ vtn_mode_uses_ssa_offset(struct vtn_builder *b, return ((mode == vtn_variable_mode_ubo || mode == vtn_variable_mode_ssbo) && b->options->lower_ubo_ssbo_access_to_offsets) || + mode == vtn_variable_mode_push_constant; +} + +static bool +vtn_mode_is_cross_invocation(struct vtn_builder *b, + enum vtn_variable_mode mode) +{ + return mode == vtn_variable_mode_ssbo || + mode == vtn_variable_mode_ubo || + mode == vtn_variable_mode_phys_ssbo || mode == vtn_variable_mode_push_constant || - (mode == vtn_variable_mode_workgroup && - b->options->lower_workgroup_access_to_offsets); + mode == vtn_variable_mode_workgroup || + mode == vtn_variable_mode_cross_workgroup; } static bool @@ -63,9 +143,7 @@ vtn_pointer_is_external_block(struct vtn_builder *b, return ptr->mode == vtn_variable_mode_ssbo || ptr->mode == vtn_variable_mode_ubo || ptr->mode == vtn_variable_mode_phys_ssbo || - ptr->mode == vtn_variable_mode_push_constant || - (ptr->mode == vtn_variable_mode_workgroup && - b->options->lower_workgroup_access_to_offsets); + ptr->mode == vtn_variable_mode_push_constant; } static nir_ssa_def * @@ -100,6 +178,8 @@ static nir_ssa_def * vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var, nir_ssa_def *desc_array_index) { + vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); + if (!desc_array_index) { vtn_assert(glsl_type_is_struct_or_ifc(var->type->type)); desc_array_index = nir_imm_int(&b->nb, 0); @@ -134,6 +214,8 @@ static nir_ssa_def * vtn_resource_reindex(struct vtn_builder *b, enum vtn_variable_mode mode, nir_ssa_def *base_index, nir_ssa_def *offset_index) { + vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); + nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, nir_intrinsic_vulkan_resource_reindex); @@ -161,6 +243,8 @@ static nir_ssa_def * vtn_descriptor_load(struct vtn_builder *b, enum vtn_variable_mode mode, nir_ssa_def *desc_index) { + vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); + nir_intrinsic_instr *desc_load = nir_intrinsic_instr_create(b->nb.shader, nir_intrinsic_load_vulkan_descriptor); @@ -190,13 +274,14 @@ vtn_nir_deref_pointer_dereference(struct vtn_builder *b, struct vtn_access_chain *deref_chain) { struct vtn_type *type = base->type; - enum gl_access_qualifier access = base->access; + enum gl_access_qualifier access = base->access | deref_chain->access; unsigned idx = 0; nir_deref_instr *tail; if (base->deref) { tail = base->deref; - } else if (vtn_pointer_is_external_block(b, base)) { + } else if (b->options->environment == NIR_SPIRV_VULKAN && + vtn_pointer_is_external_block(b, base)) { nir_ssa_def *block_index = base->block_index; /* We dereferencing an external block pointer. Correctness of this @@ -288,7 +373,8 @@ vtn_nir_deref_pointer_dereference(struct vtn_builder *b, nir_variable_mode nir_mode = base->mode == vtn_variable_mode_ssbo ? nir_var_mem_ssbo : nir_var_mem_ubo; - tail = nir_build_deref_cast(&b->nb, desc, nir_mode, type->type, + tail = nir_build_deref_cast(&b->nb, desc, nir_mode, + vtn_type_get_nir_type(b, type, base->mode), base->ptr_type->stride); } else { assert(base->var && base->var->var); @@ -529,55 +615,9 @@ vtn_pointer_dereference(struct vtn_builder *b, } } -struct vtn_pointer * -vtn_pointer_for_variable(struct vtn_builder *b, - struct vtn_variable *var, struct vtn_type *ptr_type) -{ - struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer); - - pointer->mode = var->mode; - pointer->type = var->type; - vtn_assert(ptr_type->base_type == vtn_base_type_pointer); - vtn_assert(ptr_type->deref->type == var->type->type); - pointer->ptr_type = ptr_type; - pointer->var = var; - pointer->access = var->access | var->type->access; - - return pointer; -} - -/* Returns an atomic_uint type based on the original uint type. The returned - * type will be equivalent to the original one but will have an atomic_uint - * type as leaf instead of an uint. - * - * Manages uint scalars, arrays, and arrays of arrays of any nested depth. - */ -static const struct glsl_type * -repair_atomic_type(const struct glsl_type *type) -{ - assert(glsl_get_base_type(glsl_without_array(type)) == GLSL_TYPE_UINT); - assert(glsl_type_is_scalar(glsl_without_array(type))); - - if (glsl_type_is_array(type)) { - const struct glsl_type *atomic = - repair_atomic_type(glsl_get_array_element(type)); - - return glsl_array_type(atomic, glsl_get_length(type), - glsl_get_explicit_stride(type)); - } else { - return glsl_atomic_uint_type(); - } -} - nir_deref_instr * vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr) { - if (b->wa_glslang_179) { - /* Do on-the-fly copy propagation for samplers. */ - if (ptr->var && ptr->var->copy_prop_sampler) - return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler); - } - vtn_assert(!vtn_pointer_uses_ssa_offset(b, ptr)); if (!ptr->deref) { struct vtn_access_chain chain = { @@ -655,11 +695,7 @@ vtn_local_load(struct vtn_builder *b, nir_deref_instr *src, if (src_tail != src) { val->type = src->type; - if (nir_src_is_const(src->arr.index)) - val->def = vtn_vector_extract(b, val->def, - nir_src_as_uint(src->arr.index)); - else - val->def = vtn_vector_extract_dynamic(b, val->def, src->arr.index.ssa); + val->def = nir_vector_extract(&b->nb, val->def, src->arr.index.ssa); } return val; @@ -675,12 +711,8 @@ vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); _vtn_local_load_store(b, true, dest_tail, val, access); - if (nir_src_is_const(dest->arr.index)) - val->def = vtn_vector_insert(b, val->def, src->def, - nir_src_as_uint(dest->arr.index)); - else - val->def = vtn_vector_insert_dynamic(b, val->def, src->def, - dest->arr.index.ssa); + val->def = nir_vector_insert(&b->nb, val->def, src->def, + dest->arr.index.ssa); _vtn_local_load_store(b, false, dest_tail, val, access); } else { _vtn_local_load_store(b, false, dest_tail, src, access); @@ -828,9 +860,6 @@ _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load, struct vtn_type *type, enum gl_access_qualifier access, struct vtn_ssa_value **inout) { - if (load && *inout == NULL) - *inout = vtn_create_ssa_value(b, type->type); - enum glsl_base_type base_type = glsl_get_base_type(type->type); switch (base_type) { case GLSL_TYPE_UINT: @@ -985,7 +1014,7 @@ vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src) nir_ssa_def *offset, *index = NULL; offset = vtn_pointer_to_offset(b, src, &index); - struct vtn_ssa_value *value = NULL; + struct vtn_ssa_value *value = vtn_create_ssa_value(b, src->type->type); _vtn_block_load_store(b, op, true, index, offset, access_offset, access_size, src->type, src->access, &value); @@ -1021,6 +1050,25 @@ _vtn_variable_load_store(struct vtn_builder *b, bool load, enum gl_access_qualifier access, struct vtn_ssa_value **inout) { + if (ptr->mode == vtn_variable_mode_uniform) { + if (ptr->type->base_type == vtn_base_type_image || + ptr->type->base_type == vtn_base_type_sampler) { + /* See also our handling of OpTypeSampler and OpTypeImage */ + vtn_assert(load); + (*inout)->def = vtn_pointer_to_ssa(b, ptr); + return; + } else if (ptr->type->base_type == vtn_base_type_sampled_image) { + /* See also our handling of OpTypeSampledImage */ + vtn_assert(load); + struct vtn_sampled_image si = { + .image = vtn_pointer_to_deref(b, ptr), + .sampler = vtn_pointer_to_deref(b, ptr), + }; + (*inout)->def = vtn_sampled_image_to_nir_ssa(b, si); + return; + } + } + enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type); switch (base_type) { case GLSL_TYPE_UINT: @@ -1038,18 +1086,17 @@ _vtn_variable_load_store(struct vtn_builder *b, bool load, if (glsl_type_is_vector_or_scalar(ptr->type->type)) { /* We hit a vector or scalar; go ahead and emit the load[s] */ nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr); - if (vtn_pointer_is_external_block(b, ptr)) { - /* If it's external, we call nir_load/store_deref directly. The - * vtn_local_load/store helpers are too clever and do magic to - * avoid array derefs of vectors. That magic is both less - * efficient than the direct load/store and, in the case of + if (vtn_mode_is_cross_invocation(b, ptr->mode)) { + /* If it's cross-invocation, we call nir_load/store_deref + * directly. The vtn_local_load/store helpers are too clever and + * do magic to avoid array derefs of vectors. That magic is both + * less efficient than the direct load/store and, in the case of * stores, is broken because it creates a race condition if two * threads are writing to different components of the same vector * due to the load+insert+store it uses to emulate the array * deref. */ if (load) { - *inout = vtn_create_ssa_value(b, ptr->type->type); (*inout)->def = nir_load_deref_with_access(&b->nb, deref, ptr->type->access | access); } else { @@ -1071,13 +1118,6 @@ _vtn_variable_load_store(struct vtn_builder *b, bool load, case GLSL_TYPE_ARRAY: case GLSL_TYPE_STRUCT: { unsigned elems = glsl_get_length(ptr->type->type); - if (load) { - vtn_assert(*inout == NULL); - *inout = rzalloc(b, struct vtn_ssa_value); - (*inout)->type = ptr->type->type; - (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems); - } - struct vtn_access_chain chain = { .length = 1, .link = { @@ -1104,7 +1144,7 @@ vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src) if (vtn_pointer_uses_ssa_offset(b, src)) { return vtn_block_load(b, src); } else { - struct vtn_ssa_value *val = NULL; + struct vtn_ssa_value *val = vtn_create_ssa_value(b, src->type->type); _vtn_variable_load_store(b, true, src, src->access, &val); return val; } @@ -1127,7 +1167,8 @@ static void _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, struct vtn_pointer *src) { - vtn_assert(src->type->type == dest->type->type); + vtn_assert(glsl_get_bare_type(src->type->type) == + glsl_get_bare_type(dest->type->type)); enum glsl_base_type base_type = glsl_get_base_type(src->type->type); switch (base_type) { case GLSL_TYPE_UINT: @@ -1288,8 +1329,13 @@ vtn_get_builtin_location(struct vtn_builder *b, set_mode_system_value(b, mode); break; case SpvBuiltInFragCoord: - *location = VARYING_SLOT_POS; vtn_assert(*mode == nir_var_shader_in); + if (b->options && b->options->frag_coord_is_sysval) { + *mode = nir_var_system_value; + *location = SYSTEM_VALUE_FRAG_COORD; + } else { + *location = VARYING_SLOT_POS; + } break; case SpvBuiltInPointCoord: *location = VARYING_SLOT_PNTC; @@ -1351,11 +1397,18 @@ vtn_get_builtin_location(struct vtn_builder *b, *location = SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX; set_mode_system_value(b, mode); break; + case SpvBuiltInGlobalOffset: + *location = SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID; + set_mode_system_value(b, mode); + break; case SpvBuiltInBaseVertex: /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same - * semantic as SPIR-V BaseVertex (SYSTEM_VALUE_FIRST_VERTEX). + * semantic as Vulkan BaseVertex (SYSTEM_VALUE_FIRST_VERTEX). */ - *location = SYSTEM_VALUE_FIRST_VERTEX; + if (b->options->environment == NIR_SPIRV_OPENGL) + *location = SYSTEM_VALUE_BASE_VERTEX; + else + *location = SYSTEM_VALUE_FIRST_VERTEX; set_mode_system_value(b, mode); break; case SpvBuiltInBaseInstance: @@ -1387,8 +1440,13 @@ vtn_get_builtin_location(struct vtn_builder *b, set_mode_system_value(b, mode); break; case SpvBuiltInViewIndex: - *location = SYSTEM_VALUE_VIEW_INDEX; - set_mode_system_value(b, mode); + if (b->options && b->options->view_index_is_input) { + *location = VARYING_SLOT_VIEW_INDEX; + vtn_assert(*mode == nir_var_shader_in); + } else { + *location = SYSTEM_VALUE_VIEW_INDEX; + set_mode_system_value(b, mode); + } break; case SpvBuiltInSubgroupEqMask: *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK, @@ -1422,6 +1480,34 @@ vtn_get_builtin_location(struct vtn_builder *b, *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE; set_mode_system_value(b, mode); break; + case SpvBuiltInBaryCoordNoPerspAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL; + set_mode_system_value(b, mode); + break; + case SpvBuiltInBaryCoordNoPerspCentroidAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID; + set_mode_system_value(b, mode); + break; + case SpvBuiltInBaryCoordNoPerspSampleAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE; + set_mode_system_value(b, mode); + break; + case SpvBuiltInBaryCoordSmoothAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL; + set_mode_system_value(b, mode); + break; + case SpvBuiltInBaryCoordSmoothCentroidAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID; + set_mode_system_value(b, mode); + break; + case SpvBuiltInBaryCoordSmoothSampleAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE; + set_mode_system_value(b, mode); + break; + case SpvBuiltInBaryCoordPullModelAMD: + *location = SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL; + set_mode_system_value(b, mode); + break; default: vtn_fail("Unsupported builtin: %s (%u)", spirv_builtin_to_string(builtin), builtin); @@ -1442,6 +1528,9 @@ apply_var_decoration(struct vtn_builder *b, case SpvDecorationFlat: var_data->interpolation = INTERP_MODE_FLAT; break; + case SpvDecorationExplicitInterpAMD: + var_data->interpolation = INTERP_MODE_EXPLICIT; + break; case SpvDecorationCentroid: var_data->centroid = true; break; @@ -1455,20 +1544,23 @@ apply_var_decoration(struct vtn_builder *b, var_data->read_only = true; break; case SpvDecorationNonReadable: - var_data->image.access |= ACCESS_NON_READABLE; + var_data->access |= ACCESS_NON_READABLE; break; case SpvDecorationNonWritable: var_data->read_only = true; - var_data->image.access |= ACCESS_NON_WRITEABLE; + var_data->access |= ACCESS_NON_WRITEABLE; break; case SpvDecorationRestrict: - var_data->image.access |= ACCESS_RESTRICT; + var_data->access |= ACCESS_RESTRICT; + break; + case SpvDecorationAliased: + var_data->access &= ~ACCESS_RESTRICT; break; case SpvDecorationVolatile: - var_data->image.access |= ACCESS_VOLATILE; + var_data->access |= ACCESS_VOLATILE; break; case SpvDecorationCoherent: - var_data->image.access |= ACCESS_COHERENT; + var_data->access |= ACCESS_COHERENT; break; case SpvDecorationComponent: var_data->location_frac = dec->operands[0]; @@ -1499,8 +1591,8 @@ apply_var_decoration(struct vtn_builder *b, case SpvDecorationRowMajor: case SpvDecorationColMajor: case SpvDecorationMatrixStride: - case SpvDecorationAliased: case SpvDecorationUniform: + case SpvDecorationUniformId: case SpvDecorationLinkageAttributes: break; /* Do nothing with these here */ @@ -1528,12 +1620,12 @@ apply_var_decoration(struct vtn_builder *b, case SpvDecorationXfbBuffer: var_data->explicit_xfb_buffer = true; - var_data->xfb_buffer = dec->operands[0]; + var_data->xfb.buffer = dec->operands[0]; var_data->always_active_io = true; break; case SpvDecorationXfbStride: var_data->explicit_xfb_stride = true; - var_data->xfb_stride = dec->operands[0]; + var_data->xfb.stride = dec->operands[0]; break; case SpvDecorationOffset: var_data->explicit_offset = true; @@ -1557,6 +1649,7 @@ apply_var_decoration(struct vtn_builder *b, break; case SpvDecorationUserSemantic: + case SpvDecorationUserTypeGOOGLE: /* User semantic decorations can safely be ignored by the driver. */ break; @@ -1617,7 +1710,7 @@ var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, break; case SpvDecorationCounterBuffer: /* Counter buffer decorations can safely be ignored by the driver. */ - break; + return; default: break; } @@ -1690,29 +1783,11 @@ var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, */ vtn_assert(vtn_var->mode == vtn_variable_mode_ubo || vtn_var->mode == vtn_variable_mode_ssbo || - vtn_var->mode == vtn_variable_mode_push_constant || - (vtn_var->mode == vtn_variable_mode_workgroup && - b->options->lower_workgroup_access_to_offsets)); + vtn_var->mode == vtn_variable_mode_push_constant); } } } -static void -ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, - const struct vtn_decoration *dec, void *void_ptr) -{ - struct vtn_pointer *ptr = void_ptr; - - switch (dec->decoration) { - case SpvDecorationNonUniformEXT: - ptr->access |= ACCESS_NON_UNIFORM; - break; - - default: - break; - } -} - enum vtn_variable_mode vtn_storage_class_to_mode(struct vtn_builder *b, SpvStorageClass class, @@ -1740,13 +1815,23 @@ vtn_storage_class_to_mode(struct vtn_builder *b, mode = vtn_variable_mode_ssbo; nir_mode = nir_var_mem_ssbo; break; - case SpvStorageClassPhysicalStorageBufferEXT: + case SpvStorageClassPhysicalStorageBuffer: mode = vtn_variable_mode_phys_ssbo; nir_mode = nir_var_mem_global; break; case SpvStorageClassUniformConstant: - mode = vtn_variable_mode_uniform; - nir_mode = nir_var_uniform; + if (b->shader->info.stage == MESA_SHADER_KERNEL) { + if (b->options->constant_as_global) { + mode = vtn_variable_mode_cross_workgroup; + nir_mode = nir_var_mem_global; + } else { + mode = vtn_variable_mode_ubo; + nir_mode = nir_var_mem_ubo; + } + } else { + mode = vtn_variable_mode_uniform; + nir_mode = nir_var_uniform; + } break; case SpvStorageClassPushConstant: mode = vtn_variable_mode_push_constant; @@ -1773,7 +1858,7 @@ vtn_storage_class_to_mode(struct vtn_builder *b, nir_mode = nir_var_mem_shared; break; case SpvStorageClassAtomicCounter: - mode = vtn_variable_mode_uniform; + mode = vtn_variable_mode_atomic_counter; nir_mode = nir_var_uniform; break; case SpvStorageClassCrossWorkgroup: @@ -1825,6 +1910,7 @@ vtn_mode_to_address_format(struct vtn_builder *b, enum vtn_variable_mode mode) case vtn_variable_mode_private: case vtn_variable_mode_uniform: + case vtn_variable_mode_atomic_counter: case vtn_variable_mode_input: case vtn_variable_mode_output: case vtn_variable_mode_image: @@ -1870,10 +1956,10 @@ vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr) /* In this case, we're looking for a block index and not an actual * deref. * - * For PhysicalStorageBufferEXT pointers, we don't have a block index + * For PhysicalStorageBuffer pointers, we don't have a block index * at all because we get the pointer directly from the client. This * assumes that there will never be a SSBO binding variable using the - * PhysicalStorageBufferEXT storage class. This assumption appears + * PhysicalStorageBuffer storage class. This assumption appears * to be correct according to the Vulkan spec because the table, * "Shader Resource and Storage Class Correspondence," the only the * Uniform storage class with BufferBlock or the StorageBuffer @@ -1914,20 +2000,6 @@ vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, ptr->type = ptr_type->deref; ptr->ptr_type = ptr_type; - if (b->wa_glslang_179) { - /* To work around https://github.com/KhronosGroup/glslang/issues/179 we - * need to whack the mode because it creates a function parameter with - * the Function storage class even though it's a pointer to a sampler. - * If we don't do this, then NIR won't get rid of the deref_cast for us. - */ - if (ptr->mode == vtn_variable_mode_function && - (ptr->type->base_type == vtn_base_type_sampler || - ptr->type->base_type == vtn_base_type_sampled_image)) { - ptr->mode = vtn_variable_mode_uniform; - nir_mode = nir_var_uniform; - } - } - if (vtn_pointer_uses_ssa_offset(b, ptr)) { /* This pointer type needs to have actual storage */ vtn_assert(ptr_type->type); @@ -1942,7 +2014,8 @@ vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, ptr->offset = ssa; } } else { - const struct glsl_type *deref_type = ptr_type->deref->type; + const struct glsl_type *deref_type = + vtn_type_get_nir_type(b, ptr_type->deref, ptr->mode); if (!vtn_pointer_is_external_block(b, ptr)) { ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, deref_type, ptr_type->stride); @@ -1957,18 +2030,17 @@ vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, /* This is a pointer to something internal or a pointer inside a * block. It's just a regular cast. * - * For PhysicalStorageBufferEXT pointers, we don't have a block index + * For PhysicalStorageBuffer pointers, we don't have a block index * at all because we get the pointer directly from the client. This * assumes that there will never be a SSBO binding variable using the - * PhysicalStorageBufferEXT storage class. This assumption appears + * PhysicalStorageBuffer storage class. This assumption appears * to be correct according to the Vulkan spec because the table, * "Shader Resource and Storage Class Correspondence," the only the * Uniform storage class with BufferBlock or the StorageBuffer * storage class with Block can be used. */ ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, - ptr_type->deref->type, - ptr_type->stride); + deref_type, ptr_type->stride); ptr->deref->dest.ssa.num_components = glsl_get_vector_elements(ptr_type->type); ptr->deref->dest.ssa.bit_size = glsl_get_bit_size(ptr_type->type); @@ -2043,7 +2115,7 @@ assign_missing_member_locations(struct vtn_variable *var) static void vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, struct vtn_type *ptr_type, SpvStorageClass storage_class, - nir_constant *initializer) + nir_constant *const_initializer, nir_variable *var_initializer) { vtn_assert(ptr_type->base_type == vtn_base_type_pointer); struct vtn_type *type = ptr_type->deref; @@ -2079,10 +2151,12 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, b->shader->info.num_ssbos++; break; case vtn_variable_mode_uniform: - if (glsl_type_is_image(without_array->type)) - b->shader->info.num_images++; - else if (glsl_type_is_sampler(without_array->type)) - b->shader->info.num_textures++; + if (without_array->base_type == vtn_base_type_image) { + if (glsl_type_is_image(without_array->glsl_image)) + b->shader->info.num_images++; + else if (glsl_type_is_sampler(without_array->glsl_image)) + b->shader->info.num_textures++; + } break; case vtn_variable_mode_push_constant: b->shader->num_uniforms = vtn_type_block_size(b, type); @@ -2094,7 +2168,7 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, case vtn_variable_mode_phys_ssbo: vtn_fail("Cannot create a variable with the " - "PhysicalStorageBufferEXT storage class"); + "PhysicalStorageBuffer storage class"); break; default: @@ -2107,48 +2181,46 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, var->mode = mode; var->base_location = -1; - vtn_assert(val->value_type == vtn_value_type_pointer); - val->pointer = vtn_pointer_for_variable(b, var, ptr_type); + val->pointer = rzalloc(b, struct vtn_pointer); + val->pointer->mode = var->mode; + val->pointer->type = var->type; + val->pointer->ptr_type = ptr_type; + val->pointer->var = var; + val->pointer->access = var->type->access; switch (var->mode) { case vtn_variable_mode_function: case vtn_variable_mode_private: case vtn_variable_mode_uniform: + case vtn_variable_mode_atomic_counter: /* For these, we create the variable normally */ var->var = rzalloc(b->shader, nir_variable); var->var->name = ralloc_strdup(var->var, val->name); - - if (storage_class == SpvStorageClassAtomicCounter) { - /* Need to tweak the nir type here as at vtn_handle_type we don't - * have the access to storage_class, that is the one that points us - * that is an atomic uint. - */ - var->var->type = repair_atomic_type(var->type->type); - } else { - /* Private variables don't have any explicit layout but some layouts - * may have leaked through due to type deduplication in the SPIR-V. - */ - var->var->type = var->type->type; - } + var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); var->var->data.mode = nir_mode; var->var->data.location = -1; var->var->interface_type = NULL; break; + case vtn_variable_mode_ubo: + case vtn_variable_mode_ssbo: + var->var = rzalloc(b->shader, nir_variable); + var->var->name = ralloc_strdup(var->var, val->name); + + var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); + var->var->interface_type = var->var->type; + + var->var->data.mode = nir_mode; + var->var->data.location = -1; + + break; + case vtn_variable_mode_workgroup: - if (b->options->lower_workgroup_access_to_offsets) { - var->shared_location = -1; - } else { - /* Create the variable normally */ - var->var = rzalloc(b->shader, nir_variable); - var->var->name = ralloc_strdup(var->var, val->name); - /* Workgroup variables don't have any explicit layout but some - * layouts may have leaked through due to type deduplication in the - * SPIR-V. - */ - var->var->type = var->type->type; - var->var->data.mode = nir_var_mem_shared; - } + /* Create the variable normally */ + var->var = rzalloc(b->shader, nir_variable); + var->var->name = ralloc_strdup(var->var, val->name); + var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); + var->var->data.mode = nir_var_mem_shared; break; case vtn_variable_mode_input: @@ -2199,12 +2271,7 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, var->var = rzalloc(b->shader, nir_variable); var->var->name = ralloc_strdup(var->var, val->name); - /* In Vulkan, shader I/O variables don't have any explicit layout but - * some layouts may have leaked through due to type deduplication in - * the SPIR-V. We do, however, keep the layouts in the variable's - * interface_type because we need offsets for XFB arrays of blocks. - */ - var->var->type = var->type->type; + var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); var->var->data.mode = nir_mode; var->var->data.patch = var->patch; @@ -2222,7 +2289,8 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, iface_type = iface_type->array_element; } if (iface_type->base_type == vtn_base_type_struct && iface_type->block) - var->var->interface_type = iface_type->type; + var->var->interface_type = vtn_type_get_nir_type(b, iface_type, + var->mode); if (per_vertex_type->base_type == vtn_base_type_struct && per_vertex_type->block) { @@ -2247,8 +2315,6 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, break; } - case vtn_variable_mode_ubo: - case vtn_variable_mode_ssbo: case vtn_variable_mode_push_constant: case vtn_variable_mode_cross_workgroup: /* These don't need actual variables. */ @@ -2259,21 +2325,37 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, unreachable("Should have been caught before"); } - if (initializer) { + /* We can only have one type of initializer */ + assert(!(const_initializer && var_initializer)); + if (const_initializer) { var->var->constant_initializer = - nir_constant_clone(initializer, var->var); + nir_constant_clone(const_initializer, var->var); + } + if (var_initializer) + var->var->pointer_initializer = var_initializer; + + if (var->mode == vtn_variable_mode_uniform || + var->mode == vtn_variable_mode_ssbo) { + /* SSBOs and images are assumed to not alias in the Simple, GLSL and Vulkan memory models */ + var->var->data.access |= b->mem_model != SpvMemoryModelOpenCL ? ACCESS_RESTRICT : 0; } vtn_foreach_decoration(b, val, var_decoration_cb, var); vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer); + /* Propagate access flags from the OpVariable decorations. */ + val->pointer->access |= var->access; + if ((var->mode == vtn_variable_mode_input || var->mode == vtn_variable_mode_output) && var->var->members) { assign_missing_member_locations(var); } - if (var->mode == vtn_variable_mode_uniform) { + if (var->mode == vtn_variable_mode_uniform || + var->mode == vtn_variable_mode_ubo || + var->mode == vtn_variable_mode_ssbo || + var->mode == vtn_variable_mode_atomic_counter) { /* XXX: We still need the binding information in the nir_variable * for these. We should fix that. */ @@ -2283,7 +2365,7 @@ vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, var->var->data.index = var->input_attachment_index; var->var->data.offset = var->offset; - if (glsl_type_is_image(without_array->type)) + if (glsl_type_is_image(glsl_without_array(var->var->type))) var->var->data.image.format = without_array->image_format; } @@ -2365,6 +2447,83 @@ nir_sloppy_bitcast(nir_builder *b, nir_ssa_def *val, return nir_shrink_zero_pad_vec(b, val, num_components); } +static bool +vtn_get_mem_operands(struct vtn_builder *b, const uint32_t *w, unsigned count, + unsigned *idx, SpvMemoryAccessMask *access, unsigned *alignment, + SpvScope *dest_scope, SpvScope *src_scope) +{ + *access = 0; + *alignment = 0; + if (*idx >= count) + return false; + + *access = w[(*idx)++]; + if (*access & SpvMemoryAccessAlignedMask) { + vtn_assert(*idx < count); + *alignment = w[(*idx)++]; + } + + if (*access & SpvMemoryAccessMakePointerAvailableMask) { + vtn_assert(*idx < count); + vtn_assert(dest_scope); + *dest_scope = vtn_constant_uint(b, w[(*idx)++]); + } + + if (*access & SpvMemoryAccessMakePointerVisibleMask) { + vtn_assert(*idx < count); + vtn_assert(src_scope); + *src_scope = vtn_constant_uint(b, w[(*idx)++]); + } + + return true; +} + +SpvMemorySemanticsMask +vtn_mode_to_memory_semantics(enum vtn_variable_mode mode) +{ + switch (mode) { + case vtn_variable_mode_ssbo: + case vtn_variable_mode_phys_ssbo: + return SpvMemorySemanticsUniformMemoryMask; + case vtn_variable_mode_workgroup: + return SpvMemorySemanticsWorkgroupMemoryMask; + case vtn_variable_mode_cross_workgroup: + return SpvMemorySemanticsCrossWorkgroupMemoryMask; + case vtn_variable_mode_atomic_counter: + return SpvMemorySemanticsAtomicCounterMemoryMask; + case vtn_variable_mode_image: + return SpvMemorySemanticsImageMemoryMask; + case vtn_variable_mode_output: + return SpvMemorySemanticsOutputMemoryMask; + default: + return SpvMemorySemanticsMaskNone; + } +} + +static void +vtn_emit_make_visible_barrier(struct vtn_builder *b, SpvMemoryAccessMask access, + SpvScope scope, enum vtn_variable_mode mode) +{ + if (!(access & SpvMemoryAccessMakePointerVisibleMask)) + return; + + vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeVisibleMask | + SpvMemorySemanticsAcquireMask | + vtn_mode_to_memory_semantics(mode)); +} + +static void +vtn_emit_make_available_barrier(struct vtn_builder *b, SpvMemoryAccessMask access, + SpvScope scope, enum vtn_variable_mode mode) +{ + if (!(access & SpvMemoryAccessMakePointerAvailableMask)) + return; + + vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeAvailableMask | + SpvMemorySemanticsReleaseMask | + vtn_mode_to_memory_semantics(mode)); +} + void vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, const uint32_t *w, unsigned count) @@ -2372,21 +2531,61 @@ vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, switch (opcode) { case SpvOpUndef: { struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); - val->type = vtn_value(b, w[1], vtn_value_type_type)->type; + val->type = vtn_get_type(b, w[1]); break; } case SpvOpVariable: { - struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; + struct vtn_type *ptr_type = vtn_get_type(b, w[1]); struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); SpvStorageClass storage_class = w[3]; - nir_constant *initializer = NULL; - if (count > 4) - initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant; + nir_constant *const_initializer = NULL; + nir_variable *var_initializer = NULL; + if (count > 4) { + struct vtn_value *init = vtn_untyped_value(b, w[4]); + switch (init->value_type) { + case vtn_value_type_constant: + const_initializer = init->constant; + break; + case vtn_value_type_pointer: + var_initializer = init->pointer->var->var; + break; + default: + vtn_fail("SPIR-V variable initializer %u must be constant or pointer", + w[4]); + } + } + + vtn_create_variable(b, val, ptr_type, storage_class, const_initializer, var_initializer); + + break; + } + + case SpvOpConstantSampler: { + /* Synthesize a pointer-to-sampler type, create a variable of that type, + * and give the variable a constant initializer with the sampler params */ + struct vtn_type *sampler_type = vtn_value(b, w[1], vtn_value_type_type)->type; + struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); + + struct vtn_type *ptr_type = rzalloc(b, struct vtn_type); + ptr_type = rzalloc(b, struct vtn_type); + ptr_type->base_type = vtn_base_type_pointer; + ptr_type->deref = sampler_type; + ptr_type->storage_class = SpvStorageClassUniform; + + ptr_type->type = nir_address_format_to_glsl_type( + vtn_mode_to_address_format(b, vtn_variable_mode_function)); + + vtn_create_variable(b, val, ptr_type, ptr_type->storage_class, NULL, NULL); + + nir_variable *nir_var = val->pointer->var->var; + nir_var->data.sampler.is_inline_sampler = true; + nir_var->data.sampler.addressing_mode = w[3]; + nir_var->data.sampler.normalized_coordinates = w[4]; + nir_var->data.sampler.filter_mode = w[5]; - vtn_create_variable(b, val, ptr_type, storage_class, initializer); break; } @@ -2395,6 +2594,7 @@ vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, case SpvOpInBoundsAccessChain: case SpvOpInBoundsPtrAccessChain: { struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4); + enum gl_access_qualifier access = 0; chain->ptr_as_array = (opcode == SpvOpPtrAccessChain || opcode == SpvOpInBoundsPtrAccessChain); unsigned idx = 0; @@ -2402,60 +2602,21 @@ vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, struct vtn_value *link_val = vtn_untyped_value(b, w[i]); if (link_val->value_type == vtn_value_type_constant) { chain->link[idx].mode = vtn_access_mode_literal; - const unsigned bit_size = glsl_get_bit_size(link_val->type->type); - switch (bit_size) { - case 8: - chain->link[idx].id = link_val->constant->values[0][0].i8; - break; - case 16: - chain->link[idx].id = link_val->constant->values[0][0].i16; - break; - case 32: - chain->link[idx].id = link_val->constant->values[0][0].i32; - break; - case 64: - chain->link[idx].id = link_val->constant->values[0][0].i64; - break; - default: - vtn_fail("Invalid bit size: %u", bit_size); - } + chain->link[idx].id = vtn_constant_int(b, w[i]); } else { chain->link[idx].mode = vtn_access_mode_id; chain->link[idx].id = w[i]; - } idx++; } - struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; - struct vtn_value *base_val = vtn_untyped_value(b, w[3]); - if (base_val->value_type == vtn_value_type_sampled_image) { - /* This is rather insane. SPIR-V allows you to use OpSampledImage - * to combine an array of images with a single sampler to get an - * array of sampled images that all share the same sampler. - * Fortunately, this means that we can more-or-less ignore the - * sampler when crawling the access chain, but it does leave us - * with this rather awkward little special-case. - */ - struct vtn_value *val = - vtn_push_value(b, w[2], vtn_value_type_sampled_image); - val->sampled_image = ralloc(b, struct vtn_sampled_image); - val->sampled_image->type = base_val->sampled_image->type; - val->sampled_image->image = - vtn_pointer_dereference(b, base_val->sampled_image->image, chain); - val->sampled_image->sampler = base_val->sampled_image->sampler; - vtn_foreach_decoration(b, val, ptr_decoration_cb, - val->sampled_image->image); - vtn_foreach_decoration(b, val, ptr_decoration_cb, - val->sampled_image->sampler); - } else { - vtn_assert(base_val->value_type == vtn_value_type_pointer); - struct vtn_value *val = - vtn_push_value(b, w[2], vtn_value_type_pointer); - val->pointer = vtn_pointer_dereference(b, base_val->pointer, chain); - val->pointer->ptr_type = ptr_type; - vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer); - } + struct vtn_type *ptr_type = vtn_get_type(b, w[1]); + struct vtn_pointer *base = + vtn_value(b, w[3], vtn_value_type_pointer)->pointer; + struct vtn_pointer *ptr = vtn_pointer_dereference(b, base, chain); + ptr->ptr_type = ptr_type; + ptr->access |= access; + vtn_push_pointer(b, w[2], ptr); break; } @@ -2465,25 +2626,40 @@ vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref); + unsigned idx = 3, dest_alignment, src_alignment; + SpvMemoryAccessMask dest_access, src_access; + SpvScope dest_scope, src_scope; + vtn_get_mem_operands(b, w, count, &idx, &dest_access, &dest_alignment, + &dest_scope, &src_scope); + if (!vtn_get_mem_operands(b, w, count, &idx, &src_access, &src_alignment, + NULL, &src_scope)) { + src_alignment = dest_alignment; + src_access = dest_access; + } + + vtn_emit_make_visible_barrier(b, src_access, src_scope, src->pointer->mode); + vtn_variable_copy(b, dest->pointer, src->pointer); + + vtn_emit_make_available_barrier(b, dest_access, dest_scope, dest->pointer->mode); break; } case SpvOpLoad: { - struct vtn_type *res_type = - vtn_value(b, w[1], vtn_value_type_type)->type; + struct vtn_type *res_type = vtn_get_type(b, w[1]); struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer); struct vtn_pointer *src = src_val->pointer; vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref); - if (glsl_type_is_image(res_type->type) || - glsl_type_is_sampler(res_type->type)) { - vtn_push_value(b, w[2], vtn_value_type_pointer)->pointer = src; - return; - } + unsigned idx = 4, alignment; + SpvMemoryAccessMask access; + SpvScope scope; + vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, NULL, &scope); + + vtn_emit_make_visible_barrier(b, access, scope, src->mode); - vtn_push_ssa(b, w[2], res_type, vtn_variable_load(b, src)); + vtn_push_ssa_value(b, w[2], vtn_variable_load(b, src)); break; } @@ -2517,21 +2693,15 @@ vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type); - if (glsl_type_is_sampler(dest->type->type)) { - if (b->wa_glslang_179) { - vtn_warn("OpStore of a sampler detected. Doing on-the-fly copy " - "propagation to workaround the problem."); - vtn_assert(dest->var->copy_prop_sampler == NULL); - dest->var->copy_prop_sampler = - vtn_value(b, w[2], vtn_value_type_pointer)->pointer; - } else { - vtn_fail("Vulkan does not allow OpStore of a sampler or image."); - } - break; - } + unsigned idx = 3, alignment; + SpvMemoryAccessMask access; + SpvScope scope; + vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, &scope, NULL); struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); vtn_variable_store(b, src, dest); + + vtn_emit_make_available_barrier(b, access, scope, dest->mode); break; } @@ -2576,44 +2746,46 @@ vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, nir_imm_int(&b->nb, 0u)), nir_imm_int(&b->nb, stride)); - struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); - val->ssa = vtn_create_ssa_value(b, glsl_uint_type()); - val->ssa->def = array_length; + vtn_push_nir_ssa(b, w[2], array_length); break; } case SpvOpConvertPtrToU: { - struct vtn_value *u_val = vtn_push_value(b, w[2], vtn_value_type_ssa); + struct vtn_type *u_type = vtn_get_type(b, w[1]); + struct vtn_type *ptr_type = vtn_get_value_type(b, w[3]); + + vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer || + ptr_type->type == NULL, + "OpConvertPtrToU can only be used on physical pointers"); - vtn_fail_if(u_val->type->base_type != vtn_base_type_vector && - u_val->type->base_type != vtn_base_type_scalar, + vtn_fail_if(u_type->base_type != vtn_base_type_vector && + u_type->base_type != vtn_base_type_scalar, "OpConvertPtrToU can only be used to cast to a vector or " "scalar type"); /* The pointer will be converted to an SSA value automatically */ - nir_ssa_def *ptr_ssa = vtn_ssa_value(b, w[3])->def; - - u_val->ssa = vtn_create_ssa_value(b, u_val->type->type); - u_val->ssa->def = nir_sloppy_bitcast(&b->nb, ptr_ssa, u_val->type->type); + nir_ssa_def *ptr = vtn_get_nir_ssa(b, w[3]); + nir_ssa_def *u = nir_sloppy_bitcast(&b->nb, ptr, u_type->type); + vtn_push_nir_ssa(b, w[2], u); break; } case SpvOpConvertUToPtr: { - struct vtn_value *ptr_val = - vtn_push_value(b, w[2], vtn_value_type_pointer); - struct vtn_value *u_val = vtn_value(b, w[3], vtn_value_type_ssa); + struct vtn_type *ptr_type = vtn_get_type(b, w[1]); + struct vtn_type *u_type = vtn_get_value_type(b, w[3]); - vtn_fail_if(ptr_val->type->type == NULL, + vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer || + ptr_type->type == NULL, "OpConvertUToPtr can only be used on physical pointers"); - vtn_fail_if(u_val->type->base_type != vtn_base_type_vector && - u_val->type->base_type != vtn_base_type_scalar, + vtn_fail_if(u_type->base_type != vtn_base_type_vector && + u_type->base_type != vtn_base_type_scalar, "OpConvertUToPtr can only be used to cast from a vector or " "scalar type"); - nir_ssa_def *ptr_ssa = nir_sloppy_bitcast(&b->nb, u_val->ssa->def, - ptr_val->type->type); - ptr_val->pointer = vtn_pointer_from_ssa(b, ptr_ssa, ptr_val->type); + nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]); + nir_ssa_def *ptr = nir_sloppy_bitcast(&b->nb, u, ptr_type->type); + vtn_push_pointer(b, w[2], vtn_pointer_from_ssa(b, ptr, ptr_type)); break; }