X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fnir%2Fnir_deref.c;h=f7a07c6b465d9a66a4d3d1a844b385cfb16f68c8;hb=84b3ef6a96eabc28b18e8cdf1b0d61826b1a8a67;hp=4a2e81956d1b106f8300e102e4be4dcc743325ba;hpb=16870de8a0aab97bd3534b660e7ea73e90ac11cd;p=mesa.git diff --git a/src/compiler/nir/nir_deref.c b/src/compiler/nir/nir_deref.c index 4a2e81956d1..f7a07c6b465 100644 --- a/src/compiler/nir/nir_deref.c +++ b/src/compiler/nir/nir_deref.c @@ -58,7 +58,7 @@ nir_deref_path_init(nir_deref_path *path, #ifndef NDEBUG /* Just in case someone uses short_path by accident */ for (unsigned i = 0; i < ARRAY_SIZE(path->_short_path); i++) - path->_short_path[i] = (void *)0xdeadbeef; + path->_short_path[i] = (void *)(uintptr_t)0xdeadbeef; #endif path->path = ralloc_array(mem_ctx, nir_deref_instr *, count + 1); @@ -70,7 +70,6 @@ nir_deref_path_init(nir_deref_path *path, done: assert(head == path->path); assert(tail == head + count); - assert((*head)->deref_type == nir_deref_type_var); assert(*tail == NULL); } @@ -111,7 +110,8 @@ nir_deref_instr_has_indirect(nir_deref_instr *instr) if (instr->deref_type == nir_deref_type_cast) return true; - if (instr->deref_type == nir_deref_type_array && + if ((instr->deref_type == nir_deref_type_array || + instr->deref_type == nir_deref_type_ptr_as_array) && !nir_src_is_const(instr->arr.index)) return true; @@ -121,12 +121,119 @@ nir_deref_instr_has_indirect(nir_deref_instr *instr) return false; } +bool +nir_deref_instr_is_known_out_of_bounds(nir_deref_instr *instr) +{ + for (; instr; instr = nir_deref_instr_parent(instr)) { + if (instr->deref_type == nir_deref_type_array && + nir_src_is_const(instr->arr.index) && + nir_src_as_uint(instr->arr.index) >= + glsl_get_length(nir_deref_instr_parent(instr)->type)) + return true; + } + + return false; +} + +bool +nir_deref_instr_has_complex_use(nir_deref_instr *deref) +{ + nir_foreach_use(use_src, &deref->dest.ssa) { + nir_instr *use_instr = use_src->parent_instr; + + switch (use_instr->type) { + case nir_instr_type_deref: { + nir_deref_instr *use_deref = nir_instr_as_deref(use_instr); + + /* A var deref has no sources */ + assert(use_deref->deref_type != nir_deref_type_var); + + /* If a deref shows up in an array index or something like that, it's + * a complex use. + */ + if (use_src != &use_deref->parent) + return true; + + /* Anything that isn't a basic struct or array deref is considered to + * be a "complex" use. In particular, we don't allow ptr_as_array + * because we assume that opt_deref will turn any non-complex + * ptr_as_array derefs into regular array derefs eventually so passes + * which only want to handle simple derefs will pick them up in a + * later pass. + */ + if (use_deref->deref_type != nir_deref_type_struct && + use_deref->deref_type != nir_deref_type_array_wildcard && + use_deref->deref_type != nir_deref_type_array) + return true; + + if (nir_deref_instr_has_complex_use(use_deref)) + return true; + + continue; + } + + case nir_instr_type_intrinsic: { + nir_intrinsic_instr *use_intrin = nir_instr_as_intrinsic(use_instr); + switch (use_intrin->intrinsic) { + case nir_intrinsic_load_deref: + assert(use_src == &use_intrin->src[0]); + continue; + + case nir_intrinsic_copy_deref: + assert(use_src == &use_intrin->src[0] || + use_src == &use_intrin->src[1]); + continue; + + case nir_intrinsic_store_deref: + /* A use in src[1] of a store means we're taking that pointer and + * writing it to a variable. Because we have no idea who will + * read that variable and what they will do with the pointer, it's + * considered a "complex" use. A use in src[0], on the other + * hand, is a simple use because we're just going to dereference + * it and write a value there. + */ + if (use_src == &use_intrin->src[0]) + continue; + return true; + + default: + return true; + } + unreachable("Switch default failed"); + } + + default: + return true; + } + } + + nir_foreach_if_use(use, &deref->dest.ssa) + return true; + + return false; +} + +unsigned +nir_deref_instr_ptr_as_array_stride(nir_deref_instr *deref) +{ + switch (deref->deref_type) { + case nir_deref_type_array: + return glsl_get_explicit_stride(nir_deref_instr_parent(deref)->type); + case nir_deref_type_ptr_as_array: + return nir_deref_instr_ptr_as_array_stride(nir_deref_instr_parent(deref)); + case nir_deref_type_cast: + return deref->cast.ptr_stride; + default: + return 0; + } +} + static unsigned type_get_array_stride(const struct glsl_type *elem_type, glsl_type_size_align_func size_align) { unsigned elem_size, elem_align; - glsl_get_natural_size_align_bytes(elem_type, &elem_size, &elem_align); + size_align(elem_type, &elem_size, &elem_align); return ALIGN_POT(elem_size, elem_align); } @@ -135,12 +242,11 @@ struct_type_get_field_offset(const struct glsl_type *struct_type, glsl_type_size_align_func size_align, unsigned field_idx) { - assert(glsl_type_is_struct(struct_type)); + assert(glsl_type_is_struct_or_ifc(struct_type)); unsigned offset = 0; for (unsigned i = 0; i <= field_idx; i++) { unsigned elem_size, elem_align; - glsl_get_natural_size_align_bytes(glsl_get_struct_field(struct_type, i), - &elem_size, &elem_align); + size_align(glsl_get_struct_field(struct_type, i), &elem_size, &elem_align); offset = ALIGN_POT(offset, elem_align); if (i < field_idx) offset += elem_size; @@ -190,16 +296,15 @@ nir_build_deref_offset(nir_builder *b, nir_deref_instr *deref, for (nir_deref_instr **p = &path.path[1]; *p; p++) { if ((*p)->deref_type == nir_deref_type_array) { nir_ssa_def *index = nir_ssa_for_src(b, (*p)->arr.index, 1); - nir_ssa_def *stride = - nir_imm_int(b, type_get_array_stride((*p)->type, size_align)); - offset = nir_iadd(b, offset, nir_imul(b, index, stride)); + int stride = type_get_array_stride((*p)->type, size_align); + offset = nir_iadd(b, offset, nir_imul_imm(b, index, stride)); } else if ((*p)->deref_type == nir_deref_type_struct) { /* p starts at path[1], so this is safe */ nir_deref_instr *parent = *(p - 1); unsigned field_offset = struct_type_get_field_offset(parent->type, size_align, (*p)->strct.index); - nir_iadd(b, offset, nir_imm_int(b, field_offset)); + offset = nir_iadd_imm(b, offset, field_offset); } else { unreachable("Unsupported deref type"); } @@ -255,6 +360,8 @@ nir_fixup_deref_modes(nir_shader *shader) continue; nir_deref_instr *deref = nir_instr_as_deref(instr); + if (deref->deref_type == nir_deref_type_cast) + continue; nir_variable_mode parent_mode; if (deref->deref_type == nir_deref_type_var) { @@ -272,12 +379,98 @@ nir_fixup_deref_modes(nir_shader *shader) } } +static bool +modes_may_alias(nir_variable_mode a, nir_variable_mode b) +{ + /* Generic pointers can alias with SSBOs */ + if ((a == nir_var_mem_ssbo || a == nir_var_mem_global) && + (b == nir_var_mem_ssbo || b == nir_var_mem_global)) + return true; + + /* In the general case, pointers can only alias if they have the same mode. + * + * NOTE: In future, with things like OpenCL generic pointers, this may not + * be true and will have to be re-evaluated. However, with graphics only, + * it should be safe. + */ + return a == b; +} + +static bool +deref_path_contains_coherent_decoration(nir_deref_path *path) +{ + assert(path->path[0]->deref_type == nir_deref_type_var); + + if (path->path[0]->var->data.image.access & ACCESS_COHERENT) + return true; + + for (nir_deref_instr **p = &path->path[1]; *p; p++) { + if ((*p)->deref_type != nir_deref_type_struct) + continue; + + const struct glsl_type *struct_type = (*(p - 1))->type; + const struct glsl_struct_field *field = + glsl_get_struct_field_data(struct_type, (*p)->strct.index); + if (field->memory_coherent) + return true; + } + + return false; +} + nir_deref_compare_result nir_compare_deref_paths(nir_deref_path *a_path, nir_deref_path *b_path) { - if (a_path->path[0]->var != b_path->path[0]->var) - return 0; + if (!modes_may_alias(b_path->path[0]->mode, a_path->path[0]->mode)) + return nir_derefs_do_not_alias; + + if (a_path->path[0]->deref_type != b_path->path[0]->deref_type) + return nir_derefs_may_alias_bit; + + if (a_path->path[0]->deref_type == nir_deref_type_var) { + if (a_path->path[0]->var != b_path->path[0]->var) { + /* Shader and function temporaries aren't backed by memory so two + * distinct variables never alias. + */ + static const nir_variable_mode temp_var_modes = + nir_var_shader_temp | nir_var_function_temp; + if ((a_path->path[0]->mode & temp_var_modes) || + (b_path->path[0]->mode & temp_var_modes)) + return nir_derefs_do_not_alias; + + /* If they are both declared coherent or have coherent somewhere in + * their path (due to a member of an interface being declared + * coherent), we have to assume we that we could have any kind of + * aliasing. Otherwise, they could still alias but the client didn't + * tell us and that's their fault. + */ + if (deref_path_contains_coherent_decoration(a_path) && + deref_path_contains_coherent_decoration(b_path)) + return nir_derefs_may_alias_bit; + + /* If we can chase the deref all the way back to the variable and + * they're not the same variable and at least one is not declared + * coherent, we know they can't possibly alias. + */ + return nir_derefs_do_not_alias; + } + } else { + assert(a_path->path[0]->deref_type == nir_deref_type_cast); + /* If they're not exactly the same cast, it's hard to compare them so we + * just assume they alias. Comparing casts is tricky as there are lots + * of things such as mode, type, etc. to make sure work out; for now, we + * just assume nit_opt_deref will combine them and compare the deref + * instructions. + * + * TODO: At some point in the future, we could be clever and understand + * that a float[] and int[] have the same layout and aliasing structure + * but double[] and vec3[] do not and we could potentially be a bit + * smarter here. + */ + if (a_path->path[0] != b_path->path[0]) + return nir_derefs_may_alias_bit; + } /* Start off assuming they fully compare. We ignore equality for now. In * the end, we'll determine that by containment. @@ -288,13 +481,35 @@ nir_compare_deref_paths(nir_deref_path *a_path, nir_deref_instr **a_p = &a_path->path[1]; nir_deref_instr **b_p = &b_path->path[1]; + while (*a_p != NULL && *a_p == *b_p) { + a_p++; + b_p++; + } + + /* We're at either the tail or the divergence point between the two deref + * paths. Look to see if either contains a ptr_as_array deref. It it + * does we don't know how to safely make any inferences. Hopefully, + * nir_opt_deref will clean most of these up and we can start inferring + * things again. + * + * In theory, we could do a bit better. For instance, we could detect the + * case where we have exactly one ptr_as_array deref in the chain after the + * divergence point and it's matched in both chains and the two chains have + * different constant indices. + */ + for (nir_deref_instr **t_p = a_p; *t_p; t_p++) { + if ((*t_p)->deref_type == nir_deref_type_ptr_as_array) + return nir_derefs_may_alias_bit; + } + for (nir_deref_instr **t_p = b_p; *t_p; t_p++) { + if ((*t_p)->deref_type == nir_deref_type_ptr_as_array) + return nir_derefs_may_alias_bit; + } + while (*a_p != NULL && *b_p != NULL) { nir_deref_instr *a_tail = *(a_p++); nir_deref_instr *b_tail = *(b_p++); - if (a_tail == b_tail) - continue; - switch (a_tail->deref_type) { case nir_deref_type_array: case nir_deref_type_array_wildcard: { @@ -319,7 +534,7 @@ nir_compare_deref_paths(nir_deref_path *a_path, */ if (nir_src_as_uint(a_tail->arr.index) != nir_src_as_uint(b_tail->arr.index)) - return 0; + return nir_derefs_do_not_alias; } else if (a_tail->arr.index.ssa == b_tail->arr.index.ssa) { /* They're the same indirect, continue on */ } else { @@ -335,7 +550,7 @@ nir_compare_deref_paths(nir_deref_path *a_path, case nir_deref_type_struct: { /* If they're different struct members, they don't even alias */ if (a_tail->strct.index != b_tail->strct.index) - return 0; + return nir_derefs_do_not_alias; break; } @@ -368,8 +583,10 @@ nir_compare_derefs(nir_deref_instr *a, nir_deref_instr *b) nir_deref_path a_path, b_path; nir_deref_path_init(&a_path, a, NULL); nir_deref_path_init(&b_path, b, NULL); - assert(a_path.path[0]->deref_type == nir_deref_type_var); - assert(b_path.path[0]->deref_type == nir_deref_type_var); + assert(a_path.path[0]->deref_type == nir_deref_type_var || + a_path.path[0]->deref_type == nir_deref_type_cast); + assert(b_path.path[0]->deref_type == nir_deref_type_var || + b_path.path[0]->deref_type == nir_deref_type_cast); nir_deref_compare_result result = nir_compare_deref_paths(&a_path, &b_path); @@ -394,8 +611,7 @@ rematerialize_deref_in_block(nir_deref_instr *deref, return deref; if (!state->cache) { - state->cache = _mesa_hash_table_create(NULL, _mesa_hash_pointer, - _mesa_key_pointer_equal); + state->cache = _mesa_pointer_hash_table_create(NULL); } struct hash_entry *cached = _mesa_hash_table_search(state->cache, deref); @@ -475,6 +691,8 @@ rematerialize_deref_src(nir_src *src, void *_state) * used. After this pass has been run, every use of a deref will be of a * deref in the same block as the use. Also, all unused derefs will be * deleted as a side-effect. + * + * Derefs used as sources of phi instructions are not rematerialized. */ bool nir_rematerialize_derefs_in_use_blocks_impl(nir_function_impl *impl) @@ -490,10 +708,15 @@ nir_rematerialize_derefs_in_use_blocks_impl(nir_function_impl *impl) _mesa_hash_table_clear(state.cache, NULL); nir_foreach_instr_safe(instr, block) { - if (instr->type == nir_instr_type_deref) { - nir_deref_instr_remove_if_unused(nir_instr_as_deref(instr)); + if (instr->type == nir_instr_type_deref && + nir_deref_instr_remove_if_unused(nir_instr_as_deref(instr))) + continue; + + /* If a deref is used in a phi, we can't rematerialize it, as the new + * derefs would appear before the phi, which is not valid. + */ + if (instr->type == nir_instr_type_phi) continue; - } state.builder.cursor = nir_before_instr(instr); nir_foreach_src(instr, rematerialize_deref_src, &state); @@ -510,3 +733,234 @@ nir_rematerialize_derefs_in_use_blocks_impl(nir_function_impl *impl) return state.progress; } + +static bool +is_trivial_deref_cast(nir_deref_instr *cast) +{ + nir_deref_instr *parent = nir_src_as_deref(cast->parent); + if (!parent) + return false; + + return cast->mode == parent->mode && + cast->type == parent->type && + cast->dest.ssa.num_components == parent->dest.ssa.num_components && + cast->dest.ssa.bit_size == parent->dest.ssa.bit_size; +} + +static bool +is_trivial_array_deref_cast(nir_deref_instr *cast) +{ + assert(is_trivial_deref_cast(cast)); + + nir_deref_instr *parent = nir_src_as_deref(cast->parent); + + if (parent->deref_type == nir_deref_type_array) { + return cast->cast.ptr_stride == + glsl_get_explicit_stride(nir_deref_instr_parent(parent)->type); + } else if (parent->deref_type == nir_deref_type_ptr_as_array) { + return cast->cast.ptr_stride == + nir_deref_instr_ptr_as_array_stride(parent); + } else { + return false; + } +} + +static bool +is_deref_ptr_as_array(nir_instr *instr) +{ + return instr->type == nir_instr_type_deref && + nir_instr_as_deref(instr)->deref_type == nir_deref_type_ptr_as_array; +} + +/** + * Remove casts that just wrap other casts. + */ +static bool +opt_remove_cast_cast(nir_deref_instr *cast) +{ + nir_deref_instr *first_cast = cast; + + while (true) { + nir_deref_instr *parent = nir_deref_instr_parent(first_cast); + if (parent == NULL || parent->deref_type != nir_deref_type_cast) + break; + first_cast = parent; + } + if (cast == first_cast) + return false; + + nir_instr_rewrite_src(&cast->instr, &cast->parent, + nir_src_for_ssa(first_cast->parent.ssa)); + return true; +} + +/** + * Is this casting a struct to a contained struct. + * struct a { struct b field0 }; + * ssa_5 is structa; + * deref_cast (structb *)ssa_5 (function_temp structb); + * converts to + * deref_struct &ssa_5->field0 (function_temp structb); + * This allows subsequent copy propagation to work. + */ +static bool +opt_replace_struct_wrapper_cast(nir_builder *b, nir_deref_instr *cast) +{ + nir_deref_instr *parent = nir_src_as_deref(cast->parent); + if (!parent) + return false; + + if (!glsl_type_is_struct(parent->type)) + return false; + + if (glsl_get_struct_field_offset(parent->type, 0) != 0) + return false; + + if (cast->type != glsl_get_struct_field(parent->type, 0)) + return false; + + nir_deref_instr *replace = nir_build_deref_struct(b, parent, 0); + nir_ssa_def_rewrite_uses(&cast->dest.ssa, nir_src_for_ssa(&replace->dest.ssa)); + nir_deref_instr_remove_if_unused(cast); + return true; +} + +static bool +opt_deref_cast(nir_builder *b, nir_deref_instr *cast) +{ + bool progress; + + if (opt_replace_struct_wrapper_cast(b, cast)) + return true; + + progress = opt_remove_cast_cast(cast); + if (!is_trivial_deref_cast(cast)) + return progress; + + bool trivial_array_cast = is_trivial_array_deref_cast(cast); + + assert(cast->dest.is_ssa); + assert(cast->parent.is_ssa); + + nir_foreach_use_safe(use_src, &cast->dest.ssa) { + /* If this isn't a trivial array cast, we can't propagate into + * ptr_as_array derefs. + */ + if (is_deref_ptr_as_array(use_src->parent_instr) && + !trivial_array_cast) + continue; + + nir_instr_rewrite_src(use_src->parent_instr, use_src, cast->parent); + progress = true; + } + + /* If uses would be a bit crazy */ + assert(list_empty(&cast->dest.ssa.if_uses)); + + nir_deref_instr_remove_if_unused(cast); + return progress; +} + +static bool +opt_deref_ptr_as_array(nir_builder *b, nir_deref_instr *deref) +{ + assert(deref->deref_type == nir_deref_type_ptr_as_array); + + nir_deref_instr *parent = nir_deref_instr_parent(deref); + + if (nir_src_is_const(deref->arr.index) && + nir_src_as_int(deref->arr.index) == 0) { + /* If it's a ptr_as_array deref with an index of 0, it does nothing + * and we can just replace its uses with its parent. + * + * The source of a ptr_as_array deref always has a deref_type of + * nir_deref_type_array or nir_deref_type_cast. If it's a cast, it + * may be trivial and we may be able to get rid of that too. Any + * trivial cast of trivial cast cases should be handled already by + * opt_deref_cast() above. + */ + if (parent->deref_type == nir_deref_type_cast && + is_trivial_deref_cast(parent)) + parent = nir_deref_instr_parent(parent); + nir_ssa_def_rewrite_uses(&deref->dest.ssa, + nir_src_for_ssa(&parent->dest.ssa)); + nir_instr_remove(&deref->instr); + return true; + } + + if (parent->deref_type != nir_deref_type_array && + parent->deref_type != nir_deref_type_ptr_as_array) + return false; + + assert(parent->parent.is_ssa); + assert(parent->arr.index.is_ssa); + assert(deref->arr.index.is_ssa); + + nir_ssa_def *new_idx = nir_iadd(b, parent->arr.index.ssa, + deref->arr.index.ssa); + + deref->deref_type = parent->deref_type; + nir_instr_rewrite_src(&deref->instr, &deref->parent, parent->parent); + nir_instr_rewrite_src(&deref->instr, &deref->arr.index, + nir_src_for_ssa(new_idx)); + return true; +} + +bool +nir_opt_deref_impl(nir_function_impl *impl) +{ + bool progress = false; + + nir_builder b; + nir_builder_init(&b, impl); + + nir_foreach_block(block, impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_deref) + continue; + + b.cursor = nir_before_instr(instr); + + nir_deref_instr *deref = nir_instr_as_deref(instr); + switch (deref->deref_type) { + case nir_deref_type_ptr_as_array: + if (opt_deref_ptr_as_array(&b, deref)) + progress = true; + break; + + case nir_deref_type_cast: + if (opt_deref_cast(&b, deref)) + progress = true; + break; + + default: + /* Do nothing */ + break; + } + } + } + + if (progress) { + nir_metadata_preserve(impl, nir_metadata_block_index | + nir_metadata_dominance); + } else { +#ifndef NDEBUG + impl->valid_metadata &= ~nir_metadata_not_properly_reset; +#endif + } + + return progress; +} + +bool +nir_opt_deref(nir_shader *shader) +{ + bool progress = false; + + nir_foreach_function(func, shader) { + if (func->impl && nir_opt_deref_impl(func->impl)) + progress = true; + } + + return progress; +}