X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fnir%2Fnir.c;h=d8ff265d9f3f8440e7033dd4fa6c714d8be0152f;hb=217def3ac6721bc7d8a1f99d2fc4501e5ee8627d;hp=bf7a5fa0b7e6bc799ed2f2379fdfe8a346b42835;hpb=f5804f17680e240eb6d14a306e0911b837c9a16a;p=mesa.git diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c index bf7a5fa0b7e..d8ff265d9f3 100644 --- a/src/compiler/nir/nir.c +++ b/src/compiler/nir/nir.c @@ -44,10 +44,7 @@ nir_shader_create(void *mem_ctx, { nir_shader *shader = rzalloc(mem_ctx, nir_shader); - exec_list_make_empty(&shader->uniforms); - exec_list_make_empty(&shader->inputs); - exec_list_make_empty(&shader->outputs); - exec_list_make_empty(&shader->shared); + exec_list_make_empty(&shader->variables); shader->options = options; @@ -59,8 +56,6 @@ nir_shader_create(void *mem_ctx, } exec_list_make_empty(&shader->functions); - exec_list_make_empty(&shader->globals); - exec_list_make_empty(&shader->system_values); shader->num_inputs = 0; shader->num_outputs = 0; @@ -108,45 +103,34 @@ void nir_shader_add_variable(nir_shader *shader, nir_variable *var) { switch (var->data.mode) { - case nir_var_all: - assert(!"invalid mode"); - break; - case nir_var_function_temp: assert(!"nir_shader_add_variable cannot be used for local variables"); - break; + return; case nir_var_shader_temp: - exec_list_push_tail(&shader->globals, &var->node); - break; - case nir_var_shader_in: - exec_list_push_tail(&shader->inputs, &var->node); - break; - case nir_var_shader_out: - exec_list_push_tail(&shader->outputs, &var->node); - break; - case nir_var_uniform: case nir_var_mem_ubo: case nir_var_mem_ssbo: - exec_list_push_tail(&shader->uniforms, &var->node); - break; - case nir_var_mem_shared: - assert(gl_shader_stage_is_compute(shader->info.stage)); - exec_list_push_tail(&shader->shared, &var->node); + case nir_var_system_value: break; case nir_var_mem_global: assert(!"nir_shader_add_variable cannot be used for global memory"); - break; + return; - case nir_var_system_value: - exec_list_push_tail(&shader->system_values, &var->node); - break; + case nir_var_mem_push_const: + assert(!"nir_var_push_constant is not supposed to be used for variables"); + return; + + default: + assert(!"invalid mode"); + return; } + + exec_list_push_tail(&shader->variables, &var->node); } nir_variable * @@ -187,6 +171,32 @@ nir_local_variable_create(nir_function_impl *impl, return var; } +nir_variable * +nir_find_variable_with_location(nir_shader *shader, + nir_variable_mode mode, + unsigned location) +{ + assert(util_bitcount(mode) == 1 && mode != nir_var_function_temp); + nir_foreach_variable_with_modes(var, shader, mode) { + if (var->data.location == location) + return var; + } + return NULL; +} + +nir_variable * +nir_find_variable_with_driver_location(nir_shader *shader, + nir_variable_mode mode, + unsigned location) +{ + assert(util_bitcount(mode) == 1 && mode != nir_var_function_temp); + nir_foreach_variable_with_modes(var, shader, mode) { + if (var->data.driver_location == location) + return var; + } + return NULL; +} + nir_function * nir_function_create(nir_shader *shader, const char *name) { @@ -285,6 +295,7 @@ nir_function_impl_create_bare(nir_shader *shader) impl->reg_alloc = 0; impl->ssa_alloc = 0; impl->valid_metadata = nir_metadata_none; + impl->structured = true; /* create start & end blocks */ nir_block *start_block = nir_block_create(shader); @@ -466,7 +477,10 @@ nir_jump_instr_create(nir_shader *shader, nir_jump_type type) { nir_jump_instr *instr = ralloc(shader, nir_jump_instr); instr_init(&instr->instr, nir_instr_type_jump); + src_init(&instr->condition); instr->type = type; + instr->target = NULL; + instr->else_target = NULL; return instr; } @@ -543,7 +557,6 @@ nir_tex_instr_create(nir_shader *shader, unsigned num_srcs) src_init(&instr->src[i].src); instr->texture_index = 0; - instr->texture_array_size = 0; instr->sampler_index = 0; memcpy(instr->tg4_offsets, default_tg4_offsets, sizeof(instr->tg4_offsets)); @@ -719,8 +732,6 @@ reduce_cursor(nir_cursor cursor) { switch (cursor.option) { case nir_cursor_before_block: - assert(nir_cf_node_prev(&cursor.block->cf_node) == NULL || - nir_cf_node_prev(&cursor.block->cf_node)->type != nir_cf_node_block); if (exec_list_is_empty(&cursor.block->instr_list)) { /* Empty block. After is as good as before. */ cursor.option = nir_cursor_after_block; @@ -1185,6 +1196,15 @@ visit_parallel_copy_src(nir_parallel_copy_instr *instr, return true; } +static bool +visit_jump_src(nir_jump_instr *instr, nir_foreach_src_cb cb, void *state) +{ + if (instr->type != nir_jump_goto_if) + return true; + + return visit_src(&instr->condition, cb, state); +} + typedef struct { void *state; nir_foreach_src_cb cb; @@ -1238,6 +1258,7 @@ nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state) return false; break; case nir_instr_type_jump: + return visit_jump_src(nir_instr_as_jump(instr), cb, state); case nir_instr_type_ssa_undef: return true; @@ -1252,6 +1273,32 @@ nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state) return nir_foreach_dest(instr, visit_dest_indirect, &dest_state); } +bool +nir_foreach_phi_src_leaving_block(nir_block *block, + nir_foreach_src_cb cb, + void *state) +{ + for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) { + if (block->successors[i] == NULL) + continue; + + nir_foreach_instr(instr, block->successors[i]) { + if (instr->type != nir_instr_type_phi) + break; + + nir_phi_instr *phi = nir_instr_as_phi(instr); + nir_foreach_phi_src(phi_src, phi) { + if (phi_src->pred == block) { + if (!cb(&phi_src->src, state)) + return false; + } + } + } + } + + return true; +} + nir_const_value nir_const_value_for_float(double f, unsigned bit_size) { @@ -1319,8 +1366,8 @@ nir_src_is_dynamically_uniform(nir_src src) /* As are uniform variables */ if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) { nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr); - - if (intr->intrinsic == nir_intrinsic_load_uniform) + if (intr->intrinsic == nir_intrinsic_load_uniform && + nir_src_is_dynamically_uniform(intr->src[0])) return true; } @@ -1416,7 +1463,7 @@ nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest) { if (dest->is_ssa) { /* We can only overwrite an SSA destination if it has no uses. */ - assert(list_empty(&dest->ssa.uses) && list_empty(&dest->ssa.if_uses)); + assert(list_is_empty(&dest->ssa.uses) && list_is_empty(&dest->ssa.if_uses)); } else { list_del(&dest->reg.def_link); if (dest->reg.indirect) @@ -1447,6 +1494,7 @@ nir_ssa_def_init(nir_instr *instr, nir_ssa_def *def, list_inithead(&def->if_uses); def->num_components = num_components; def->bit_size = bit_size; + def->divergent = true; /* This is the safer default */ if (instr->block) { nir_function_impl *impl = @@ -1547,12 +1595,39 @@ nir_ssa_def_components_read(const nir_ssa_def *def) } } - if (!list_empty(&def->if_uses)) + if (!list_is_empty(&def->if_uses)) read_mask |= 1; return read_mask; } +nir_block * +nir_block_unstructured_next(nir_block *block) +{ + if (block == NULL) { + /* nir_foreach_block_unstructured_safe() will call this function on a + * NULL block after the last iteration, but it won't use the result so + * just return NULL here. + */ + return NULL; + } + + nir_cf_node *cf_next = nir_cf_node_next(&block->cf_node); + if (cf_next == NULL && block->cf_node.parent->type == nir_cf_node_function) + return NULL; + + if (cf_next && cf_next->type == nir_cf_node_block) + return nir_cf_node_as_block(cf_next); + + return nir_block_cf_tree_next(block); +} + +nir_block * +nir_unstructured_start_block(nir_function_impl *impl) +{ + return nir_start_block(impl); +} + nir_block * nir_block_cf_tree_next(nir_block *block) { @@ -1564,6 +1639,8 @@ nir_block_cf_tree_next(nir_block *block) return NULL; } + assert(nir_cf_node_get_function(&block->cf_node)->structured); + nir_cf_node *cf_next = nir_cf_node_next(&block->cf_node); if (cf_next) return nir_cf_node_cf_tree_first(cf_next); @@ -1578,8 +1655,8 @@ nir_block_cf_tree_next(nir_block *block) return nir_if_first_else_block(if_stmt); assert(block == nir_if_last_else_block(if_stmt)); - /* fall through */ } + /* fallthrough */ case nir_cf_node_loop: return nir_cf_node_as_block(nir_cf_node_next(parent)); @@ -1600,6 +1677,8 @@ nir_block_cf_tree_prev(nir_block *block) return NULL; } + assert(nir_cf_node_get_function(&block->cf_node)->structured); + nir_cf_node *cf_prev = nir_cf_node_prev(&block->cf_node); if (cf_prev) return nir_cf_node_cf_tree_last(cf_prev); @@ -1614,8 +1693,8 @@ nir_block_cf_tree_prev(nir_block *block) return nir_if_last_then_block(if_stmt); assert(block == nir_if_first_then_block(if_stmt)); - /* fall through */ } + /* fallthrough */ case nir_cf_node_loop: return nir_cf_node_as_block(nir_cf_node_prev(parent)); @@ -1734,7 +1813,7 @@ nir_index_blocks(nir_function_impl *impl) if (impl->valid_metadata & nir_metadata_block_index) return; - nir_foreach_block(block, impl) { + nir_foreach_block_unstructured(block, impl) { block->index = index++; } @@ -1762,7 +1841,7 @@ nir_index_ssa_defs(nir_function_impl *impl) { unsigned index = 0; - nir_foreach_block(block, impl) { + nir_foreach_block_unstructured(block, impl) { nir_foreach_instr(instr, block) nir_foreach_ssa_def(instr, index_ssa_def_cb, &index); } @@ -1787,6 +1866,24 @@ nir_index_instrs(nir_function_impl *impl) return index; } +unsigned +nir_shader_index_vars(nir_shader *shader, nir_variable_mode modes) +{ + unsigned count = 0; + nir_foreach_variable_with_modes(var, shader, modes) + var->index = count++; + return count; +} + +unsigned +nir_function_impl_index_vars(nir_function_impl *impl) +{ + unsigned count = 0; + nir_foreach_function_temp_variable(var, impl) + var->index = count++; + return count; +} + static nir_instr * cursor_next_instr(nir_cursor cursor) { @@ -1823,9 +1920,10 @@ cursor_next_instr(nir_cursor cursor) unreachable("Inavlid cursor option"); } -static bool +ASSERTED static bool dest_is_ssa(nir_dest *dest, void *_state) { + (void) _state; return dest->is_ssa; } @@ -1838,7 +1936,7 @@ nir_function_impl_lower_instructions(nir_function_impl *impl, nir_builder b; nir_builder_init(&b, impl); - nir_metadata preserved = nir_metadata_block_index || + nir_metadata preserved = nir_metadata_block_index | nir_metadata_dominance; bool progress = false; @@ -1876,7 +1974,7 @@ nir_function_impl_lower_instructions(nir_function_impl *impl, b.cursor = nir_after_instr(instr); nir_ssa_def *new_def = lower(&b, instr, cb_data); - if (new_def) { + if (new_def && new_def != NIR_LOWER_INSTR_PROGRESS) { assert(old_def != NULL); if (new_def->parent_instr->block != instr->block) preserved = nir_metadata_none; @@ -1888,7 +1986,7 @@ nir_function_impl_lower_instructions(nir_function_impl *impl, list_for_each_entry_safe(nir_src, use_src, &old_if_uses, use_link) nir_if_rewrite_condition(use_src->parent_if, new_src); - if (list_empty(&old_def->uses) && list_empty(&old_def->if_uses)) { + if (list_is_empty(&old_def->uses) && list_is_empty(&old_def->if_uses)) { iter = nir_instr_remove(instr); } else { iter = nir_after_instr(instr); @@ -1901,15 +1999,16 @@ nir_function_impl_lower_instructions(nir_function_impl *impl, list_replace(&old_if_uses, &old_def->if_uses); } iter = nir_after_instr(instr); + + if (new_def == NIR_LOWER_INSTR_PROGRESS) + progress = true; } } if (progress) { nir_metadata_preserve(impl, preserved); } else { -#ifndef NDEBUG - impl->valid_metadata &= ~nir_metadata_not_properly_reset; -#endif + nir_metadata_preserve(impl, nir_metadata_all); } return progress; @@ -1959,6 +2058,8 @@ nir_intrinsic_from_system_value(gl_system_value val) return nir_intrinsic_load_frag_coord; case SYSTEM_VALUE_POINT_COORD: return nir_intrinsic_load_point_coord; + case SYSTEM_VALUE_LINE_COORD: + return nir_intrinsic_load_line_coord; case SYSTEM_VALUE_FRONT_FACE: return nir_intrinsic_load_front_face; case SYSTEM_VALUE_SAMPLE_ID: @@ -1983,6 +2084,10 @@ nir_intrinsic_from_system_value(gl_system_value val) return nir_intrinsic_load_tess_level_outer; case SYSTEM_VALUE_TESS_LEVEL_INNER: return nir_intrinsic_load_tess_level_inner; + case SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT: + return nir_intrinsic_load_tess_level_outer_default; + case SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT: + return nir_intrinsic_load_tess_level_inner_default; case SYSTEM_VALUE_VERTICES_IN: return nir_intrinsic_load_patch_vertices_in; case SYSTEM_VALUE_HELPER_INVOCATION: @@ -2019,6 +2124,8 @@ nir_intrinsic_from_system_value(gl_system_value val) return nir_intrinsic_load_global_invocation_index; case SYSTEM_VALUE_WORK_DIM: return nir_intrinsic_load_work_dim; + case SYSTEM_VALUE_USER_DATA_AMD: + return nir_intrinsic_load_user_data_amd; default: unreachable("system value does not directly correspond to intrinsic"); } @@ -2050,6 +2157,8 @@ nir_system_value_from_intrinsic(nir_intrinsic_op intrin) return SYSTEM_VALUE_FRAG_COORD; case nir_intrinsic_load_point_coord: return SYSTEM_VALUE_POINT_COORD; + case nir_intrinsic_load_line_coord: + return SYSTEM_VALUE_LINE_COORD; case nir_intrinsic_load_front_face: return SYSTEM_VALUE_FRONT_FACE; case nir_intrinsic_load_sample_id: @@ -2074,6 +2183,10 @@ nir_system_value_from_intrinsic(nir_intrinsic_op intrin) return SYSTEM_VALUE_TESS_LEVEL_OUTER; case nir_intrinsic_load_tess_level_inner: return SYSTEM_VALUE_TESS_LEVEL_INNER; + case nir_intrinsic_load_tess_level_outer_default: + return SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT; + case nir_intrinsic_load_tess_level_inner_default: + return SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT; case nir_intrinsic_load_patch_vertices_in: return SYSTEM_VALUE_VERTICES_IN; case nir_intrinsic_load_helper_invocation: @@ -2106,6 +2219,8 @@ nir_system_value_from_intrinsic(nir_intrinsic_op intrin) return SYSTEM_VALUE_LOCAL_GROUP_SIZE; case nir_intrinsic_load_global_invocation_id: return SYSTEM_VALUE_GLOBAL_INVOCATION_ID; + case nir_intrinsic_load_user_data_amd: + return SYSTEM_VALUE_USER_DATA_AMD; default: unreachable("intrinsic doesn't produce a system value"); } @@ -2127,14 +2242,14 @@ nir_remap_dual_slot_attributes(nir_shader *shader, uint64_t *dual_slot) assert(shader->info.stage == MESA_SHADER_VERTEX); *dual_slot = 0; - nir_foreach_variable(var, &shader->inputs) { + nir_foreach_shader_in_variable(var, shader) { if (glsl_type_is_dual_slot(glsl_without_array(var->type))) { unsigned slots = glsl_count_attribute_slots(var->type, true); *dual_slot |= BITFIELD64_MASK(slots) << var->data.location; } } - nir_foreach_variable(var, &shader->inputs) { + nir_foreach_shader_in_variable(var, shader) { var->data.location += util_bitcount64(*dual_slot & BITFIELD64_MASK(var->data.location)); } @@ -2159,6 +2274,8 @@ void nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src, bool bindless) { + enum gl_access_qualifier access = nir_intrinsic_access(intrin); + switch (intrin->intrinsic) { #define CASE(op) \ case nir_intrinsic_image_deref_##op: \ @@ -2168,14 +2285,18 @@ nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src, CASE(load) CASE(store) CASE(atomic_add) - CASE(atomic_min) - CASE(atomic_max) + CASE(atomic_imin) + CASE(atomic_umin) + CASE(atomic_imax) + CASE(atomic_umax) CASE(atomic_and) CASE(atomic_or) CASE(atomic_xor) CASE(atomic_exchange) CASE(atomic_comp_swap) CASE(atomic_fadd) + CASE(atomic_inc_wrap) + CASE(atomic_dec_wrap) CASE(size) CASE(samples) CASE(load_raw_intel) @@ -2190,8 +2311,20 @@ nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src, nir_intrinsic_set_image_dim(intrin, glsl_get_sampler_dim(deref->type)); nir_intrinsic_set_image_array(intrin, glsl_sampler_type_is_array(deref->type)); + nir_intrinsic_set_access(intrin, access | var->data.access); nir_intrinsic_set_format(intrin, var->data.image.format); nir_instr_rewrite_src(&intrin->instr, &intrin->src[0], nir_src_for_ssa(src)); } + +unsigned +nir_image_intrinsic_coord_components(const nir_intrinsic_instr *instr) +{ + enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr); + int coords = glsl_get_sampler_dim_coordinate_components(dim); + if (dim == GLSL_SAMPLER_DIM_CUBE) + return coords; + else + return coords + nir_intrinsic_image_array(instr); +}