*/
#include "nir.h"
+#include "nir_builder.h"
#include "nir_control_flow_private.h"
#include "util/half_float.h"
#include <limits.h>
nir_shader_add_variable(nir_shader *shader, nir_variable *var)
{
switch (var->data.mode) {
+ case nir_num_variable_modes:
case nir_var_all:
assert(!"invalid mode");
break;
case nir_var_system_value:
exec_list_push_tail(&shader->system_values, &var->node);
break;
+
+ case nir_var_mem_push_const:
+ assert(!"nir_var_push_constant is not supposed to be used for variables");
+ break;
}
}
src_init(&instr->src[i].src);
instr->texture_index = 0;
- instr->texture_array_size = 0;
instr->sampler_index = 0;
memcpy(instr->tg4_offsets, default_tg4_offsets, sizeof(instr->tg4_offsets));
return nir_foreach_dest(instr, visit_dest_indirect, &dest_state);
}
+bool
+nir_foreach_phi_src_leaving_block(nir_block *block,
+ nir_foreach_src_cb cb,
+ void *state)
+{
+ for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) {
+ if (block->successors[i] == NULL)
+ continue;
+
+ nir_foreach_instr(instr, block->successors[i]) {
+ if (instr->type != nir_instr_type_phi)
+ break;
+
+ nir_phi_instr *phi = nir_instr_as_phi(instr);
+ nir_foreach_phi_src(phi_src, phi) {
+ if (phi_src->pred == block) {
+ if (!cb(&phi_src->src, state))
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
nir_const_value
nir_const_value_for_float(double f, unsigned bit_size)
{
/* As are uniform variables */
if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr);
-
- if (intr->intrinsic == nir_intrinsic_load_uniform)
+ if (intr->intrinsic == nir_intrinsic_load_uniform &&
+ nir_src_is_dynamically_uniform(intr->src[0]))
return true;
}
{
if (dest->is_ssa) {
/* We can only overwrite an SSA destination if it has no uses. */
- assert(list_empty(&dest->ssa.uses) && list_empty(&dest->ssa.if_uses));
+ assert(list_is_empty(&dest->ssa.uses) && list_is_empty(&dest->ssa.if_uses));
} else {
list_del(&dest->reg.def_link);
if (dest->reg.indirect)
list_inithead(&def->if_uses);
def->num_components = num_components;
def->bit_size = bit_size;
+ def->divergent = true; /* This is the safer default */
if (instr->block) {
nir_function_impl *impl =
}
}
- if (!list_empty(&def->if_uses))
+ if (!list_is_empty(&def->if_uses))
read_mask |= 1;
return read_mask;
return nir_if_first_else_block(if_stmt);
assert(block == nir_if_last_else_block(if_stmt));
- /* fall through */
}
+ /* fallthrough */
case nir_cf_node_loop:
return nir_cf_node_as_block(nir_cf_node_next(parent));
return nir_if_last_then_block(if_stmt);
assert(block == nir_if_first_then_block(if_stmt));
- /* fall through */
}
+ /* fallthrough */
case nir_cf_node_loop:
return nir_cf_node_as_block(nir_cf_node_prev(parent));
return index;
}
+static void
+index_var_list(struct exec_list *list)
+{
+ unsigned next_index = 0;
+ nir_foreach_variable(var, list)
+ var->index = next_index++;
+}
+
+void
+nir_index_vars(nir_shader *shader, nir_function_impl *impl, nir_variable_mode modes)
+{
+ if ((modes & nir_var_function_temp) && impl)
+ index_var_list(&impl->locals);
+
+ if (modes & nir_var_shader_temp)
+ index_var_list(&shader->globals);
+
+ if (modes & nir_var_shader_in)
+ index_var_list(&shader->inputs);
+
+ if (modes & nir_var_shader_out)
+ index_var_list(&shader->outputs);
+
+ if (modes & (nir_var_uniform | nir_var_mem_ubo | nir_var_mem_ssbo))
+ index_var_list(&shader->uniforms);
+
+ if (modes & nir_var_mem_shared)
+ index_var_list(&shader->shared);
+
+ if (modes & nir_var_system_value)
+ index_var_list(&shader->system_values);
+}
+
+static nir_instr *
+cursor_next_instr(nir_cursor cursor)
+{
+ switch (cursor.option) {
+ case nir_cursor_before_block:
+ for (nir_block *block = cursor.block; block;
+ block = nir_block_cf_tree_next(block)) {
+ nir_instr *instr = nir_block_first_instr(block);
+ if (instr)
+ return instr;
+ }
+ return NULL;
+
+ case nir_cursor_after_block:
+ cursor.block = nir_block_cf_tree_next(cursor.block);
+ if (cursor.block == NULL)
+ return NULL;
+
+ cursor.option = nir_cursor_before_block;
+ return cursor_next_instr(cursor);
+
+ case nir_cursor_before_instr:
+ return cursor.instr;
+
+ case nir_cursor_after_instr:
+ if (nir_instr_next(cursor.instr))
+ return nir_instr_next(cursor.instr);
+
+ cursor.option = nir_cursor_after_block;
+ cursor.block = cursor.instr->block;
+ return cursor_next_instr(cursor);
+ }
+
+ unreachable("Inavlid cursor option");
+}
+
+ASSERTED static bool
+dest_is_ssa(nir_dest *dest, void *_state)
+{
+ (void) _state;
+ return dest->is_ssa;
+}
+
+bool
+nir_function_impl_lower_instructions(nir_function_impl *impl,
+ nir_instr_filter_cb filter,
+ nir_lower_instr_cb lower,
+ void *cb_data)
+{
+ nir_builder b;
+ nir_builder_init(&b, impl);
+
+ nir_metadata preserved = nir_metadata_block_index |
+ nir_metadata_dominance;
+
+ bool progress = false;
+ nir_cursor iter = nir_before_cf_list(&impl->body);
+ nir_instr *instr;
+ while ((instr = cursor_next_instr(iter)) != NULL) {
+ if (filter && !filter(instr, cb_data)) {
+ iter = nir_after_instr(instr);
+ continue;
+ }
+
+ assert(nir_foreach_dest(instr, dest_is_ssa, NULL));
+ nir_ssa_def *old_def = nir_instr_ssa_def(instr);
+ if (old_def == NULL) {
+ iter = nir_after_instr(instr);
+ continue;
+ }
+
+ /* We're about to ask the callback to generate a replacement for instr.
+ * Save off the uses from instr's SSA def so we know what uses to
+ * rewrite later. If we use nir_ssa_def_rewrite_uses, it fails in the
+ * case where the generated replacement code uses the result of instr
+ * itself. If we use nir_ssa_def_rewrite_uses_after (which is the
+ * normal solution to this problem), it doesn't work well if control-
+ * flow is inserted as part of the replacement, doesn't handle cases
+ * where the replacement is something consumed by instr, and suffers
+ * from performance issues. This is the only way to 100% guarantee
+ * that we rewrite the correct set efficiently.
+ */
+ struct list_head old_uses, old_if_uses;
+ list_replace(&old_def->uses, &old_uses);
+ list_inithead(&old_def->uses);
+ list_replace(&old_def->if_uses, &old_if_uses);
+ list_inithead(&old_def->if_uses);
+
+ b.cursor = nir_after_instr(instr);
+ nir_ssa_def *new_def = lower(&b, instr, cb_data);
+ if (new_def && new_def != NIR_LOWER_INSTR_PROGRESS) {
+ assert(old_def != NULL);
+ if (new_def->parent_instr->block != instr->block)
+ preserved = nir_metadata_none;
+
+ nir_src new_src = nir_src_for_ssa(new_def);
+ list_for_each_entry_safe(nir_src, use_src, &old_uses, use_link)
+ nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
+
+ list_for_each_entry_safe(nir_src, use_src, &old_if_uses, use_link)
+ nir_if_rewrite_condition(use_src->parent_if, new_src);
+
+ if (list_is_empty(&old_def->uses) && list_is_empty(&old_def->if_uses)) {
+ iter = nir_instr_remove(instr);
+ } else {
+ iter = nir_after_instr(instr);
+ }
+ progress = true;
+ } else {
+ /* We didn't end up lowering after all. Put the uses back */
+ if (old_def) {
+ list_replace(&old_uses, &old_def->uses);
+ list_replace(&old_if_uses, &old_def->if_uses);
+ }
+ iter = nir_after_instr(instr);
+
+ if (new_def == NIR_LOWER_INSTR_PROGRESS)
+ progress = true;
+ }
+ }
+
+ if (progress) {
+ nir_metadata_preserve(impl, preserved);
+ } else {
+ nir_metadata_preserve(impl, nir_metadata_all);
+ }
+
+ return progress;
+}
+
+bool
+nir_shader_lower_instructions(nir_shader *shader,
+ nir_instr_filter_cb filter,
+ nir_lower_instr_cb lower,
+ void *cb_data)
+{
+ bool progress = false;
+
+ nir_foreach_function(function, shader) {
+ if (function->impl &&
+ nir_function_impl_lower_instructions(function->impl,
+ filter, lower, cb_data))
+ progress = true;
+ }
+
+ return progress;
+}
+
nir_intrinsic_op
nir_intrinsic_from_system_value(gl_system_value val)
{
return nir_intrinsic_load_invocation_id;
case SYSTEM_VALUE_FRAG_COORD:
return nir_intrinsic_load_frag_coord;
+ case SYSTEM_VALUE_POINT_COORD:
+ return nir_intrinsic_load_point_coord;
case SYSTEM_VALUE_FRONT_FACE:
return nir_intrinsic_load_front_face;
case SYSTEM_VALUE_SAMPLE_ID:
return nir_intrinsic_load_tess_level_outer;
case SYSTEM_VALUE_TESS_LEVEL_INNER:
return nir_intrinsic_load_tess_level_inner;
+ case SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT:
+ return nir_intrinsic_load_tess_level_outer_default;
+ case SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT:
+ return nir_intrinsic_load_tess_level_inner_default;
case SYSTEM_VALUE_VERTICES_IN:
return nir_intrinsic_load_patch_vertices_in;
case SYSTEM_VALUE_HELPER_INVOCATION:
return nir_intrinsic_load_global_invocation_index;
case SYSTEM_VALUE_WORK_DIM:
return nir_intrinsic_load_work_dim;
+ case SYSTEM_VALUE_USER_DATA_AMD:
+ return nir_intrinsic_load_user_data_amd;
default:
unreachable("system value does not directly correspond to intrinsic");
}
return SYSTEM_VALUE_INVOCATION_ID;
case nir_intrinsic_load_frag_coord:
return SYSTEM_VALUE_FRAG_COORD;
+ case nir_intrinsic_load_point_coord:
+ return SYSTEM_VALUE_POINT_COORD;
case nir_intrinsic_load_front_face:
return SYSTEM_VALUE_FRONT_FACE;
case nir_intrinsic_load_sample_id:
return SYSTEM_VALUE_TESS_LEVEL_OUTER;
case nir_intrinsic_load_tess_level_inner:
return SYSTEM_VALUE_TESS_LEVEL_INNER;
+ case nir_intrinsic_load_tess_level_outer_default:
+ return SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT;
+ case nir_intrinsic_load_tess_level_inner_default:
+ return SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT;
case nir_intrinsic_load_patch_vertices_in:
return SYSTEM_VALUE_VERTICES_IN;
case nir_intrinsic_load_helper_invocation:
return SYSTEM_VALUE_LOCAL_GROUP_SIZE;
case nir_intrinsic_load_global_invocation_id:
return SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
+ case nir_intrinsic_load_user_data_amd:
+ return SYSTEM_VALUE_USER_DATA_AMD;
default:
unreachable("intrinsic doesn't produce a system value");
}
nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src,
bool bindless)
{
+ enum gl_access_qualifier access = nir_intrinsic_access(intrin);
+
switch (intrin->intrinsic) {
#define CASE(op) \
case nir_intrinsic_image_deref_##op: \
CASE(load)
CASE(store)
CASE(atomic_add)
- CASE(atomic_min)
- CASE(atomic_max)
+ CASE(atomic_imin)
+ CASE(atomic_umin)
+ CASE(atomic_imax)
+ CASE(atomic_umax)
CASE(atomic_and)
CASE(atomic_or)
CASE(atomic_xor)
CASE(atomic_exchange)
CASE(atomic_comp_swap)
CASE(atomic_fadd)
+ CASE(atomic_inc_wrap)
+ CASE(atomic_dec_wrap)
CASE(size)
CASE(samples)
CASE(load_raw_intel)
nir_intrinsic_set_image_dim(intrin, glsl_get_sampler_dim(deref->type));
nir_intrinsic_set_image_array(intrin, glsl_sampler_type_is_array(deref->type));
+ nir_intrinsic_set_access(intrin, access | var->data.access);
nir_intrinsic_set_format(intrin, var->data.image.format);
nir_instr_rewrite_src(&intrin->instr, &intrin->src[0],
nir_src_for_ssa(src));
}
+
+unsigned
+nir_image_intrinsic_coord_components(const nir_intrinsic_instr *instr)
+{
+ enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
+ int coords = glsl_get_sampler_dim_coordinate_components(dim);
+ if (dim == GLSL_SAMPLER_DIM_CUBE)
+ return coords;
+ else
+ return coords + nir_intrinsic_image_array(instr);
+}
projects / mesa.git / blobdiff