case nir_var_uniform:
op = nir_intrinsic_load_uniform;
break;
+ case nir_var_shared:
+ op = nir_intrinsic_load_shared;
+ break;
default:
unreachable("Unknown variable mode");
}
return op;
}
+static nir_intrinsic_op
+store_op(struct lower_io_state *state,
+ nir_variable_mode mode, bool per_vertex)
+{
+ nir_intrinsic_op op;
+ switch (mode) {
+ case nir_var_shader_in:
+ case nir_var_shader_out:
+ op = per_vertex ? nir_intrinsic_store_per_vertex_output :
+ nir_intrinsic_store_output;
+ break;
+ case nir_var_shared:
+ op = nir_intrinsic_store_shared;
+ break;
+ default:
+ unreachable("Unknown variable mode");
+ }
+ return op;
+}
+
+static nir_intrinsic_op
+atomic_op(nir_intrinsic_op opcode)
+{
+ switch (opcode) {
+#define OP(O) case nir_intrinsic_var_##O: return nir_intrinsic_shared_##O;
+ OP(atomic_exchange)
+ OP(atomic_comp_swap)
+ OP(atomic_add)
+ OP(atomic_imin)
+ OP(atomic_umin)
+ OP(atomic_imax)
+ OP(atomic_umax)
+ OP(atomic_and)
+ OP(atomic_or)
+ OP(atomic_xor)
+#undef OP
+ default:
+ unreachable("Invalid atomic");
+ }
+}
+
static bool
nir_lower_io_block(nir_block *block, void *void_state)
{
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
- if (intrin->intrinsic != nir_intrinsic_load_var &&
- intrin->intrinsic != nir_intrinsic_store_var)
+ switch (intrin->intrinsic) {
+ case nir_intrinsic_load_var:
+ case nir_intrinsic_store_var:
+ case nir_intrinsic_var_atomic_add:
+ case nir_intrinsic_var_atomic_imin:
+ case nir_intrinsic_var_atomic_umin:
+ case nir_intrinsic_var_atomic_imax:
+ case nir_intrinsic_var_atomic_umax:
+ case nir_intrinsic_var_atomic_and:
+ case nir_intrinsic_var_atomic_or:
+ case nir_intrinsic_var_atomic_xor:
+ case nir_intrinsic_var_atomic_exchange:
+ case nir_intrinsic_var_atomic_comp_swap:
+ /* We can lower the io for this nir instrinsic */
+ break;
+ default:
+ /* We can't lower the io for this nir instrinsic, so skip it */
continue;
+ }
nir_variable_mode mode = intrin->variables[0]->var->data.mode;
if (mode != nir_var_shader_in &&
mode != nir_var_shader_out &&
+ mode != nir_var_shared &&
mode != nir_var_uniform)
continue;
load->const_index[0] =
intrin->variables[0]->var->data.driver_location;
+ if (load->intrinsic == nir_intrinsic_load_uniform) {
+ load->const_index[1] =
+ state->type_size(intrin->variables[0]->var->type);
+ }
+
if (per_vertex)
load->src[0] = nir_src_for_ssa(vertex_index);
}
case nir_intrinsic_store_var: {
- assert(mode == nir_var_shader_out);
+ assert(mode == nir_var_shader_out || mode == nir_var_shared);
nir_ssa_def *offset;
nir_ssa_def *vertex_index;
per_vertex ? &vertex_index : NULL,
state->type_size);
- nir_intrinsic_op store_op =
- per_vertex ? nir_intrinsic_store_per_vertex_output :
- nir_intrinsic_store_output;
-
- nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx,
- store_op);
+ nir_intrinsic_instr *store =
+ nir_intrinsic_instr_create(state->mem_ctx,
+ store_op(state, mode, per_vertex));
store->num_components = intrin->num_components;
nir_src_copy(&store->src[0], &intrin->src[0], store);
break;
}
+ case nir_intrinsic_var_atomic_add:
+ case nir_intrinsic_var_atomic_imin:
+ case nir_intrinsic_var_atomic_umin:
+ case nir_intrinsic_var_atomic_imax:
+ case nir_intrinsic_var_atomic_umax:
+ case nir_intrinsic_var_atomic_and:
+ case nir_intrinsic_var_atomic_or:
+ case nir_intrinsic_var_atomic_xor:
+ case nir_intrinsic_var_atomic_exchange:
+ case nir_intrinsic_var_atomic_comp_swap: {
+ assert(mode == nir_var_shared);
+
+ nir_ssa_def *offset;
+
+ offset = get_io_offset(b, intrin->variables[0],
+ NULL, state->type_size);
+
+ nir_intrinsic_instr *atomic =
+ nir_intrinsic_instr_create(state->mem_ctx,
+ atomic_op(intrin->intrinsic));
+
+ atomic->src[0] = nir_src_for_ssa(offset);
+
+ atomic->const_index[0] =
+ intrin->variables[0]->var->data.driver_location;
+
+ nir_src_copy(&atomic->src[1], &intrin->src[0], atomic);
+
+ if (intrin->intrinsic == nir_intrinsic_var_atomic_comp_swap)
+ nir_src_copy(&atomic->src[2], &intrin->src[1], atomic);
+
+ if (intrin->dest.is_ssa) {
+ nir_ssa_dest_init(&atomic->instr, &atomic->dest,
+ intrin->dest.ssa.num_components, NULL);
+ nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
+ nir_src_for_ssa(&atomic->dest.ssa));
+ } else {
+ nir_dest_copy(&atomic->dest, &intrin->dest, state->mem_ctx);
+ }
+
+ nir_instr_insert_before(&intrin->instr, &atomic->instr);
+ nir_instr_remove(&intrin->instr);
+ break;
+ }
+
default:
break;
}
case nir_intrinsic_load_output:
case nir_intrinsic_load_uniform:
return &instr->src[0];
+ case nir_intrinsic_load_ubo:
+ case nir_intrinsic_load_ssbo:
case nir_intrinsic_load_per_vertex_input:
case nir_intrinsic_load_per_vertex_output:
case nir_intrinsic_store_output:
return &instr->src[1];
+ case nir_intrinsic_store_ssbo:
case nir_intrinsic_store_per_vertex_output:
return &instr->src[2];
default: