2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Connor Abbott (cwabbott0@gmail.com)
25 * Jason Ekstrand (jason@jlekstrand.net)
30 * This lowering pass converts references to input/output variables with
31 * loads/stores to actual input/output intrinsics.
35 #include "nir_builder.h"
37 struct lower_io_state
{
40 int (*type_size
)(const struct glsl_type
*type
);
41 nir_variable_mode mode
;
45 nir_assign_var_locations(struct exec_list
*var_list
, unsigned *size
,
46 int (*type_size
)(const struct glsl_type
*))
48 unsigned location
= 0;
50 nir_foreach_variable(var
, var_list
) {
52 * UBO's have their own address spaces, so don't count them towards the
53 * number of global uniforms
55 if ((var
->data
.mode
== nir_var_uniform
|| var
->data
.mode
== nir_var_shader_storage
) &&
56 var
->interface_type
!= NULL
)
59 var
->data
.driver_location
= location
;
60 location
+= type_size(var
->type
);
67 * Returns true if we're processing a stage whose inputs are arrays indexed
68 * by a vertex number (such as geometry shader inputs).
71 is_per_vertex_input(struct lower_io_state
*state
, nir_variable
*var
)
73 gl_shader_stage stage
= state
->builder
.shader
->stage
;
75 return var
->data
.mode
== nir_var_shader_in
&& !var
->data
.patch
&&
76 (stage
== MESA_SHADER_TESS_CTRL
||
77 stage
== MESA_SHADER_TESS_EVAL
||
78 stage
== MESA_SHADER_GEOMETRY
);
82 is_per_vertex_output(struct lower_io_state
*state
, nir_variable
*var
)
84 gl_shader_stage stage
= state
->builder
.shader
->stage
;
85 return var
->data
.mode
== nir_var_shader_out
&& !var
->data
.patch
&&
86 stage
== MESA_SHADER_TESS_CTRL
;
90 get_io_offset(nir_deref_var
*deref
, nir_instr
*instr
,
91 nir_ssa_def
**vertex_index
,
92 nir_ssa_def
**out_indirect
,
93 struct lower_io_state
*state
)
95 nir_ssa_def
*indirect
= NULL
;
96 unsigned base_offset
= 0;
98 nir_builder
*b
= &state
->builder
;
99 b
->cursor
= nir_before_instr(instr
);
101 nir_deref
*tail
= &deref
->deref
;
103 /* For per-vertex input arrays (i.e. geometry shader inputs), keep the
104 * outermost array index separate. Process the rest normally.
106 if (vertex_index
!= NULL
) {
108 assert(tail
->deref_type
== nir_deref_type_array
);
109 nir_deref_array
*deref_array
= nir_deref_as_array(tail
);
111 nir_ssa_def
*vtx
= nir_imm_int(b
, deref_array
->base_offset
);
112 if (deref_array
->deref_array_type
== nir_deref_array_type_indirect
) {
113 vtx
= nir_iadd(b
, vtx
, nir_ssa_for_src(b
, deref_array
->indirect
, 1));
118 while (tail
->child
!= NULL
) {
119 const struct glsl_type
*parent_type
= tail
->type
;
122 if (tail
->deref_type
== nir_deref_type_array
) {
123 nir_deref_array
*deref_array
= nir_deref_as_array(tail
);
124 unsigned size
= state
->type_size(tail
->type
);
126 base_offset
+= size
* deref_array
->base_offset
;
128 if (deref_array
->deref_array_type
== nir_deref_array_type_indirect
) {
130 nir_imul(b
, nir_imm_int(b
, size
),
131 nir_ssa_for_src(b
, deref_array
->indirect
, 1));
133 indirect
= indirect
? nir_iadd(b
, indirect
, mul
) : mul
;
135 } else if (tail
->deref_type
== nir_deref_type_struct
) {
136 nir_deref_struct
*deref_struct
= nir_deref_as_struct(tail
);
138 for (unsigned i
= 0; i
< deref_struct
->index
; i
++) {
140 state
->type_size(glsl_get_struct_field(parent_type
, i
));
145 *out_indirect
= indirect
;
149 static nir_intrinsic_op
150 load_op(struct lower_io_state
*state
,
151 nir_variable_mode mode
, bool per_vertex
, bool has_indirect
)
155 case nir_var_shader_in
:
157 op
= has_indirect
? nir_intrinsic_load_per_vertex_input_indirect
:
158 nir_intrinsic_load_per_vertex_input
;
160 op
= has_indirect
? nir_intrinsic_load_input_indirect
:
161 nir_intrinsic_load_input
;
164 case nir_var_shader_out
:
166 op
= has_indirect
? nir_intrinsic_load_per_vertex_output_indirect
:
167 nir_intrinsic_load_per_vertex_output
;
169 op
= has_indirect
? nir_intrinsic_load_output_indirect
:
170 nir_intrinsic_load_output
;
173 case nir_var_uniform
:
174 op
= has_indirect
? nir_intrinsic_load_uniform_indirect
:
175 nir_intrinsic_load_uniform
;
178 unreachable("Unknown variable mode");
184 nir_lower_io_block(nir_block
*block
, void *void_state
)
186 struct lower_io_state
*state
= void_state
;
188 nir_foreach_instr_safe(block
, instr
) {
189 if (instr
->type
!= nir_instr_type_intrinsic
)
192 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
194 if (intrin
->intrinsic
!= nir_intrinsic_load_var
&&
195 intrin
->intrinsic
!= nir_intrinsic_store_var
)
198 nir_variable_mode mode
= intrin
->variables
[0]->var
->data
.mode
;
200 if (state
->mode
!= nir_var_all
&& state
->mode
!= mode
)
203 if (mode
!= nir_var_shader_in
&&
204 mode
!= nir_var_shader_out
&&
205 mode
!= nir_var_uniform
)
208 switch (intrin
->intrinsic
) {
209 case nir_intrinsic_load_var
: {
211 is_per_vertex_input(state
, intrin
->variables
[0]->var
) ||
212 is_per_vertex_output(state
, intrin
->variables
[0]->var
);
214 nir_ssa_def
*indirect
;
215 nir_ssa_def
*vertex_index
;
217 unsigned offset
= get_io_offset(intrin
->variables
[0], &intrin
->instr
,
218 per_vertex
? &vertex_index
: NULL
,
221 nir_intrinsic_instr
*load
=
222 nir_intrinsic_instr_create(state
->mem_ctx
,
223 load_op(state
, mode
, per_vertex
,
225 load
->num_components
= intrin
->num_components
;
227 unsigned location
= intrin
->variables
[0]->var
->data
.driver_location
;
228 if (mode
== nir_var_uniform
) {
229 load
->const_index
[0] = location
;
230 load
->const_index
[1] = offset
;
232 load
->const_index
[0] = location
+ offset
;
236 load
->src
[0] = nir_src_for_ssa(vertex_index
);
239 load
->src
[per_vertex
? 1 : 0] = nir_src_for_ssa(indirect
);
241 if (intrin
->dest
.is_ssa
) {
242 nir_ssa_dest_init(&load
->instr
, &load
->dest
,
243 intrin
->num_components
, NULL
);
244 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
245 nir_src_for_ssa(&load
->dest
.ssa
));
247 nir_dest_copy(&load
->dest
, &intrin
->dest
, state
->mem_ctx
);
250 nir_instr_insert_before(&intrin
->instr
, &load
->instr
);
251 nir_instr_remove(&intrin
->instr
);
255 case nir_intrinsic_store_var
: {
256 assert(mode
== nir_var_shader_out
);
258 nir_ssa_def
*indirect
;
259 nir_ssa_def
*vertex_index
;
262 is_per_vertex_output(state
, intrin
->variables
[0]->var
);
264 unsigned offset
= get_io_offset(intrin
->variables
[0], &intrin
->instr
,
265 per_vertex
? &vertex_index
: NULL
,
267 offset
+= intrin
->variables
[0]->var
->data
.driver_location
;
269 nir_intrinsic_op store_op
;
271 store_op
= indirect
? nir_intrinsic_store_per_vertex_output_indirect
272 : nir_intrinsic_store_per_vertex_output
;
274 store_op
= indirect
? nir_intrinsic_store_output_indirect
275 : nir_intrinsic_store_output
;
278 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(state
->mem_ctx
,
280 store
->num_components
= intrin
->num_components
;
281 store
->const_index
[0] = offset
;
283 nir_src_copy(&store
->src
[0], &intrin
->src
[0], store
);
286 store
->src
[1] = nir_src_for_ssa(vertex_index
);
289 store
->src
[per_vertex
? 2 : 1] = nir_src_for_ssa(indirect
);
291 nir_instr_insert_before(&intrin
->instr
, &store
->instr
);
292 nir_instr_remove(&intrin
->instr
);
305 nir_lower_io_impl(nir_function_impl
*impl
,
306 nir_variable_mode mode
,
307 int (*type_size
)(const struct glsl_type
*))
309 struct lower_io_state state
;
311 nir_builder_init(&state
.builder
, impl
);
312 state
.mem_ctx
= ralloc_parent(impl
);
314 state
.type_size
= type_size
;
316 nir_foreach_block(impl
, nir_lower_io_block
, &state
);
318 nir_metadata_preserve(impl
, nir_metadata_block_index
|
319 nir_metadata_dominance
);
323 nir_lower_io(nir_shader
*shader
, nir_variable_mode mode
,
324 int (*type_size
)(const struct glsl_type
*))
326 nir_foreach_overload(shader
, overload
) {
328 nir_lower_io_impl(overload
->impl
, mode
, type_size
);
333 * Return the indirect source for a load/store indirect intrinsic.
336 nir_get_io_indirect_src(nir_intrinsic_instr
*instr
)
338 switch (instr
->intrinsic
) {
339 case nir_intrinsic_load_input_indirect
:
340 case nir_intrinsic_load_output_indirect
:
341 case nir_intrinsic_load_uniform_indirect
:
342 return &instr
->src
[0];
343 case nir_intrinsic_load_per_vertex_input_indirect
:
344 case nir_intrinsic_load_per_vertex_output_indirect
:
345 case nir_intrinsic_store_output_indirect
:
346 return &instr
->src
[1];
347 case nir_intrinsic_store_per_vertex_output_indirect
:
348 return &instr
->src
[2];
355 * Return the vertex index source for a load/store per_vertex intrinsic.
358 nir_get_io_vertex_index_src(nir_intrinsic_instr
*instr
)
360 switch (instr
->intrinsic
) {
361 case nir_intrinsic_load_per_vertex_input
:
362 case nir_intrinsic_load_per_vertex_output
:
363 case nir_intrinsic_load_per_vertex_input_indirect
:
364 case nir_intrinsic_load_per_vertex_output_indirect
:
365 return &instr
->src
[0];
366 case nir_intrinsic_store_per_vertex_output
:
367 case nir_intrinsic_store_per_vertex_output_indirect
:
368 return &instr
->src
[1];