2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "program/prog_parameter.h"
26 #include "nir/nir_builder.h"
28 struct apply_pipeline_layout_state
{
32 struct anv_pipeline_layout
*layout
;
33 bool add_bounds_checks
;
36 uint8_t constants_offset
;
39 uint8_t *surface_offsets
;
40 uint8_t *sampler_offsets
;
41 uint8_t *image_offsets
;
46 add_binding(struct apply_pipeline_layout_state
*state
,
47 uint32_t set
, uint32_t binding
)
49 BITSET_SET(state
->set
[set
].used
, binding
);
53 add_var_binding(struct apply_pipeline_layout_state
*state
, nir_variable
*var
)
55 add_binding(state
, var
->data
.descriptor_set
, var
->data
.binding
);
59 add_deref_src_binding(struct apply_pipeline_layout_state
*state
, nir_src src
)
61 nir_deref_instr
*deref
= nir_src_as_deref(src
);
62 add_var_binding(state
, nir_deref_instr_get_variable(deref
));
66 add_tex_src_binding(struct apply_pipeline_layout_state
*state
,
67 nir_tex_instr
*tex
, nir_tex_src_type deref_src_type
)
69 int deref_src_idx
= nir_tex_instr_src_index(tex
, deref_src_type
);
70 if (deref_src_idx
< 0)
73 add_deref_src_binding(state
, tex
->src
[deref_src_idx
].src
);
77 get_used_bindings_block(nir_block
*block
,
78 struct apply_pipeline_layout_state
*state
)
80 nir_foreach_instr_safe(instr
, block
) {
81 switch (instr
->type
) {
82 case nir_instr_type_intrinsic
: {
83 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
84 switch (intrin
->intrinsic
) {
85 case nir_intrinsic_vulkan_resource_index
:
86 add_binding(state
, nir_intrinsic_desc_set(intrin
),
87 nir_intrinsic_binding(intrin
));
90 case nir_intrinsic_image_deref_load
:
91 case nir_intrinsic_image_deref_store
:
92 case nir_intrinsic_image_deref_atomic_add
:
93 case nir_intrinsic_image_deref_atomic_min
:
94 case nir_intrinsic_image_deref_atomic_max
:
95 case nir_intrinsic_image_deref_atomic_and
:
96 case nir_intrinsic_image_deref_atomic_or
:
97 case nir_intrinsic_image_deref_atomic_xor
:
98 case nir_intrinsic_image_deref_atomic_exchange
:
99 case nir_intrinsic_image_deref_atomic_comp_swap
:
100 case nir_intrinsic_image_deref_size
:
101 case nir_intrinsic_image_deref_samples
:
102 add_deref_src_binding(state
, intrin
->src
[0]);
105 case nir_intrinsic_load_constant
:
106 state
->uses_constants
= true;
114 case nir_instr_type_tex
: {
115 nir_tex_instr
*tex
= nir_instr_as_tex(instr
);
116 add_tex_src_binding(state
, tex
, nir_tex_src_texture_deref
);
117 add_tex_src_binding(state
, tex
, nir_tex_src_sampler_deref
);
127 lower_res_index_intrinsic(nir_intrinsic_instr
*intrin
,
128 struct apply_pipeline_layout_state
*state
)
130 nir_builder
*b
= &state
->builder
;
132 b
->cursor
= nir_before_instr(&intrin
->instr
);
134 uint32_t set
= nir_intrinsic_desc_set(intrin
);
135 uint32_t binding
= nir_intrinsic_binding(intrin
);
137 uint32_t surface_index
= state
->set
[set
].surface_offsets
[binding
];
138 uint32_t array_size
=
139 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
141 nir_const_value
*const_array_index
= nir_src_as_const_value(intrin
->src
[0]);
143 nir_ssa_def
*block_index
;
144 if (const_array_index
) {
145 unsigned array_index
= const_array_index
->u32
[0];
146 array_index
= MIN2(array_index
, array_size
- 1);
147 block_index
= nir_imm_int(b
, surface_index
+ array_index
);
149 block_index
= nir_ssa_for_src(b
, intrin
->src
[0], 1);
151 if (state
->add_bounds_checks
)
152 block_index
= nir_umin(b
, block_index
, nir_imm_int(b
, array_size
- 1));
154 block_index
= nir_iadd(b
, nir_imm_int(b
, surface_index
), block_index
);
157 assert(intrin
->dest
.is_ssa
);
158 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(block_index
));
159 nir_instr_remove(&intrin
->instr
);
163 lower_res_reindex_intrinsic(nir_intrinsic_instr
*intrin
,
164 struct apply_pipeline_layout_state
*state
)
166 nir_builder
*b
= &state
->builder
;
168 /* For us, the resource indices are just indices into the binding table and
169 * array elements are sequential. A resource_reindex just turns into an
170 * add of the two indices.
172 assert(intrin
->src
[0].is_ssa
&& intrin
->src
[1].is_ssa
);
173 nir_ssa_def
*new_index
= nir_iadd(b
, intrin
->src
[0].ssa
,
176 assert(intrin
->dest
.is_ssa
);
177 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(new_index
));
178 nir_instr_remove(&intrin
->instr
);
182 lower_load_constant(nir_intrinsic_instr
*intrin
,
183 struct apply_pipeline_layout_state
*state
)
185 nir_builder
*b
= &state
->builder
;
187 b
->cursor
= nir_before_instr(&intrin
->instr
);
189 nir_ssa_def
*index
= nir_imm_int(b
, state
->constants_offset
);
190 nir_ssa_def
*offset
= nir_iadd(b
, nir_ssa_for_src(b
, intrin
->src
[0], 1),
191 nir_imm_int(b
, nir_intrinsic_base(intrin
)));
193 nir_intrinsic_instr
*load_ubo
=
194 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_ubo
);
195 load_ubo
->num_components
= intrin
->num_components
;
196 load_ubo
->src
[0] = nir_src_for_ssa(index
);
197 load_ubo
->src
[1] = nir_src_for_ssa(offset
);
198 nir_ssa_dest_init(&load_ubo
->instr
, &load_ubo
->dest
,
199 intrin
->dest
.ssa
.num_components
,
200 intrin
->dest
.ssa
.bit_size
, NULL
);
201 nir_builder_instr_insert(b
, &load_ubo
->instr
);
203 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
204 nir_src_for_ssa(&load_ubo
->dest
.ssa
));
205 nir_instr_remove(&intrin
->instr
);
209 lower_tex_deref(nir_tex_instr
*tex
, nir_tex_src_type deref_src_type
,
210 unsigned *base_index
,
211 struct apply_pipeline_layout_state
*state
)
213 int deref_src_idx
= nir_tex_instr_src_index(tex
, deref_src_type
);
214 if (deref_src_idx
< 0)
217 nir_deref_instr
*deref
= nir_src_as_deref(tex
->src
[deref_src_idx
].src
);
218 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
220 unsigned set
= var
->data
.descriptor_set
;
221 unsigned binding
= var
->data
.binding
;
222 unsigned array_size
=
223 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
225 nir_tex_src_type offset_src_type
;
226 if (deref_src_type
== nir_tex_src_texture_deref
) {
227 offset_src_type
= nir_tex_src_texture_offset
;
228 *base_index
= state
->set
[set
].surface_offsets
[binding
];
230 assert(deref_src_type
== nir_tex_src_sampler_deref
);
231 offset_src_type
= nir_tex_src_sampler_offset
;
232 *base_index
= state
->set
[set
].sampler_offsets
[binding
];
235 nir_ssa_def
*index
= NULL
;
236 if (deref
->deref_type
!= nir_deref_type_var
) {
237 assert(deref
->deref_type
== nir_deref_type_array
);
239 nir_const_value
*const_index
= nir_src_as_const_value(deref
->arr
.index
);
241 *base_index
+= MIN2(const_index
->u32
[0], array_size
- 1);
243 nir_builder
*b
= &state
->builder
;
245 /* From VK_KHR_sampler_ycbcr_conversion:
247 * If sampler Y’CBCR conversion is enabled, the combined image
248 * sampler must be indexed only by constant integral expressions when
249 * aggregated into arrays in shader code, irrespective of the
250 * shaderSampledImageArrayDynamicIndexing feature.
252 assert(nir_tex_instr_src_index(tex
, nir_tex_src_plane
) == -1);
254 index
= nir_ssa_for_src(b
, deref
->arr
.index
, 1);
256 if (state
->add_bounds_checks
)
257 index
= nir_umin(b
, index
, nir_imm_int(b
, array_size
- 1));
262 nir_instr_rewrite_src(&tex
->instr
, &tex
->src
[deref_src_idx
].src
,
263 nir_src_for_ssa(index
));
264 tex
->src
[deref_src_idx
].src_type
= offset_src_type
;
266 nir_tex_instr_remove_src(tex
, deref_src_idx
);
271 tex_instr_get_and_remove_plane_src(nir_tex_instr
*tex
)
273 int plane_src_idx
= nir_tex_instr_src_index(tex
, nir_tex_src_plane
);
274 if (plane_src_idx
< 0)
278 nir_src_as_const_value(tex
->src
[plane_src_idx
].src
)->u32
[0];
280 nir_tex_instr_remove_src(tex
, plane_src_idx
);
286 lower_tex(nir_tex_instr
*tex
, struct apply_pipeline_layout_state
*state
)
288 state
->builder
.cursor
= nir_before_instr(&tex
->instr
);
290 unsigned plane
= tex_instr_get_and_remove_plane_src(tex
);
292 lower_tex_deref(tex
, nir_tex_src_texture_deref
,
293 &tex
->texture_index
, state
);
294 tex
->texture_index
+= plane
;
296 lower_tex_deref(tex
, nir_tex_src_sampler_deref
,
297 &tex
->sampler_index
, state
);
298 tex
->sampler_index
+= plane
;
300 /* The backend only ever uses this to mark used surfaces. We don't care
301 * about that little optimization so it just needs to be non-zero.
303 tex
->texture_array_size
= 1;
307 apply_pipeline_layout_block(nir_block
*block
,
308 struct apply_pipeline_layout_state
*state
)
310 nir_foreach_instr_safe(instr
, block
) {
311 switch (instr
->type
) {
312 case nir_instr_type_intrinsic
: {
313 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
314 switch (intrin
->intrinsic
) {
315 case nir_intrinsic_vulkan_resource_index
:
316 lower_res_index_intrinsic(intrin
, state
);
318 case nir_intrinsic_vulkan_resource_reindex
:
319 lower_res_reindex_intrinsic(intrin
, state
);
321 case nir_intrinsic_load_constant
:
322 lower_load_constant(intrin
, state
);
329 case nir_instr_type_tex
:
330 lower_tex(nir_instr_as_tex(instr
), state
);
339 setup_vec4_uniform_value(uint32_t *params
, uint32_t offset
, unsigned n
)
341 for (unsigned i
= 0; i
< n
; ++i
)
342 params
[i
] = ANV_PARAM_PUSH(offset
+ i
* sizeof(uint32_t));
344 for (unsigned i
= n
; i
< 4; ++i
)
345 params
[i
] = BRW_PARAM_BUILTIN_ZERO
;
349 anv_nir_apply_pipeline_layout(struct anv_pipeline
*pipeline
,
350 struct anv_pipeline_layout
*layout
,
352 struct brw_stage_prog_data
*prog_data
,
353 struct anv_pipeline_bind_map
*map
)
355 gl_shader_stage stage
= shader
->info
.stage
;
357 struct apply_pipeline_layout_state state
= {
360 .add_bounds_checks
= pipeline
->device
->robust_buffer_access
,
363 void *mem_ctx
= ralloc_context(NULL
);
365 for (unsigned s
= 0; s
< layout
->num_sets
; s
++) {
366 const unsigned count
= layout
->set
[s
].layout
->binding_count
;
367 const unsigned words
= BITSET_WORDS(count
);
368 state
.set
[s
].used
= rzalloc_array(mem_ctx
, BITSET_WORD
, words
);
369 state
.set
[s
].surface_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
370 state
.set
[s
].sampler_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
371 state
.set
[s
].image_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
374 nir_foreach_function(function
, shader
) {
378 nir_foreach_block(block
, function
->impl
)
379 get_used_bindings_block(block
, &state
);
382 if (state
.uses_constants
)
383 map
->surface_count
++;
385 for (uint32_t set
= 0; set
< layout
->num_sets
; set
++) {
386 struct anv_descriptor_set_layout
*set_layout
= layout
->set
[set
].layout
;
389 BITSET_FOREACH_SET(b
, _tmp
, state
.set
[set
].used
,
390 set_layout
->binding_count
) {
391 if (set_layout
->binding
[b
].stage
[stage
].surface_index
>= 0) {
392 map
->surface_count
+=
393 anv_descriptor_set_binding_layout_get_hw_size(&set_layout
->binding
[b
]);
395 if (set_layout
->binding
[b
].stage
[stage
].sampler_index
>= 0) {
396 map
->sampler_count
+=
397 anv_descriptor_set_binding_layout_get_hw_size(&set_layout
->binding
[b
]);
399 if (set_layout
->binding
[b
].stage
[stage
].image_index
>= 0)
400 map
->image_count
+= set_layout
->binding
[b
].array_size
;
404 unsigned surface
= 0;
405 unsigned sampler
= 0;
408 if (state
.uses_constants
) {
409 state
.constants_offset
= surface
;
410 map
->surface_to_descriptor
[surface
].set
=
411 ANV_DESCRIPTOR_SET_SHADER_CONSTANTS
;
415 for (uint32_t set
= 0; set
< layout
->num_sets
; set
++) {
416 struct anv_descriptor_set_layout
*set_layout
= layout
->set
[set
].layout
;
419 BITSET_FOREACH_SET(b
, _tmp
, state
.set
[set
].used
,
420 set_layout
->binding_count
) {
421 struct anv_descriptor_set_binding_layout
*binding
=
422 &set_layout
->binding
[b
];
424 if (binding
->stage
[stage
].surface_index
>= 0) {
425 state
.set
[set
].surface_offsets
[b
] = surface
;
426 struct anv_sampler
**samplers
= binding
->immutable_samplers
;
427 for (unsigned i
= 0; i
< binding
->array_size
; i
++) {
428 uint8_t planes
= samplers
? samplers
[i
]->n_planes
: 1;
429 for (uint8_t p
= 0; p
< planes
; p
++) {
430 map
->surface_to_descriptor
[surface
].set
= set
;
431 map
->surface_to_descriptor
[surface
].binding
= b
;
432 map
->surface_to_descriptor
[surface
].index
= i
;
433 map
->surface_to_descriptor
[surface
].plane
= p
;
439 if (binding
->stage
[stage
].sampler_index
>= 0) {
440 state
.set
[set
].sampler_offsets
[b
] = sampler
;
441 struct anv_sampler
**samplers
= binding
->immutable_samplers
;
442 for (unsigned i
= 0; i
< binding
->array_size
; i
++) {
443 uint8_t planes
= samplers
? samplers
[i
]->n_planes
: 1;
444 for (uint8_t p
= 0; p
< planes
; p
++) {
445 map
->sampler_to_descriptor
[sampler
].set
= set
;
446 map
->sampler_to_descriptor
[sampler
].binding
= b
;
447 map
->sampler_to_descriptor
[sampler
].index
= i
;
448 map
->sampler_to_descriptor
[sampler
].plane
= p
;
454 if (binding
->stage
[stage
].image_index
>= 0) {
455 state
.set
[set
].image_offsets
[b
] = image
;
456 image
+= binding
->array_size
;
461 nir_foreach_variable(var
, &shader
->uniforms
) {
462 const struct glsl_type
*glsl_type
= glsl_without_array(var
->type
);
464 if (!glsl_type_is_image(glsl_type
))
467 enum glsl_sampler_dim dim
= glsl_get_sampler_dim(glsl_type
);
469 const uint32_t set
= var
->data
.descriptor_set
;
470 const uint32_t binding
= var
->data
.binding
;
471 const uint32_t array_size
=
472 layout
->set
[set
].layout
->binding
[binding
].array_size
;
474 if (!BITSET_TEST(state
.set
[set
].used
, binding
))
477 struct anv_pipeline_binding
*pipe_binding
=
478 &map
->surface_to_descriptor
[state
.set
[set
].surface_offsets
[binding
]];
479 for (unsigned i
= 0; i
< array_size
; i
++) {
480 assert(pipe_binding
[i
].set
== set
);
481 assert(pipe_binding
[i
].binding
== binding
);
482 assert(pipe_binding
[i
].index
== i
);
484 if (dim
== GLSL_SAMPLER_DIM_SUBPASS
||
485 dim
== GLSL_SAMPLER_DIM_SUBPASS_MS
)
486 pipe_binding
[i
].input_attachment_index
= var
->data
.index
+ i
;
488 pipe_binding
[i
].write_only
= var
->data
.image
.write_only
;
492 nir_foreach_function(function
, shader
) {
496 nir_builder_init(&state
.builder
, function
->impl
);
497 nir_foreach_block(block
, function
->impl
)
498 apply_pipeline_layout_block(block
, &state
);
499 nir_metadata_preserve(function
->impl
, nir_metadata_block_index
|
500 nir_metadata_dominance
);
503 if (map
->image_count
> 0) {
504 assert(map
->image_count
<= MAX_IMAGES
);
505 nir_foreach_variable(var
, &shader
->uniforms
) {
506 if (glsl_type_is_image(var
->type
) ||
507 (glsl_type_is_array(var
->type
) &&
508 glsl_type_is_image(glsl_get_array_element(var
->type
)))) {
509 /* Images are represented as uniform push constants and the actual
510 * information required for reading/writing to/from the image is
511 * storred in the uniform.
513 unsigned set
= var
->data
.descriptor_set
;
514 unsigned binding
= var
->data
.binding
;
515 unsigned image_index
= state
.set
[set
].image_offsets
[binding
];
517 var
->data
.driver_location
= shader
->num_uniforms
+
518 image_index
* BRW_IMAGE_PARAM_SIZE
* 4;
522 uint32_t *param
= brw_stage_prog_data_add_params(prog_data
,
524 BRW_IMAGE_PARAM_SIZE
);
525 struct anv_push_constants
*null_data
= NULL
;
526 const struct brw_image_param
*image_param
= null_data
->images
;
527 for (uint32_t i
= 0; i
< map
->image_count
; i
++) {
528 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_SURFACE_IDX_OFFSET
,
529 (uintptr_t)&image_param
->surface_idx
, 1);
530 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_OFFSET_OFFSET
,
531 (uintptr_t)image_param
->offset
, 2);
532 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_SIZE_OFFSET
,
533 (uintptr_t)image_param
->size
, 3);
534 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_STRIDE_OFFSET
,
535 (uintptr_t)image_param
->stride
, 4);
536 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_TILING_OFFSET
,
537 (uintptr_t)image_param
->tiling
, 3);
538 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_SWIZZLING_OFFSET
,
539 (uintptr_t)image_param
->swizzling
, 2);
541 param
+= BRW_IMAGE_PARAM_SIZE
;
544 assert(param
== prog_data
->param
+ prog_data
->nr_params
);
546 shader
->num_uniforms
+= map
->image_count
* BRW_IMAGE_PARAM_SIZE
* 4;
549 ralloc_free(mem_ctx
);