2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "program/prog_parameter.h"
26 #include "nir/nir_builder.h"
27 #include "compiler/brw_nir.h"
29 struct apply_pipeline_layout_state
{
33 struct anv_pipeline_layout
*layout
;
34 bool add_bounds_checks
;
36 unsigned first_image_uniform
;
39 uint8_t constants_offset
;
42 uint8_t *surface_offsets
;
43 uint8_t *sampler_offsets
;
44 uint8_t *image_offsets
;
49 add_binding(struct apply_pipeline_layout_state
*state
,
50 uint32_t set
, uint32_t binding
)
52 BITSET_SET(state
->set
[set
].used
, binding
);
56 add_deref_src_binding(struct apply_pipeline_layout_state
*state
, nir_src src
)
58 nir_deref_instr
*deref
= nir_src_as_deref(src
);
59 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
60 add_binding(state
, var
->data
.descriptor_set
, var
->data
.binding
);
64 add_tex_src_binding(struct apply_pipeline_layout_state
*state
,
65 nir_tex_instr
*tex
, nir_tex_src_type deref_src_type
)
67 int deref_src_idx
= nir_tex_instr_src_index(tex
, deref_src_type
);
68 if (deref_src_idx
< 0)
71 add_deref_src_binding(state
, tex
->src
[deref_src_idx
].src
);
75 get_used_bindings_block(nir_block
*block
,
76 struct apply_pipeline_layout_state
*state
)
78 nir_foreach_instr_safe(instr
, block
) {
79 switch (instr
->type
) {
80 case nir_instr_type_intrinsic
: {
81 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
82 switch (intrin
->intrinsic
) {
83 case nir_intrinsic_vulkan_resource_index
:
84 add_binding(state
, nir_intrinsic_desc_set(intrin
),
85 nir_intrinsic_binding(intrin
));
88 case nir_intrinsic_image_deref_load
:
89 case nir_intrinsic_image_deref_store
:
90 case nir_intrinsic_image_deref_atomic_add
:
91 case nir_intrinsic_image_deref_atomic_min
:
92 case nir_intrinsic_image_deref_atomic_max
:
93 case nir_intrinsic_image_deref_atomic_and
:
94 case nir_intrinsic_image_deref_atomic_or
:
95 case nir_intrinsic_image_deref_atomic_xor
:
96 case nir_intrinsic_image_deref_atomic_exchange
:
97 case nir_intrinsic_image_deref_atomic_comp_swap
:
98 case nir_intrinsic_image_deref_size
:
99 case nir_intrinsic_image_deref_samples
:
100 case nir_intrinsic_image_deref_load_param_intel
:
101 case nir_intrinsic_image_deref_load_raw_intel
:
102 case nir_intrinsic_image_deref_store_raw_intel
:
103 add_deref_src_binding(state
, intrin
->src
[0]);
106 case nir_intrinsic_load_constant
:
107 state
->uses_constants
= true;
115 case nir_instr_type_tex
: {
116 nir_tex_instr
*tex
= nir_instr_as_tex(instr
);
117 add_tex_src_binding(state
, tex
, nir_tex_src_texture_deref
);
118 add_tex_src_binding(state
, tex
, nir_tex_src_sampler_deref
);
128 lower_res_index_intrinsic(nir_intrinsic_instr
*intrin
,
129 struct apply_pipeline_layout_state
*state
)
131 nir_builder
*b
= &state
->builder
;
133 b
->cursor
= nir_before_instr(&intrin
->instr
);
135 uint32_t set
= nir_intrinsic_desc_set(intrin
);
136 uint32_t binding
= nir_intrinsic_binding(intrin
);
138 uint32_t surface_index
= state
->set
[set
].surface_offsets
[binding
];
139 uint32_t array_size
=
140 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
142 nir_ssa_def
*array_index
= nir_ssa_for_src(b
, intrin
->src
[0], 1);
143 if (nir_src_is_const(intrin
->src
[0]) || state
->add_bounds_checks
)
144 array_index
= nir_umin(b
, array_index
, nir_imm_int(b
, array_size
- 1));
146 nir_ssa_def
*block_index
= nir_iadd_imm(b
, array_index
, surface_index
);
148 assert(intrin
->dest
.is_ssa
);
149 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(block_index
));
150 nir_instr_remove(&intrin
->instr
);
154 lower_res_reindex_intrinsic(nir_intrinsic_instr
*intrin
,
155 struct apply_pipeline_layout_state
*state
)
157 nir_builder
*b
= &state
->builder
;
159 b
->cursor
= nir_before_instr(&intrin
->instr
);
161 /* For us, the resource indices are just indices into the binding table and
162 * array elements are sequential. A resource_reindex just turns into an
163 * add of the two indices.
165 assert(intrin
->src
[0].is_ssa
&& intrin
->src
[1].is_ssa
);
166 nir_ssa_def
*new_index
= nir_iadd(b
, intrin
->src
[0].ssa
,
169 assert(intrin
->dest
.is_ssa
);
170 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(new_index
));
171 nir_instr_remove(&intrin
->instr
);
175 lower_load_vulkan_descriptor(nir_intrinsic_instr
*intrin
,
176 struct apply_pipeline_layout_state
*state
)
178 nir_builder
*b
= &state
->builder
;
180 b
->cursor
= nir_before_instr(&intrin
->instr
);
182 /* We follow the nir_address_format_vk_index_offset model */
183 assert(intrin
->src
[0].is_ssa
);
184 nir_ssa_def
*vec2
= nir_vec2(b
, intrin
->src
[0].ssa
, nir_imm_int(b
, 0));
186 assert(intrin
->dest
.is_ssa
);
187 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(vec2
));
188 nir_instr_remove(&intrin
->instr
);
192 lower_image_intrinsic(nir_intrinsic_instr
*intrin
,
193 struct apply_pipeline_layout_state
*state
)
195 nir_deref_instr
*deref
= nir_src_as_deref(intrin
->src
[0]);
196 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
198 unsigned set
= var
->data
.descriptor_set
;
199 unsigned binding
= var
->data
.binding
;
200 unsigned array_size
=
201 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
203 nir_builder
*b
= &state
->builder
;
204 b
->cursor
= nir_before_instr(&intrin
->instr
);
206 nir_ssa_def
*index
= NULL
;
207 if (deref
->deref_type
!= nir_deref_type_var
) {
208 assert(deref
->deref_type
== nir_deref_type_array
);
209 index
= nir_ssa_for_src(b
, deref
->arr
.index
, 1);
210 if (state
->add_bounds_checks
)
211 index
= nir_umin(b
, index
, nir_imm_int(b
, array_size
- 1));
213 index
= nir_imm_int(b
, 0);
216 if (intrin
->intrinsic
== nir_intrinsic_image_deref_load_param_intel
) {
217 b
->cursor
= nir_instr_remove(&intrin
->instr
);
219 nir_intrinsic_instr
*load
=
220 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_uniform
);
222 nir_intrinsic_set_base(load
, state
->first_image_uniform
+
223 state
->set
[set
].image_offsets
[binding
] *
224 BRW_IMAGE_PARAM_SIZE
* 4);
225 nir_intrinsic_set_range(load
, array_size
* BRW_IMAGE_PARAM_SIZE
* 4);
227 const unsigned param
= nir_intrinsic_base(intrin
);
228 nir_ssa_def
*offset
=
229 nir_imul(b
, index
, nir_imm_int(b
, BRW_IMAGE_PARAM_SIZE
* 4));
230 offset
= nir_iadd(b
, offset
, nir_imm_int(b
, param
* 16));
231 load
->src
[0] = nir_src_for_ssa(offset
);
233 load
->num_components
= intrin
->dest
.ssa
.num_components
;
234 nir_ssa_dest_init(&load
->instr
, &load
->dest
,
235 intrin
->dest
.ssa
.num_components
,
236 intrin
->dest
.ssa
.bit_size
, NULL
);
237 nir_builder_instr_insert(b
, &load
->instr
);
239 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
240 nir_src_for_ssa(&load
->dest
.ssa
));
242 unsigned binding_offset
= state
->set
[set
].surface_offsets
[binding
];
243 index
= nir_iadd(b
, index
, nir_imm_int(b
, binding_offset
));
244 brw_nir_rewrite_image_intrinsic(intrin
, index
);
249 lower_load_constant(nir_intrinsic_instr
*intrin
,
250 struct apply_pipeline_layout_state
*state
)
252 nir_builder
*b
= &state
->builder
;
254 b
->cursor
= nir_before_instr(&intrin
->instr
);
256 nir_ssa_def
*index
= nir_imm_int(b
, state
->constants_offset
);
257 nir_ssa_def
*offset
= nir_iadd(b
, nir_ssa_for_src(b
, intrin
->src
[0], 1),
258 nir_imm_int(b
, nir_intrinsic_base(intrin
)));
260 nir_intrinsic_instr
*load_ubo
=
261 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_ubo
);
262 load_ubo
->num_components
= intrin
->num_components
;
263 load_ubo
->src
[0] = nir_src_for_ssa(index
);
264 load_ubo
->src
[1] = nir_src_for_ssa(offset
);
265 nir_ssa_dest_init(&load_ubo
->instr
, &load_ubo
->dest
,
266 intrin
->dest
.ssa
.num_components
,
267 intrin
->dest
.ssa
.bit_size
, NULL
);
268 nir_builder_instr_insert(b
, &load_ubo
->instr
);
270 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
271 nir_src_for_ssa(&load_ubo
->dest
.ssa
));
272 nir_instr_remove(&intrin
->instr
);
276 lower_tex_deref(nir_tex_instr
*tex
, nir_tex_src_type deref_src_type
,
277 unsigned *base_index
,
278 struct apply_pipeline_layout_state
*state
)
280 int deref_src_idx
= nir_tex_instr_src_index(tex
, deref_src_type
);
281 if (deref_src_idx
< 0)
284 nir_deref_instr
*deref
= nir_src_as_deref(tex
->src
[deref_src_idx
].src
);
285 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
287 unsigned set
= var
->data
.descriptor_set
;
288 unsigned binding
= var
->data
.binding
;
289 unsigned array_size
=
290 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
292 nir_tex_src_type offset_src_type
;
293 if (deref_src_type
== nir_tex_src_texture_deref
) {
294 offset_src_type
= nir_tex_src_texture_offset
;
295 *base_index
= state
->set
[set
].surface_offsets
[binding
];
297 assert(deref_src_type
== nir_tex_src_sampler_deref
);
298 offset_src_type
= nir_tex_src_sampler_offset
;
299 *base_index
= state
->set
[set
].sampler_offsets
[binding
];
302 nir_ssa_def
*index
= NULL
;
303 if (deref
->deref_type
!= nir_deref_type_var
) {
304 assert(deref
->deref_type
== nir_deref_type_array
);
306 if (nir_src_is_const(deref
->arr
.index
)) {
307 unsigned arr_index
= nir_src_as_uint(deref
->arr
.index
);
308 *base_index
+= MIN2(arr_index
, array_size
- 1);
310 nir_builder
*b
= &state
->builder
;
312 /* From VK_KHR_sampler_ycbcr_conversion:
314 * If sampler Y’CBCR conversion is enabled, the combined image
315 * sampler must be indexed only by constant integral expressions when
316 * aggregated into arrays in shader code, irrespective of the
317 * shaderSampledImageArrayDynamicIndexing feature.
319 assert(nir_tex_instr_src_index(tex
, nir_tex_src_plane
) == -1);
321 index
= nir_ssa_for_src(b
, deref
->arr
.index
, 1);
323 if (state
->add_bounds_checks
)
324 index
= nir_umin(b
, index
, nir_imm_int(b
, array_size
- 1));
329 nir_instr_rewrite_src(&tex
->instr
, &tex
->src
[deref_src_idx
].src
,
330 nir_src_for_ssa(index
));
331 tex
->src
[deref_src_idx
].src_type
= offset_src_type
;
333 nir_tex_instr_remove_src(tex
, deref_src_idx
);
338 tex_instr_get_and_remove_plane_src(nir_tex_instr
*tex
)
340 int plane_src_idx
= nir_tex_instr_src_index(tex
, nir_tex_src_plane
);
341 if (plane_src_idx
< 0)
344 unsigned plane
= nir_src_as_uint(tex
->src
[plane_src_idx
].src
);
346 nir_tex_instr_remove_src(tex
, plane_src_idx
);
352 lower_tex(nir_tex_instr
*tex
, struct apply_pipeline_layout_state
*state
)
354 state
->builder
.cursor
= nir_before_instr(&tex
->instr
);
356 unsigned plane
= tex_instr_get_and_remove_plane_src(tex
);
358 lower_tex_deref(tex
, nir_tex_src_texture_deref
,
359 &tex
->texture_index
, state
);
360 tex
->texture_index
+= plane
;
362 lower_tex_deref(tex
, nir_tex_src_sampler_deref
,
363 &tex
->sampler_index
, state
);
364 tex
->sampler_index
+= plane
;
366 /* The backend only ever uses this to mark used surfaces. We don't care
367 * about that little optimization so it just needs to be non-zero.
369 tex
->texture_array_size
= 1;
373 apply_pipeline_layout_block(nir_block
*block
,
374 struct apply_pipeline_layout_state
*state
)
376 nir_foreach_instr_safe(instr
, block
) {
377 switch (instr
->type
) {
378 case nir_instr_type_intrinsic
: {
379 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
380 switch (intrin
->intrinsic
) {
381 case nir_intrinsic_vulkan_resource_index
:
382 lower_res_index_intrinsic(intrin
, state
);
384 case nir_intrinsic_vulkan_resource_reindex
:
385 lower_res_reindex_intrinsic(intrin
, state
);
387 case nir_intrinsic_load_vulkan_descriptor
:
388 lower_load_vulkan_descriptor(intrin
, state
);
390 case nir_intrinsic_image_deref_load
:
391 case nir_intrinsic_image_deref_store
:
392 case nir_intrinsic_image_deref_atomic_add
:
393 case nir_intrinsic_image_deref_atomic_min
:
394 case nir_intrinsic_image_deref_atomic_max
:
395 case nir_intrinsic_image_deref_atomic_and
:
396 case nir_intrinsic_image_deref_atomic_or
:
397 case nir_intrinsic_image_deref_atomic_xor
:
398 case nir_intrinsic_image_deref_atomic_exchange
:
399 case nir_intrinsic_image_deref_atomic_comp_swap
:
400 case nir_intrinsic_image_deref_size
:
401 case nir_intrinsic_image_deref_samples
:
402 case nir_intrinsic_image_deref_load_param_intel
:
403 case nir_intrinsic_image_deref_load_raw_intel
:
404 case nir_intrinsic_image_deref_store_raw_intel
:
405 lower_image_intrinsic(intrin
, state
);
407 case nir_intrinsic_load_constant
:
408 lower_load_constant(intrin
, state
);
415 case nir_instr_type_tex
:
416 lower_tex(nir_instr_as_tex(instr
), state
);
425 setup_vec4_uniform_value(uint32_t *params
, uint32_t offset
, unsigned n
)
427 for (unsigned i
= 0; i
< n
; ++i
)
428 params
[i
] = ANV_PARAM_PUSH(offset
+ i
* sizeof(uint32_t));
430 for (unsigned i
= n
; i
< 4; ++i
)
431 params
[i
] = BRW_PARAM_BUILTIN_ZERO
;
435 anv_nir_apply_pipeline_layout(const struct anv_physical_device
*pdevice
,
436 bool robust_buffer_access
,
437 struct anv_pipeline_layout
*layout
,
439 struct brw_stage_prog_data
*prog_data
,
440 struct anv_pipeline_bind_map
*map
)
442 struct apply_pipeline_layout_state state
= {
445 .add_bounds_checks
= robust_buffer_access
,
448 void *mem_ctx
= ralloc_context(NULL
);
450 for (unsigned s
= 0; s
< layout
->num_sets
; s
++) {
451 const unsigned count
= layout
->set
[s
].layout
->binding_count
;
452 const unsigned words
= BITSET_WORDS(count
);
453 state
.set
[s
].used
= rzalloc_array(mem_ctx
, BITSET_WORD
, words
);
454 state
.set
[s
].surface_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
455 state
.set
[s
].sampler_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
456 state
.set
[s
].image_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
459 nir_foreach_function(function
, shader
) {
463 nir_foreach_block(block
, function
->impl
)
464 get_used_bindings_block(block
, &state
);
467 if (state
.uses_constants
) {
468 state
.constants_offset
= map
->surface_count
;
469 map
->surface_to_descriptor
[map
->surface_count
].set
=
470 ANV_DESCRIPTOR_SET_SHADER_CONSTANTS
;
471 map
->surface_count
++;
474 for (uint32_t set
= 0; set
< layout
->num_sets
; set
++) {
475 struct anv_descriptor_set_layout
*set_layout
= layout
->set
[set
].layout
;
478 BITSET_FOREACH_SET(b
, _tmp
, state
.set
[set
].used
,
479 set_layout
->binding_count
) {
480 struct anv_descriptor_set_binding_layout
*binding
=
481 &set_layout
->binding
[b
];
483 if (binding
->array_size
== 0)
486 if (binding
->data
& ANV_DESCRIPTOR_SURFACE_STATE
) {
487 state
.set
[set
].surface_offsets
[b
] = map
->surface_count
;
488 struct anv_sampler
**samplers
= binding
->immutable_samplers
;
489 for (unsigned i
= 0; i
< binding
->array_size
; i
++) {
490 uint8_t planes
= samplers
? samplers
[i
]->n_planes
: 1;
491 for (uint8_t p
= 0; p
< planes
; p
++) {
492 map
->surface_to_descriptor
[map
->surface_count
++] =
493 (struct anv_pipeline_binding
) {
503 if (binding
->data
& ANV_DESCRIPTOR_SAMPLER_STATE
) {
504 state
.set
[set
].sampler_offsets
[b
] = map
->sampler_count
;
505 struct anv_sampler
**samplers
= binding
->immutable_samplers
;
506 for (unsigned i
= 0; i
< binding
->array_size
; i
++) {
507 uint8_t planes
= samplers
? samplers
[i
]->n_planes
: 1;
508 for (uint8_t p
= 0; p
< planes
; p
++) {
509 map
->sampler_to_descriptor
[map
->sampler_count
++] =
510 (struct anv_pipeline_binding
) {
520 if (binding
->data
& ANV_DESCRIPTOR_IMAGE_PARAM
) {
521 state
.set
[set
].image_offsets
[b
] = map
->image_param_count
;
522 map
->image_param_count
+= binding
->array_size
;
527 if (map
->image_param_count
> 0) {
528 assert(map
->image_param_count
<= MAX_GEN8_IMAGES
);
529 assert(shader
->num_uniforms
== prog_data
->nr_params
* 4);
530 state
.first_image_uniform
= shader
->num_uniforms
;
531 uint32_t *param
= brw_stage_prog_data_add_params(prog_data
,
532 map
->image_param_count
*
533 BRW_IMAGE_PARAM_SIZE
);
534 struct anv_push_constants
*null_data
= NULL
;
535 const struct brw_image_param
*image_param
= null_data
->images
;
536 for (uint32_t i
= 0; i
< map
->image_param_count
; i
++) {
537 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_OFFSET_OFFSET
,
538 (uintptr_t)image_param
->offset
, 2);
539 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_SIZE_OFFSET
,
540 (uintptr_t)image_param
->size
, 3);
541 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_STRIDE_OFFSET
,
542 (uintptr_t)image_param
->stride
, 4);
543 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_TILING_OFFSET
,
544 (uintptr_t)image_param
->tiling
, 3);
545 setup_vec4_uniform_value(param
+ BRW_IMAGE_PARAM_SWIZZLING_OFFSET
,
546 (uintptr_t)image_param
->swizzling
, 2);
548 param
+= BRW_IMAGE_PARAM_SIZE
;
551 assert(param
== prog_data
->param
+ prog_data
->nr_params
);
553 shader
->num_uniforms
+= map
->image_param_count
*
554 BRW_IMAGE_PARAM_SIZE
* 4;
555 assert(shader
->num_uniforms
== prog_data
->nr_params
* 4);
558 nir_foreach_variable(var
, &shader
->uniforms
) {
559 const struct glsl_type
*glsl_type
= glsl_without_array(var
->type
);
561 if (!glsl_type_is_image(glsl_type
))
564 enum glsl_sampler_dim dim
= glsl_get_sampler_dim(glsl_type
);
566 const uint32_t set
= var
->data
.descriptor_set
;
567 const uint32_t binding
= var
->data
.binding
;
568 const uint32_t array_size
=
569 layout
->set
[set
].layout
->binding
[binding
].array_size
;
571 if (!BITSET_TEST(state
.set
[set
].used
, binding
))
574 struct anv_pipeline_binding
*pipe_binding
=
575 &map
->surface_to_descriptor
[state
.set
[set
].surface_offsets
[binding
]];
576 for (unsigned i
= 0; i
< array_size
; i
++) {
577 assert(pipe_binding
[i
].set
== set
);
578 assert(pipe_binding
[i
].binding
== binding
);
579 assert(pipe_binding
[i
].index
== i
);
581 if (dim
== GLSL_SAMPLER_DIM_SUBPASS
||
582 dim
== GLSL_SAMPLER_DIM_SUBPASS_MS
)
583 pipe_binding
[i
].input_attachment_index
= var
->data
.index
+ i
;
585 pipe_binding
[i
].write_only
=
586 (var
->data
.image
.access
& ACCESS_NON_READABLE
) != 0;
590 nir_foreach_function(function
, shader
) {
594 nir_builder_init(&state
.builder
, function
->impl
);
595 nir_foreach_block(block
, function
->impl
)
596 apply_pipeline_layout_block(block
, &state
);
597 nir_metadata_preserve(function
->impl
, nir_metadata_block_index
|
598 nir_metadata_dominance
);
601 ralloc_free(mem_ctx
);