2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "program/prog_parameter.h"
26 #include "nir/nir_builder.h"
27 #include "compiler/brw_nir.h"
30 /* Sampler tables don't actually have a maximum size but we pick one just so
31 * that we don't end up emitting too much state on-the-fly.
33 #define MAX_SAMPLER_TABLE_SIZE 128
34 #define BINDLESS_OFFSET 255
36 struct apply_pipeline_layout_state
{
37 const struct anv_physical_device
*pdevice
;
42 struct anv_pipeline_layout
*layout
;
43 bool add_bounds_checks
;
45 /* Place to flag lowered instructions so we don't lower them twice */
46 struct set
*lowered_instrs
;
48 int dynamic_offset_uniform_start
;
51 uint8_t constants_offset
;
53 bool desc_buffer_used
;
57 uint8_t *surface_offsets
;
58 uint8_t *sampler_offsets
;
63 add_binding(struct apply_pipeline_layout_state
*state
,
64 uint32_t set
, uint32_t binding
)
66 const struct anv_descriptor_set_binding_layout
*bind_layout
=
67 &state
->layout
->set
[set
].layout
->binding
[binding
];
69 if (state
->set
[set
].use_count
[binding
] < UINT8_MAX
)
70 state
->set
[set
].use_count
[binding
]++;
72 /* Only flag the descriptor buffer as used if there's actually data for
73 * this binding. This lets us be lazy and call this function constantly
74 * without worrying about unnecessarily enabling the buffer.
76 if (anv_descriptor_size(bind_layout
))
77 state
->set
[set
].desc_buffer_used
= true;
81 add_deref_src_binding(struct apply_pipeline_layout_state
*state
, nir_src src
)
83 nir_deref_instr
*deref
= nir_src_as_deref(src
);
84 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
85 add_binding(state
, var
->data
.descriptor_set
, var
->data
.binding
);
89 add_tex_src_binding(struct apply_pipeline_layout_state
*state
,
90 nir_tex_instr
*tex
, nir_tex_src_type deref_src_type
)
92 int deref_src_idx
= nir_tex_instr_src_index(tex
, deref_src_type
);
93 if (deref_src_idx
< 0)
96 add_deref_src_binding(state
, tex
->src
[deref_src_idx
].src
);
100 get_used_bindings_block(nir_block
*block
,
101 struct apply_pipeline_layout_state
*state
)
103 nir_foreach_instr_safe(instr
, block
) {
104 switch (instr
->type
) {
105 case nir_instr_type_intrinsic
: {
106 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
107 switch (intrin
->intrinsic
) {
108 case nir_intrinsic_vulkan_resource_index
:
109 add_binding(state
, nir_intrinsic_desc_set(intrin
),
110 nir_intrinsic_binding(intrin
));
113 case nir_intrinsic_image_deref_load
:
114 case nir_intrinsic_image_deref_store
:
115 case nir_intrinsic_image_deref_atomic_add
:
116 case nir_intrinsic_image_deref_atomic_min
:
117 case nir_intrinsic_image_deref_atomic_max
:
118 case nir_intrinsic_image_deref_atomic_and
:
119 case nir_intrinsic_image_deref_atomic_or
:
120 case nir_intrinsic_image_deref_atomic_xor
:
121 case nir_intrinsic_image_deref_atomic_exchange
:
122 case nir_intrinsic_image_deref_atomic_comp_swap
:
123 case nir_intrinsic_image_deref_size
:
124 case nir_intrinsic_image_deref_samples
:
125 case nir_intrinsic_image_deref_load_param_intel
:
126 case nir_intrinsic_image_deref_load_raw_intel
:
127 case nir_intrinsic_image_deref_store_raw_intel
:
128 add_deref_src_binding(state
, intrin
->src
[0]);
131 case nir_intrinsic_load_constant
:
132 state
->uses_constants
= true;
140 case nir_instr_type_tex
: {
141 nir_tex_instr
*tex
= nir_instr_as_tex(instr
);
142 add_tex_src_binding(state
, tex
, nir_tex_src_texture_deref
);
143 add_tex_src_binding(state
, tex
, nir_tex_src_sampler_deref
);
153 find_descriptor_for_index_src(nir_src src
,
154 struct apply_pipeline_layout_state
*state
)
156 nir_intrinsic_instr
*intrin
= nir_src_as_intrinsic(src
);
158 while (intrin
&& intrin
->intrinsic
== nir_intrinsic_vulkan_resource_reindex
)
159 intrin
= nir_src_as_intrinsic(intrin
->src
[0]);
161 if (!intrin
|| intrin
->intrinsic
!= nir_intrinsic_vulkan_resource_index
)
164 uint32_t set
= nir_intrinsic_desc_set(intrin
);
165 uint32_t binding
= nir_intrinsic_binding(intrin
);
166 uint32_t surface_index
= state
->set
[set
].surface_offsets
[binding
];
168 /* Only lower to a BTI message if we have a valid binding table index. */
169 return surface_index
< MAX_BINDING_TABLE_SIZE
;
173 nir_deref_find_descriptor(nir_deref_instr
*deref
,
174 struct apply_pipeline_layout_state
*state
)
177 /* Nothing we will use this on has a variable */
178 assert(deref
->deref_type
!= nir_deref_type_var
);
180 nir_deref_instr
*parent
= nir_src_as_deref(deref
->parent
);
186 assert(deref
->deref_type
== nir_deref_type_cast
);
188 nir_intrinsic_instr
*intrin
= nir_src_as_intrinsic(deref
->parent
);
189 if (!intrin
|| intrin
->intrinsic
!= nir_intrinsic_load_vulkan_descriptor
)
192 return find_descriptor_for_index_src(intrin
->src
[0], state
);
196 build_index_for_res_reindex(nir_intrinsic_instr
*intrin
,
197 struct apply_pipeline_layout_state
*state
)
199 nir_builder
*b
= &state
->builder
;
201 if (intrin
->intrinsic
== nir_intrinsic_vulkan_resource_reindex
) {
203 build_index_for_res_reindex(nir_src_as_intrinsic(intrin
->src
[0]), state
);
205 b
->cursor
= nir_before_instr(&intrin
->instr
);
206 return nir_iadd(b
, bti
, nir_ssa_for_src(b
, intrin
->src
[1], 1));
209 assert(intrin
->intrinsic
== nir_intrinsic_vulkan_resource_index
);
211 uint32_t set
= nir_intrinsic_desc_set(intrin
);
212 uint32_t binding
= nir_intrinsic_binding(intrin
);
214 const struct anv_descriptor_set_binding_layout
*bind_layout
=
215 &state
->layout
->set
[set
].layout
->binding
[binding
];
217 uint32_t surface_index
= state
->set
[set
].surface_offsets
[binding
];
218 uint32_t array_size
= bind_layout
->array_size
;
220 b
->cursor
= nir_before_instr(&intrin
->instr
);
222 nir_ssa_def
*array_index
= nir_ssa_for_src(b
, intrin
->src
[0], 1);
223 if (nir_src_is_const(intrin
->src
[0]) || state
->add_bounds_checks
)
224 array_index
= nir_umin(b
, array_index
, nir_imm_int(b
, array_size
- 1));
226 return nir_iadd_imm(b
, array_index
, surface_index
);
230 build_index_offset_for_deref(nir_deref_instr
*deref
,
231 struct apply_pipeline_layout_state
*state
)
233 nir_builder
*b
= &state
->builder
;
235 nir_deref_instr
*parent
= nir_deref_instr_parent(deref
);
237 nir_ssa_def
*addr
= build_index_offset_for_deref(parent
, state
);
239 b
->cursor
= nir_before_instr(&deref
->instr
);
240 return nir_explicit_io_address_from_deref(b
, deref
, addr
,
241 nir_address_format_32bit_index_offset
);
244 nir_intrinsic_instr
*load_desc
= nir_src_as_intrinsic(deref
->parent
);
245 assert(load_desc
->intrinsic
== nir_intrinsic_load_vulkan_descriptor
);
248 build_index_for_res_reindex(nir_src_as_intrinsic(load_desc
->src
[0]), state
);
250 /* Return a 0 offset which will get picked up by the recursion */
251 b
->cursor
= nir_before_instr(&deref
->instr
);
252 return nir_vec2(b
, index
, nir_imm_int(b
, 0));
256 try_lower_direct_buffer_intrinsic(nir_intrinsic_instr
*intrin
, bool is_atomic
,
257 struct apply_pipeline_layout_state
*state
)
259 nir_builder
*b
= &state
->builder
;
261 nir_deref_instr
*deref
= nir_src_as_deref(intrin
->src
[0]);
262 if (deref
->mode
!= nir_var_mem_ssbo
)
265 /* 64-bit atomics only support A64 messages so we can't lower them to the
266 * index+offset model.
268 if (is_atomic
&& nir_dest_bit_size(intrin
->dest
) == 64)
271 if (!nir_deref_find_descriptor(deref
, state
))
274 nir_ssa_def
*addr
= build_index_offset_for_deref(deref
, state
);
276 b
->cursor
= nir_before_instr(&intrin
->instr
);
277 nir_lower_explicit_io_instr(b
, intrin
, addr
,
278 nir_address_format_32bit_index_offset
);
283 lower_direct_buffer_access(nir_function_impl
*impl
,
284 struct apply_pipeline_layout_state
*state
)
286 nir_foreach_block(block
, impl
) {
287 nir_foreach_instr_safe(instr
, block
) {
288 if (instr
->type
!= nir_instr_type_intrinsic
)
291 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
292 switch (intrin
->intrinsic
) {
293 case nir_intrinsic_load_deref
:
294 case nir_intrinsic_store_deref
:
295 try_lower_direct_buffer_intrinsic(intrin
, false, state
);
297 case nir_intrinsic_deref_atomic_add
:
298 case nir_intrinsic_deref_atomic_imin
:
299 case nir_intrinsic_deref_atomic_umin
:
300 case nir_intrinsic_deref_atomic_imax
:
301 case nir_intrinsic_deref_atomic_umax
:
302 case nir_intrinsic_deref_atomic_and
:
303 case nir_intrinsic_deref_atomic_or
:
304 case nir_intrinsic_deref_atomic_xor
:
305 case nir_intrinsic_deref_atomic_exchange
:
306 case nir_intrinsic_deref_atomic_comp_swap
:
307 case nir_intrinsic_deref_atomic_fmin
:
308 case nir_intrinsic_deref_atomic_fmax
:
309 case nir_intrinsic_deref_atomic_fcomp_swap
:
310 try_lower_direct_buffer_intrinsic(intrin
, true, state
);
313 case nir_intrinsic_get_buffer_size
: {
314 /* The get_buffer_size intrinsic always just takes a
315 * index/reindex intrinsic.
317 if (!find_descriptor_for_index_src(intrin
->src
[0], state
))
321 build_index_for_res_reindex(nir_src_as_intrinsic(intrin
->src
[0]),
323 nir_instr_rewrite_src(&intrin
->instr
, &intrin
->src
[0],
324 nir_src_for_ssa(index
));
325 _mesa_set_add(state
->lowered_instrs
, intrin
);
336 lower_res_index_intrinsic(nir_intrinsic_instr
*intrin
,
337 struct apply_pipeline_layout_state
*state
)
339 nir_builder
*b
= &state
->builder
;
341 b
->cursor
= nir_before_instr(&intrin
->instr
);
343 uint32_t set
= nir_intrinsic_desc_set(intrin
);
344 uint32_t binding
= nir_intrinsic_binding(intrin
);
345 const VkDescriptorType desc_type
= nir_intrinsic_desc_type(intrin
);
347 const struct anv_descriptor_set_binding_layout
*bind_layout
=
348 &state
->layout
->set
[set
].layout
->binding
[binding
];
350 uint32_t surface_index
= state
->set
[set
].surface_offsets
[binding
];
351 uint32_t array_size
= bind_layout
->array_size
;
353 nir_ssa_def
*array_index
= nir_ssa_for_src(b
, intrin
->src
[0], 1);
354 if (nir_src_is_const(intrin
->src
[0]) || state
->add_bounds_checks
)
355 array_index
= nir_umin(b
, array_index
, nir_imm_int(b
, array_size
- 1));
358 if (state
->pdevice
->has_a64_buffer_access
&&
359 (desc_type
== VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
||
360 desc_type
== VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC
)) {
361 /* We store the descriptor offset as 16.8.8 where the top 16 bits are
362 * the offset into the descriptor set, the next 8 are the binding table
363 * index of the descriptor buffer, and the bottom 8 bits are the offset
364 * (in bytes) into the dynamic offset table.
366 assert(bind_layout
->dynamic_offset_index
< MAX_DYNAMIC_BUFFERS
);
367 uint32_t dynamic_offset_index
= 0xff; /* No dynamic offset */
368 if (bind_layout
->dynamic_offset_index
>= 0) {
369 dynamic_offset_index
=
370 state
->layout
->set
[set
].dynamic_offset_start
+
371 bind_layout
->dynamic_offset_index
;
374 const uint32_t desc_offset
=
375 bind_layout
->descriptor_offset
<< 16 |
376 (uint32_t)state
->set
[set
].desc_offset
<< 8 |
377 dynamic_offset_index
;
379 if (state
->add_bounds_checks
) {
380 /* We're using nir_address_format_64bit_bounded_global */
381 assert(intrin
->dest
.ssa
.num_components
== 4);
382 assert(intrin
->dest
.ssa
.bit_size
== 32);
383 index
= nir_vec4(b
, nir_imm_int(b
, desc_offset
),
384 nir_ssa_for_src(b
, intrin
->src
[0], 1),
385 nir_imm_int(b
, array_size
- 1),
386 nir_ssa_undef(b
, 1, 32));
388 /* We're using nir_address_format_64bit_global */
389 assert(intrin
->dest
.ssa
.num_components
== 1);
390 assert(intrin
->dest
.ssa
.bit_size
== 64);
391 index
= nir_pack_64_2x32_split(b
, nir_imm_int(b
, desc_offset
),
392 nir_ssa_for_src(b
, intrin
->src
[0], 1));
394 } else if (bind_layout
->data
& ANV_DESCRIPTOR_INLINE_UNIFORM
) {
395 /* This is an inline uniform block. Just reference the descriptor set
396 * and use the descriptor offset as the base. Inline uniforms always
397 * use nir_address_format_32bit_index_offset
399 assert(intrin
->dest
.ssa
.num_components
== 2);
400 assert(intrin
->dest
.ssa
.bit_size
== 32);
401 index
= nir_imm_ivec2(b
, state
->set
[set
].desc_offset
,
402 bind_layout
->descriptor_offset
);
404 /* We're using nir_address_format_32bit_index_offset */
405 assert(intrin
->dest
.ssa
.num_components
== 2);
406 assert(intrin
->dest
.ssa
.bit_size
== 32);
407 index
= nir_vec2(b
, nir_iadd_imm(b
, array_index
, surface_index
),
411 assert(intrin
->dest
.is_ssa
);
412 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(index
));
413 nir_instr_remove(&intrin
->instr
);
417 lower_res_reindex_intrinsic(nir_intrinsic_instr
*intrin
,
418 struct apply_pipeline_layout_state
*state
)
420 nir_builder
*b
= &state
->builder
;
422 b
->cursor
= nir_before_instr(&intrin
->instr
);
424 const VkDescriptorType desc_type
= nir_intrinsic_desc_type(intrin
);
426 /* For us, the resource indices are just indices into the binding table and
427 * array elements are sequential. A resource_reindex just turns into an
428 * add of the two indices.
430 assert(intrin
->src
[0].is_ssa
&& intrin
->src
[1].is_ssa
);
431 nir_ssa_def
*old_index
= intrin
->src
[0].ssa
;
432 nir_ssa_def
*offset
= intrin
->src
[1].ssa
;
434 nir_ssa_def
*new_index
;
435 if (state
->pdevice
->has_a64_buffer_access
&&
436 (desc_type
== VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
||
437 desc_type
== VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC
)) {
438 if (state
->add_bounds_checks
) {
439 /* We're using nir_address_format_64bit_bounded_global */
440 assert(intrin
->dest
.ssa
.num_components
== 4);
441 assert(intrin
->dest
.ssa
.bit_size
== 32);
442 new_index
= nir_vec4(b
, nir_channel(b
, old_index
, 0),
443 nir_iadd(b
, nir_channel(b
, old_index
, 1),
445 nir_channel(b
, old_index
, 2),
446 nir_ssa_undef(b
, 1, 32));
448 /* We're using nir_address_format_64bit_global */
449 assert(intrin
->dest
.ssa
.num_components
== 1);
450 assert(intrin
->dest
.ssa
.bit_size
== 64);
451 nir_ssa_def
*base
= nir_unpack_64_2x32_split_x(b
, old_index
);
452 nir_ssa_def
*arr_idx
= nir_unpack_64_2x32_split_y(b
, old_index
);
453 new_index
= nir_pack_64_2x32_split(b
, base
, nir_iadd(b
, arr_idx
, offset
));
456 /* We're using nir_address_format_32bit_index_offset */
457 assert(intrin
->dest
.ssa
.num_components
== 2);
458 assert(intrin
->dest
.ssa
.bit_size
== 32);
459 new_index
= nir_vec2(b
, nir_iadd(b
, nir_channel(b
, old_index
, 0), offset
),
460 nir_channel(b
, old_index
, 1));
463 assert(intrin
->dest
.is_ssa
);
464 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(new_index
));
465 nir_instr_remove(&intrin
->instr
);
469 build_ssbo_descriptor_load(const VkDescriptorType desc_type
,
471 struct apply_pipeline_layout_state
*state
)
473 nir_builder
*b
= &state
->builder
;
475 nir_ssa_def
*desc_offset
, *array_index
;
476 if (state
->add_bounds_checks
) {
477 /* We're using nir_address_format_64bit_bounded_global */
478 desc_offset
= nir_channel(b
, index
, 0);
479 array_index
= nir_umin(b
, nir_channel(b
, index
, 1),
480 nir_channel(b
, index
, 2));
482 desc_offset
= nir_unpack_64_2x32_split_x(b
, index
);
483 array_index
= nir_unpack_64_2x32_split_y(b
, index
);
486 /* The desc_offset is actually 16.8.8 */
487 nir_ssa_def
*desc_buffer_index
=
488 nir_extract_u8(b
, desc_offset
, nir_imm_int(b
, 1));
489 nir_ssa_def
*desc_offset_base
=
490 nir_extract_u16(b
, desc_offset
, nir_imm_int(b
, 1));
492 /* Compute the actual descriptor offset */
493 const unsigned descriptor_size
=
494 anv_descriptor_type_size(state
->pdevice
, desc_type
);
495 desc_offset
= nir_iadd(b
, desc_offset_base
,
496 nir_imul_imm(b
, array_index
, descriptor_size
));
498 nir_intrinsic_instr
*desc_load
=
499 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_ubo
);
500 desc_load
->src
[0] = nir_src_for_ssa(desc_buffer_index
);
501 desc_load
->src
[1] = nir_src_for_ssa(desc_offset
);
502 desc_load
->num_components
= 4;
503 nir_ssa_dest_init(&desc_load
->instr
, &desc_load
->dest
, 4, 32, NULL
);
504 nir_builder_instr_insert(b
, &desc_load
->instr
);
506 return &desc_load
->dest
.ssa
;
510 lower_load_vulkan_descriptor(nir_intrinsic_instr
*intrin
,
511 struct apply_pipeline_layout_state
*state
)
513 nir_builder
*b
= &state
->builder
;
515 b
->cursor
= nir_before_instr(&intrin
->instr
);
517 const VkDescriptorType desc_type
= nir_intrinsic_desc_type(intrin
);
519 assert(intrin
->src
[0].is_ssa
);
520 nir_ssa_def
*index
= intrin
->src
[0].ssa
;
523 if (state
->pdevice
->has_a64_buffer_access
&&
524 (desc_type
== VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
||
525 desc_type
== VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC
)) {
526 desc
= build_ssbo_descriptor_load(desc_type
, index
, state
);
528 /* We want nir_address_format_64bit_global */
529 if (!state
->add_bounds_checks
)
530 desc
= nir_pack_64_2x32(b
, nir_channels(b
, desc
, 0x3));
532 if (state
->dynamic_offset_uniform_start
>= 0) {
533 /* This shader has dynamic offsets and we have no way of knowing
534 * (save from the dynamic offset base index) if this buffer has a
537 nir_ssa_def
*desc_offset
, *array_index
;
538 if (state
->add_bounds_checks
) {
539 /* We're using nir_address_format_64bit_bounded_global */
540 desc_offset
= nir_channel(b
, index
, 0);
541 array_index
= nir_umin(b
, nir_channel(b
, index
, 1),
542 nir_channel(b
, index
, 2));
544 desc_offset
= nir_unpack_64_2x32_split_x(b
, index
);
545 array_index
= nir_unpack_64_2x32_split_y(b
, index
);
548 nir_ssa_def
*dyn_offset_base
=
549 nir_extract_u8(b
, desc_offset
, nir_imm_int(b
, 0));
550 nir_ssa_def
*dyn_offset_idx
=
551 nir_iadd(b
, dyn_offset_base
, array_index
);
552 if (state
->add_bounds_checks
) {
553 dyn_offset_idx
= nir_umin(b
, dyn_offset_idx
,
554 nir_imm_int(b
, MAX_DYNAMIC_BUFFERS
));
557 nir_intrinsic_instr
*dyn_load
=
558 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_uniform
);
559 nir_intrinsic_set_base(dyn_load
, state
->dynamic_offset_uniform_start
);
560 nir_intrinsic_set_range(dyn_load
, MAX_DYNAMIC_BUFFERS
* 4);
561 dyn_load
->src
[0] = nir_src_for_ssa(nir_imul_imm(b
, dyn_offset_idx
, 4));
562 dyn_load
->num_components
= 1;
563 nir_ssa_dest_init(&dyn_load
->instr
, &dyn_load
->dest
, 1, 32, NULL
);
564 nir_builder_instr_insert(b
, &dyn_load
->instr
);
566 nir_ssa_def
*dynamic_offset
=
567 nir_bcsel(b
, nir_ieq(b
, dyn_offset_base
, nir_imm_int(b
, 0xff)),
568 nir_imm_int(b
, 0), &dyn_load
->dest
.ssa
);
570 if (state
->add_bounds_checks
) {
571 /* The dynamic offset gets added to the base pointer so that we
572 * have a sliding window range.
574 * We're using nir_address_format_64bit_bounded_global.
576 nir_ssa_def
*base_ptr
=
577 nir_pack_64_2x32(b
, nir_channels(b
, desc
, 0x3));
578 base_ptr
= nir_iadd(b
, base_ptr
, nir_u2u64(b
, dynamic_offset
));
579 desc
= nir_vec4(b
, nir_unpack_64_2x32_split_x(b
, base_ptr
),
580 nir_unpack_64_2x32_split_y(b
, base_ptr
),
581 nir_channel(b
, desc
, 2),
582 nir_channel(b
, desc
, 3));
584 /* We're using nir_address_format_64bit_global */
585 desc
= nir_iadd(b
, desc
, nir_u2u64(b
, dynamic_offset
));
589 /* We follow the nir_address_format_32bit_index_offset model */
593 assert(intrin
->dest
.is_ssa
);
594 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(desc
));
595 nir_instr_remove(&intrin
->instr
);
599 lower_get_buffer_size(nir_intrinsic_instr
*intrin
,
600 struct apply_pipeline_layout_state
*state
)
602 if (_mesa_set_search(state
->lowered_instrs
, intrin
))
605 nir_builder
*b
= &state
->builder
;
607 b
->cursor
= nir_before_instr(&intrin
->instr
);
609 const VkDescriptorType desc_type
= VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
;
611 assert(intrin
->src
[0].is_ssa
);
612 nir_ssa_def
*index
= intrin
->src
[0].ssa
;
614 if (state
->pdevice
->has_a64_buffer_access
) {
615 nir_ssa_def
*desc
= build_ssbo_descriptor_load(desc_type
, index
, state
);
616 nir_ssa_def
*size
= nir_channel(b
, desc
, 2);
617 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(size
));
618 nir_instr_remove(&intrin
->instr
);
620 /* We're following the nir_address_format_32bit_index_offset model so
621 * the binding table index is the first component of the address. The
622 * back-end wants a scalar binding table index source.
624 nir_instr_rewrite_src(&intrin
->instr
, &intrin
->src
[0],
625 nir_src_for_ssa(nir_channel(b
, index
, 0)));
630 build_descriptor_load(nir_deref_instr
*deref
, unsigned offset
,
631 unsigned num_components
, unsigned bit_size
,
632 struct apply_pipeline_layout_state
*state
)
634 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
636 unsigned set
= var
->data
.descriptor_set
;
637 unsigned binding
= var
->data
.binding
;
638 unsigned array_size
=
639 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
641 const struct anv_descriptor_set_binding_layout
*bind_layout
=
642 &state
->layout
->set
[set
].layout
->binding
[binding
];
644 nir_builder
*b
= &state
->builder
;
646 nir_ssa_def
*desc_buffer_index
=
647 nir_imm_int(b
, state
->set
[set
].desc_offset
);
649 nir_ssa_def
*desc_offset
=
650 nir_imm_int(b
, bind_layout
->descriptor_offset
+ offset
);
651 if (deref
->deref_type
!= nir_deref_type_var
) {
652 assert(deref
->deref_type
== nir_deref_type_array
);
654 const unsigned descriptor_size
= anv_descriptor_size(bind_layout
);
655 nir_ssa_def
*arr_index
= nir_ssa_for_src(b
, deref
->arr
.index
, 1);
656 if (state
->add_bounds_checks
)
657 arr_index
= nir_umin(b
, arr_index
, nir_imm_int(b
, array_size
- 1));
659 desc_offset
= nir_iadd(b
, desc_offset
,
660 nir_imul_imm(b
, arr_index
, descriptor_size
));
663 nir_intrinsic_instr
*desc_load
=
664 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_ubo
);
665 desc_load
->src
[0] = nir_src_for_ssa(desc_buffer_index
);
666 desc_load
->src
[1] = nir_src_for_ssa(desc_offset
);
667 desc_load
->num_components
= num_components
;
668 nir_ssa_dest_init(&desc_load
->instr
, &desc_load
->dest
,
669 num_components
, bit_size
, NULL
);
670 nir_builder_instr_insert(b
, &desc_load
->instr
);
672 return &desc_load
->dest
.ssa
;
676 lower_image_intrinsic(nir_intrinsic_instr
*intrin
,
677 struct apply_pipeline_layout_state
*state
)
679 nir_deref_instr
*deref
= nir_src_as_deref(intrin
->src
[0]);
681 nir_builder
*b
= &state
->builder
;
682 b
->cursor
= nir_before_instr(&intrin
->instr
);
684 if (intrin
->intrinsic
== nir_intrinsic_image_deref_load_param_intel
) {
685 b
->cursor
= nir_instr_remove(&intrin
->instr
);
687 const unsigned param
= nir_intrinsic_base(intrin
);
690 build_descriptor_load(deref
, param
* 16,
691 intrin
->dest
.ssa
.num_components
,
692 intrin
->dest
.ssa
.bit_size
, state
);
694 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
, nir_src_for_ssa(desc
));
696 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
698 unsigned set
= var
->data
.descriptor_set
;
699 unsigned binding
= var
->data
.binding
;
700 unsigned binding_offset
= state
->set
[set
].surface_offsets
[binding
];
701 unsigned array_size
=
702 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
704 nir_ssa_def
*index
= NULL
;
705 if (deref
->deref_type
!= nir_deref_type_var
) {
706 assert(deref
->deref_type
== nir_deref_type_array
);
707 index
= nir_ssa_for_src(b
, deref
->arr
.index
, 1);
708 if (state
->add_bounds_checks
)
709 index
= nir_umin(b
, index
, nir_imm_int(b
, array_size
- 1));
711 index
= nir_imm_int(b
, 0);
714 index
= nir_iadd_imm(b
, index
, binding_offset
);
715 nir_rewrite_image_intrinsic(intrin
, index
, false);
720 lower_load_constant(nir_intrinsic_instr
*intrin
,
721 struct apply_pipeline_layout_state
*state
)
723 nir_builder
*b
= &state
->builder
;
725 b
->cursor
= nir_before_instr(&intrin
->instr
);
727 nir_ssa_def
*index
= nir_imm_int(b
, state
->constants_offset
);
728 nir_ssa_def
*offset
= nir_iadd(b
, nir_ssa_for_src(b
, intrin
->src
[0], 1),
729 nir_imm_int(b
, nir_intrinsic_base(intrin
)));
731 nir_intrinsic_instr
*load_ubo
=
732 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_ubo
);
733 load_ubo
->num_components
= intrin
->num_components
;
734 load_ubo
->src
[0] = nir_src_for_ssa(index
);
735 load_ubo
->src
[1] = nir_src_for_ssa(offset
);
736 nir_ssa_dest_init(&load_ubo
->instr
, &load_ubo
->dest
,
737 intrin
->dest
.ssa
.num_components
,
738 intrin
->dest
.ssa
.bit_size
, NULL
);
739 nir_builder_instr_insert(b
, &load_ubo
->instr
);
741 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
742 nir_src_for_ssa(&load_ubo
->dest
.ssa
));
743 nir_instr_remove(&intrin
->instr
);
747 lower_tex_deref(nir_tex_instr
*tex
, nir_tex_src_type deref_src_type
,
748 unsigned *base_index
, unsigned plane
,
749 struct apply_pipeline_layout_state
*state
)
751 int deref_src_idx
= nir_tex_instr_src_index(tex
, deref_src_type
);
752 if (deref_src_idx
< 0)
755 nir_deref_instr
*deref
= nir_src_as_deref(tex
->src
[deref_src_idx
].src
);
756 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
758 unsigned set
= var
->data
.descriptor_set
;
759 unsigned binding
= var
->data
.binding
;
760 unsigned array_size
=
761 state
->layout
->set
[set
].layout
->binding
[binding
].array_size
;
763 unsigned binding_offset
;
764 if (deref_src_type
== nir_tex_src_texture_deref
) {
765 binding_offset
= state
->set
[set
].surface_offsets
[binding
];
767 assert(deref_src_type
== nir_tex_src_sampler_deref
);
768 binding_offset
= state
->set
[set
].sampler_offsets
[binding
];
771 nir_builder
*b
= &state
->builder
;
773 nir_tex_src_type offset_src_type
;
774 nir_ssa_def
*index
= NULL
;
775 if (binding_offset
> MAX_BINDING_TABLE_SIZE
) {
776 const unsigned plane_offset
=
777 plane
* sizeof(struct anv_sampled_image_descriptor
);
780 build_descriptor_load(deref
, plane_offset
, 2, 32, state
);
782 if (deref_src_type
== nir_tex_src_texture_deref
) {
783 offset_src_type
= nir_tex_src_texture_handle
;
784 index
= nir_channel(b
, desc
, 0);
786 assert(deref_src_type
== nir_tex_src_sampler_deref
);
787 offset_src_type
= nir_tex_src_sampler_handle
;
788 index
= nir_channel(b
, desc
, 1);
791 if (deref_src_type
== nir_tex_src_texture_deref
) {
792 offset_src_type
= nir_tex_src_texture_offset
;
794 assert(deref_src_type
== nir_tex_src_sampler_deref
);
795 offset_src_type
= nir_tex_src_sampler_offset
;
798 *base_index
= binding_offset
+ plane
;
800 if (deref
->deref_type
!= nir_deref_type_var
) {
801 assert(deref
->deref_type
== nir_deref_type_array
);
803 if (nir_src_is_const(deref
->arr
.index
)) {
804 unsigned arr_index
= nir_src_as_uint(deref
->arr
.index
);
805 *base_index
+= MIN2(arr_index
, array_size
- 1);
807 /* From VK_KHR_sampler_ycbcr_conversion:
809 * If sampler Y’CBCR conversion is enabled, the combined image
810 * sampler must be indexed only by constant integral expressions
811 * when aggregated into arrays in shader code, irrespective of
812 * the shaderSampledImageArrayDynamicIndexing feature.
814 assert(nir_tex_instr_src_index(tex
, nir_tex_src_plane
) == -1);
816 index
= nir_ssa_for_src(b
, deref
->arr
.index
, 1);
818 if (state
->add_bounds_checks
)
819 index
= nir_umin(b
, index
, nir_imm_int(b
, array_size
- 1));
825 nir_instr_rewrite_src(&tex
->instr
, &tex
->src
[deref_src_idx
].src
,
826 nir_src_for_ssa(index
));
827 tex
->src
[deref_src_idx
].src_type
= offset_src_type
;
829 nir_tex_instr_remove_src(tex
, deref_src_idx
);
834 tex_instr_get_and_remove_plane_src(nir_tex_instr
*tex
)
836 int plane_src_idx
= nir_tex_instr_src_index(tex
, nir_tex_src_plane
);
837 if (plane_src_idx
< 0)
840 unsigned plane
= nir_src_as_uint(tex
->src
[plane_src_idx
].src
);
842 nir_tex_instr_remove_src(tex
, plane_src_idx
);
848 lower_tex(nir_tex_instr
*tex
, struct apply_pipeline_layout_state
*state
)
850 state
->builder
.cursor
= nir_before_instr(&tex
->instr
);
852 unsigned plane
= tex_instr_get_and_remove_plane_src(tex
);
854 lower_tex_deref(tex
, nir_tex_src_texture_deref
,
855 &tex
->texture_index
, plane
, state
);
857 lower_tex_deref(tex
, nir_tex_src_sampler_deref
,
858 &tex
->sampler_index
, plane
, state
);
860 /* The backend only ever uses this to mark used surfaces. We don't care
861 * about that little optimization so it just needs to be non-zero.
863 tex
->texture_array_size
= 1;
867 apply_pipeline_layout_block(nir_block
*block
,
868 struct apply_pipeline_layout_state
*state
)
870 nir_foreach_instr_safe(instr
, block
) {
871 switch (instr
->type
) {
872 case nir_instr_type_intrinsic
: {
873 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
874 switch (intrin
->intrinsic
) {
875 case nir_intrinsic_vulkan_resource_index
:
876 lower_res_index_intrinsic(intrin
, state
);
878 case nir_intrinsic_vulkan_resource_reindex
:
879 lower_res_reindex_intrinsic(intrin
, state
);
881 case nir_intrinsic_load_vulkan_descriptor
:
882 lower_load_vulkan_descriptor(intrin
, state
);
884 case nir_intrinsic_get_buffer_size
:
885 lower_get_buffer_size(intrin
, state
);
887 case nir_intrinsic_image_deref_load
:
888 case nir_intrinsic_image_deref_store
:
889 case nir_intrinsic_image_deref_atomic_add
:
890 case nir_intrinsic_image_deref_atomic_min
:
891 case nir_intrinsic_image_deref_atomic_max
:
892 case nir_intrinsic_image_deref_atomic_and
:
893 case nir_intrinsic_image_deref_atomic_or
:
894 case nir_intrinsic_image_deref_atomic_xor
:
895 case nir_intrinsic_image_deref_atomic_exchange
:
896 case nir_intrinsic_image_deref_atomic_comp_swap
:
897 case nir_intrinsic_image_deref_size
:
898 case nir_intrinsic_image_deref_samples
:
899 case nir_intrinsic_image_deref_load_param_intel
:
900 case nir_intrinsic_image_deref_load_raw_intel
:
901 case nir_intrinsic_image_deref_store_raw_intel
:
902 lower_image_intrinsic(intrin
, state
);
904 case nir_intrinsic_load_constant
:
905 lower_load_constant(intrin
, state
);
912 case nir_instr_type_tex
:
913 lower_tex(nir_instr_as_tex(instr
), state
);
921 struct binding_info
{
928 compare_binding_infos(const void *_a
, const void *_b
)
930 const struct binding_info
*a
= _a
, *b
= _b
;
931 if (a
->score
!= b
->score
)
932 return b
->score
- a
->score
;
934 if (a
->set
!= b
->set
)
935 return a
->set
- b
->set
;
937 return a
->binding
- b
->binding
;
941 anv_nir_apply_pipeline_layout(const struct anv_physical_device
*pdevice
,
942 bool robust_buffer_access
,
943 struct anv_pipeline_layout
*layout
,
945 struct brw_stage_prog_data
*prog_data
,
946 struct anv_pipeline_bind_map
*map
)
948 void *mem_ctx
= ralloc_context(NULL
);
950 struct apply_pipeline_layout_state state
= {
954 .add_bounds_checks
= robust_buffer_access
,
955 .lowered_instrs
= _mesa_pointer_set_create(mem_ctx
),
956 .dynamic_offset_uniform_start
= -1,
959 for (unsigned s
= 0; s
< layout
->num_sets
; s
++) {
960 const unsigned count
= layout
->set
[s
].layout
->binding_count
;
961 state
.set
[s
].use_count
= rzalloc_array(mem_ctx
, uint8_t, count
);
962 state
.set
[s
].surface_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
963 state
.set
[s
].sampler_offsets
= rzalloc_array(mem_ctx
, uint8_t, count
);
966 nir_foreach_function(function
, shader
) {
970 nir_foreach_block(block
, function
->impl
)
971 get_used_bindings_block(block
, &state
);
974 for (unsigned s
= 0; s
< layout
->num_sets
; s
++) {
975 if (state
.set
[s
].desc_buffer_used
) {
976 map
->surface_to_descriptor
[map
->surface_count
] =
977 (struct anv_pipeline_binding
) {
978 .set
= ANV_DESCRIPTOR_SET_DESCRIPTORS
,
981 state
.set
[s
].desc_offset
= map
->surface_count
;
982 map
->surface_count
++;
986 if (state
.uses_constants
) {
987 state
.constants_offset
= map
->surface_count
;
988 map
->surface_to_descriptor
[map
->surface_count
].set
=
989 ANV_DESCRIPTOR_SET_SHADER_CONSTANTS
;
990 map
->surface_count
++;
993 unsigned used_binding_count
= 0;
994 for (uint32_t set
= 0; set
< layout
->num_sets
; set
++) {
995 struct anv_descriptor_set_layout
*set_layout
= layout
->set
[set
].layout
;
996 for (unsigned b
= 0; b
< set_layout
->binding_count
; b
++) {
997 if (state
.set
[set
].use_count
[b
] == 0)
1000 used_binding_count
++;
1004 struct binding_info
*infos
=
1005 rzalloc_array(mem_ctx
, struct binding_info
, used_binding_count
);
1006 used_binding_count
= 0;
1007 for (uint32_t set
= 0; set
< layout
->num_sets
; set
++) {
1008 struct anv_descriptor_set_layout
*set_layout
= layout
->set
[set
].layout
;
1009 for (unsigned b
= 0; b
< set_layout
->binding_count
; b
++) {
1010 if (state
.set
[set
].use_count
[b
] == 0)
1013 struct anv_descriptor_set_binding_layout
*binding
=
1014 &layout
->set
[set
].layout
->binding
[b
];
1016 /* Do a fixed-point calculation to generate a score based on the
1017 * number of uses and the binding array size. We shift by 7 instead
1018 * of 8 because we're going to use the top bit below to make
1019 * everything which does not support bindless super higher priority
1020 * than things which do.
1022 uint16_t score
= ((uint16_t)state
.set
[set
].use_count
[b
] << 7) /
1023 binding
->array_size
;
1025 /* If the descriptor type doesn't support bindless then put it at the
1026 * beginning so we guarantee it gets a slot.
1028 if (!anv_descriptor_supports_bindless(pdevice
, binding
, true) ||
1029 !anv_descriptor_supports_bindless(pdevice
, binding
, false))
1032 infos
[used_binding_count
++] = (struct binding_info
) {
1040 /* Order the binding infos based on score with highest scores first. If
1041 * scores are equal we then order by set and binding.
1043 qsort(infos
, used_binding_count
, sizeof(struct binding_info
),
1044 compare_binding_infos
);
1046 bool have_dynamic_buffers
= false;
1048 for (unsigned i
= 0; i
< used_binding_count
; i
++) {
1049 unsigned set
= infos
[i
].set
, b
= infos
[i
].binding
;
1050 struct anv_descriptor_set_binding_layout
*binding
=
1051 &layout
->set
[set
].layout
->binding
[b
];
1053 if (binding
->dynamic_offset_index
>= 0)
1054 have_dynamic_buffers
= true;
1056 const uint32_t array_size
= binding
->array_size
;
1058 if (binding
->data
& ANV_DESCRIPTOR_SURFACE_STATE
) {
1059 if (map
->surface_count
+ array_size
> MAX_BINDING_TABLE_SIZE
||
1060 anv_descriptor_requires_bindless(pdevice
, binding
, false)) {
1061 /* If this descriptor doesn't fit in the binding table or if it
1062 * requires bindless for some reason, flag it as bindless.
1064 assert(anv_descriptor_supports_bindless(pdevice
, binding
, false));
1065 state
.set
[set
].surface_offsets
[b
] = BINDLESS_OFFSET
;
1067 state
.set
[set
].surface_offsets
[b
] = map
->surface_count
;
1068 struct anv_sampler
**samplers
= binding
->immutable_samplers
;
1069 for (unsigned i
= 0; i
< binding
->array_size
; i
++) {
1070 uint8_t planes
= samplers
? samplers
[i
]->n_planes
: 1;
1071 for (uint8_t p
= 0; p
< planes
; p
++) {
1072 map
->surface_to_descriptor
[map
->surface_count
++] =
1073 (struct anv_pipeline_binding
) {
1082 assert(map
->surface_count
<= MAX_BINDING_TABLE_SIZE
);
1085 if (binding
->data
& ANV_DESCRIPTOR_SAMPLER_STATE
) {
1086 if (map
->sampler_count
+ array_size
> MAX_SAMPLER_TABLE_SIZE
||
1087 anv_descriptor_requires_bindless(pdevice
, binding
, true)) {
1088 /* If this descriptor doesn't fit in the binding table or if it
1089 * requires bindless for some reason, flag it as bindless.
1091 * We also make large sampler arrays bindless because we can avoid
1092 * using indirect sends thanks to bindless samplers being packed
1093 * less tightly than the sampler table.
1095 assert(anv_descriptor_supports_bindless(pdevice
, binding
, true));
1096 state
.set
[set
].sampler_offsets
[b
] = BINDLESS_OFFSET
;
1098 state
.set
[set
].sampler_offsets
[b
] = map
->sampler_count
;
1099 struct anv_sampler
**samplers
= binding
->immutable_samplers
;
1100 for (unsigned i
= 0; i
< binding
->array_size
; i
++) {
1101 uint8_t planes
= samplers
? samplers
[i
]->n_planes
: 1;
1102 for (uint8_t p
= 0; p
< planes
; p
++) {
1103 map
->sampler_to_descriptor
[map
->sampler_count
++] =
1104 (struct anv_pipeline_binding
) {
1116 if (have_dynamic_buffers
) {
1117 state
.dynamic_offset_uniform_start
= shader
->num_uniforms
;
1118 uint32_t *param
= brw_stage_prog_data_add_params(prog_data
,
1119 MAX_DYNAMIC_BUFFERS
);
1120 for (unsigned i
= 0; i
< MAX_DYNAMIC_BUFFERS
; i
++)
1121 param
[i
] = ANV_PARAM_DYN_OFFSET(i
);
1122 shader
->num_uniforms
+= MAX_DYNAMIC_BUFFERS
* 4;
1123 assert(shader
->num_uniforms
== prog_data
->nr_params
* 4);
1126 nir_foreach_variable(var
, &shader
->uniforms
) {
1127 const struct glsl_type
*glsl_type
= glsl_without_array(var
->type
);
1129 if (!glsl_type_is_image(glsl_type
))
1132 enum glsl_sampler_dim dim
= glsl_get_sampler_dim(glsl_type
);
1134 const uint32_t set
= var
->data
.descriptor_set
;
1135 const uint32_t binding
= var
->data
.binding
;
1136 const uint32_t array_size
=
1137 layout
->set
[set
].layout
->binding
[binding
].array_size
;
1139 if (state
.set
[set
].use_count
[binding
] == 0)
1142 if (state
.set
[set
].surface_offsets
[binding
] >= MAX_BINDING_TABLE_SIZE
)
1145 struct anv_pipeline_binding
*pipe_binding
=
1146 &map
->surface_to_descriptor
[state
.set
[set
].surface_offsets
[binding
]];
1147 for (unsigned i
= 0; i
< array_size
; i
++) {
1148 assert(pipe_binding
[i
].set
== set
);
1149 assert(pipe_binding
[i
].binding
== binding
);
1150 assert(pipe_binding
[i
].index
== i
);
1152 if (dim
== GLSL_SAMPLER_DIM_SUBPASS
||
1153 dim
== GLSL_SAMPLER_DIM_SUBPASS_MS
)
1154 pipe_binding
[i
].input_attachment_index
= var
->data
.index
+ i
;
1156 pipe_binding
[i
].write_only
=
1157 (var
->data
.image
.access
& ACCESS_NON_READABLE
) != 0;
1161 nir_foreach_function(function
, shader
) {
1162 if (!function
->impl
)
1165 /* Before we do the normal lowering, we look for any SSBO operations
1166 * that we can lower to the BTI model and lower them up-front. The BTI
1167 * model can perform better than the A64 model for a couple reasons:
1169 * 1. 48-bit address calculations are potentially expensive and using
1170 * the BTI model lets us simply compute 32-bit offsets and the
1171 * hardware adds the 64-bit surface base address.
1173 * 2. The BTI messages, because they use surface states, do bounds
1174 * checking for us. With the A64 model, we have to do our own
1175 * bounds checking and this means wider pointers and extra
1176 * calculations and branching in the shader.
1178 * The solution to both of these is to convert things to the BTI model
1179 * opportunistically. The reason why we need to do this as a pre-pass
1180 * is for two reasons:
1182 * 1. The BTI model requires nir_address_format_32bit_index_offset
1183 * pointers which are not the same type as the pointers needed for
1184 * the A64 model. Because all our derefs are set up for the A64
1185 * model (in case we have variable pointers), we have to crawl all
1186 * the way back to the vulkan_resource_index intrinsic and build a
1187 * completely fresh index+offset calculation.
1189 * 2. Because the variable-pointers-capable lowering that we do as part
1190 * of apply_pipeline_layout_block is destructive (It really has to
1191 * be to handle variable pointers properly), we've lost the deref
1192 * information by the time we get to the load/store/atomic
1193 * intrinsics in that pass.
1195 lower_direct_buffer_access(function
->impl
, &state
);
1197 nir_builder_init(&state
.builder
, function
->impl
);
1198 nir_foreach_block(block
, function
->impl
)
1199 apply_pipeline_layout_block(block
, &state
);
1200 nir_metadata_preserve(function
->impl
, nir_metadata_block_index
|
1201 nir_metadata_dominance
);
1204 ralloc_free(mem_ctx
);