2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "vtn_private.h"
25 #include "nir/nir_vla.h"
27 static struct vtn_block
*
28 vtn_block(struct vtn_builder
*b
, uint32_t value_id
)
30 return vtn_value(b
, value_id
, vtn_value_type_block
)->block
;
33 static struct vtn_pointer
*
34 vtn_load_param_pointer(struct vtn_builder
*b
,
35 struct vtn_type
*param_type
,
38 struct vtn_type
*ptr_type
= param_type
;
39 if (param_type
->base_type
!= vtn_base_type_pointer
) {
40 assert(param_type
->base_type
== vtn_base_type_image
||
41 param_type
->base_type
== vtn_base_type_sampler
);
42 ptr_type
= rzalloc(b
, struct vtn_type
);
43 ptr_type
->base_type
= vtn_base_type_pointer
;
44 ptr_type
->deref
= param_type
;
45 ptr_type
->storage_class
= SpvStorageClassUniformConstant
;
48 return vtn_pointer_from_ssa(b
, nir_load_param(&b
->nb
, param_idx
), ptr_type
);
52 vtn_type_count_function_params(struct vtn_type
*type
)
54 switch (type
->base_type
) {
55 case vtn_base_type_array
:
56 case vtn_base_type_matrix
:
57 return type
->length
* vtn_type_count_function_params(type
->array_element
);
59 case vtn_base_type_struct
: {
61 for (unsigned i
= 0; i
< type
->length
; i
++)
62 count
+= vtn_type_count_function_params(type
->members
[i
]);
66 case vtn_base_type_sampled_image
:
75 vtn_type_add_to_function_params(struct vtn_type
*type
,
79 static const nir_parameter nir_deref_param
= {
84 switch (type
->base_type
) {
85 case vtn_base_type_array
:
86 case vtn_base_type_matrix
:
87 for (unsigned i
= 0; i
< type
->length
; i
++)
88 vtn_type_add_to_function_params(type
->array_element
, func
, param_idx
);
91 case vtn_base_type_struct
:
92 for (unsigned i
= 0; i
< type
->length
; i
++)
93 vtn_type_add_to_function_params(type
->members
[i
], func
, param_idx
);
96 case vtn_base_type_sampled_image
:
97 func
->params
[(*param_idx
)++] = nir_deref_param
;
98 func
->params
[(*param_idx
)++] = nir_deref_param
;
101 case vtn_base_type_image
:
102 case vtn_base_type_sampler
:
103 func
->params
[(*param_idx
)++] = nir_deref_param
;
106 case vtn_base_type_pointer
:
108 func
->params
[(*param_idx
)++] = (nir_parameter
) {
109 .num_components
= glsl_get_vector_elements(type
->type
),
110 .bit_size
= glsl_get_bit_size(type
->type
),
113 func
->params
[(*param_idx
)++] = nir_deref_param
;
118 func
->params
[(*param_idx
)++] = (nir_parameter
) {
119 .num_components
= glsl_get_vector_elements(type
->type
),
120 .bit_size
= glsl_get_bit_size(type
->type
),
126 vtn_ssa_value_add_to_call_params(struct vtn_builder
*b
,
127 struct vtn_ssa_value
*value
,
128 struct vtn_type
*type
,
129 nir_call_instr
*call
,
132 switch (type
->base_type
) {
133 case vtn_base_type_array
:
134 case vtn_base_type_matrix
:
135 for (unsigned i
= 0; i
< type
->length
; i
++) {
136 vtn_ssa_value_add_to_call_params(b
, value
->elems
[i
],
142 case vtn_base_type_struct
:
143 for (unsigned i
= 0; i
< type
->length
; i
++) {
144 vtn_ssa_value_add_to_call_params(b
, value
->elems
[i
],
151 call
->params
[(*param_idx
)++] = nir_src_for_ssa(value
->def
);
157 vtn_ssa_value_load_function_param(struct vtn_builder
*b
,
158 struct vtn_ssa_value
*value
,
159 struct vtn_type
*type
,
162 switch (type
->base_type
) {
163 case vtn_base_type_array
:
164 case vtn_base_type_matrix
:
165 for (unsigned i
= 0; i
< type
->length
; i
++) {
166 vtn_ssa_value_load_function_param(b
, value
->elems
[i
],
167 type
->array_element
, param_idx
);
171 case vtn_base_type_struct
:
172 for (unsigned i
= 0; i
< type
->length
; i
++) {
173 vtn_ssa_value_load_function_param(b
, value
->elems
[i
],
174 type
->members
[i
], param_idx
);
179 value
->def
= nir_load_param(&b
->nb
, (*param_idx
)++);
185 vtn_handle_function_call(struct vtn_builder
*b
, SpvOp opcode
,
186 const uint32_t *w
, unsigned count
)
188 struct vtn_type
*res_type
= vtn_value(b
, w
[1], vtn_value_type_type
)->type
;
189 struct vtn_function
*vtn_callee
=
190 vtn_value(b
, w
[3], vtn_value_type_function
)->func
;
191 struct nir_function
*callee
= vtn_callee
->impl
->function
;
193 vtn_callee
->referenced
= true;
195 nir_call_instr
*call
= nir_call_instr_create(b
->nb
.shader
, callee
);
197 unsigned param_idx
= 0;
199 nir_deref_instr
*ret_deref
= NULL
;
200 struct vtn_type
*ret_type
= vtn_callee
->type
->return_type
;
201 if (ret_type
->base_type
!= vtn_base_type_void
) {
202 nir_variable
*ret_tmp
=
203 nir_local_variable_create(b
->nb
.impl
,
204 glsl_get_bare_type(ret_type
->type
),
206 ret_deref
= nir_build_deref_var(&b
->nb
, ret_tmp
);
207 call
->params
[param_idx
++] = nir_src_for_ssa(&ret_deref
->dest
.ssa
);
210 for (unsigned i
= 0; i
< vtn_callee
->type
->length
; i
++) {
211 struct vtn_type
*arg_type
= vtn_callee
->type
->params
[i
];
212 unsigned arg_id
= w
[4 + i
];
214 if (arg_type
->base_type
== vtn_base_type_sampled_image
) {
215 struct vtn_sampled_image
*sampled_image
=
216 vtn_value(b
, arg_id
, vtn_value_type_sampled_image
)->sampled_image
;
218 call
->params
[param_idx
++] =
219 nir_src_for_ssa(&sampled_image
->image
->deref
->dest
.ssa
);
220 call
->params
[param_idx
++] =
221 nir_src_for_ssa(&sampled_image
->sampler
->deref
->dest
.ssa
);
222 } else if (arg_type
->base_type
== vtn_base_type_pointer
||
223 arg_type
->base_type
== vtn_base_type_image
||
224 arg_type
->base_type
== vtn_base_type_sampler
) {
225 struct vtn_pointer
*pointer
=
226 vtn_value(b
, arg_id
, vtn_value_type_pointer
)->pointer
;
227 call
->params
[param_idx
++] =
228 nir_src_for_ssa(vtn_pointer_to_ssa(b
, pointer
));
230 vtn_ssa_value_add_to_call_params(b
, vtn_ssa_value(b
, arg_id
),
231 arg_type
, call
, ¶m_idx
);
234 assert(param_idx
== call
->num_params
);
236 nir_builder_instr_insert(&b
->nb
, &call
->instr
);
238 if (ret_type
->base_type
== vtn_base_type_void
) {
239 vtn_push_value(b
, w
[2], vtn_value_type_undef
);
241 vtn_push_ssa(b
, w
[2], res_type
, vtn_local_load(b
, ret_deref
, 0));
246 vtn_cfg_handle_prepass_instruction(struct vtn_builder
*b
, SpvOp opcode
,
247 const uint32_t *w
, unsigned count
)
250 case SpvOpFunction
: {
251 vtn_assert(b
->func
== NULL
);
252 b
->func
= rzalloc(b
, struct vtn_function
);
254 b
->func
->node
.type
= vtn_cf_node_type_function
;
255 b
->func
->node
.parent
= NULL
;
256 list_inithead(&b
->func
->body
);
257 b
->func
->control
= w
[3];
259 UNUSED
const struct glsl_type
*result_type
=
260 vtn_value(b
, w
[1], vtn_value_type_type
)->type
->type
;
261 struct vtn_value
*val
= vtn_push_value(b
, w
[2], vtn_value_type_function
);
264 b
->func
->type
= vtn_value(b
, w
[4], vtn_value_type_type
)->type
;
265 const struct vtn_type
*func_type
= b
->func
->type
;
267 vtn_assert(func_type
->return_type
->type
== result_type
);
270 nir_function_create(b
->shader
, ralloc_strdup(b
->shader
, val
->name
));
272 unsigned num_params
= 0;
273 for (unsigned i
= 0; i
< func_type
->length
; i
++)
274 num_params
+= vtn_type_count_function_params(func_type
->params
[i
]);
276 /* Add one parameter for the function return value */
277 if (func_type
->return_type
->base_type
!= vtn_base_type_void
)
280 func
->num_params
= num_params
;
281 func
->params
= ralloc_array(b
->shader
, nir_parameter
, num_params
);
284 if (func_type
->return_type
->base_type
!= vtn_base_type_void
) {
285 nir_address_format addr_format
=
286 vtn_mode_to_address_format(b
, vtn_variable_mode_function
);
287 /* The return value is a regular pointer */
288 func
->params
[idx
++] = (nir_parameter
) {
289 .num_components
= nir_address_format_num_components(addr_format
),
290 .bit_size
= nir_address_format_bit_size(addr_format
),
294 for (unsigned i
= 0; i
< func_type
->length
; i
++)
295 vtn_type_add_to_function_params(func_type
->params
[i
], func
, &idx
);
296 assert(idx
== num_params
);
298 b
->func
->impl
= nir_function_impl_create(func
);
299 nir_builder_init(&b
->nb
, func
->impl
);
300 b
->nb
.cursor
= nir_before_cf_list(&b
->func
->impl
->body
);
301 b
->nb
.exact
= b
->exact
;
303 b
->func_param_idx
= 0;
305 /* The return value is the first parameter */
306 if (func_type
->return_type
->base_type
!= vtn_base_type_void
)
311 case SpvOpFunctionEnd
:
316 case SpvOpFunctionParameter
: {
317 struct vtn_type
*type
= vtn_value(b
, w
[1], vtn_value_type_type
)->type
;
319 vtn_assert(b
->func_param_idx
< b
->func
->impl
->function
->num_params
);
321 if (type
->base_type
== vtn_base_type_sampled_image
) {
322 /* Sampled images are actually two parameters. The first is the
323 * image and the second is the sampler.
325 struct vtn_value
*val
=
326 vtn_push_value(b
, w
[2], vtn_value_type_sampled_image
);
328 val
->sampled_image
= ralloc(b
, struct vtn_sampled_image
);
330 struct vtn_type
*sampler_type
= rzalloc(b
, struct vtn_type
);
331 sampler_type
->base_type
= vtn_base_type_sampler
;
332 sampler_type
->type
= glsl_bare_sampler_type();
334 val
->sampled_image
->image
=
335 vtn_load_param_pointer(b
, type
, b
->func_param_idx
++);
336 val
->sampled_image
->sampler
=
337 vtn_load_param_pointer(b
, sampler_type
, b
->func_param_idx
++);
338 } else if (type
->base_type
== vtn_base_type_pointer
&&
339 type
->type
!= NULL
) {
340 /* This is a pointer with an actual storage type */
341 nir_ssa_def
*ssa_ptr
= nir_load_param(&b
->nb
, b
->func_param_idx
++);
342 vtn_push_value_pointer(b
, w
[2], vtn_pointer_from_ssa(b
, ssa_ptr
, type
));
343 } else if (type
->base_type
== vtn_base_type_pointer
||
344 type
->base_type
== vtn_base_type_image
||
345 type
->base_type
== vtn_base_type_sampler
) {
346 vtn_push_value_pointer(b
, w
[2], vtn_load_param_pointer(b
, type
, b
->func_param_idx
++));
348 /* We're a regular SSA value. */
349 struct vtn_ssa_value
*value
= vtn_create_ssa_value(b
, type
->type
);
350 vtn_ssa_value_load_function_param(b
, value
, type
, &b
->func_param_idx
);
351 vtn_push_ssa(b
, w
[2], type
, value
);
357 vtn_assert(b
->block
== NULL
);
358 b
->block
= rzalloc(b
, struct vtn_block
);
359 b
->block
->node
.type
= vtn_cf_node_type_block
;
361 vtn_push_value(b
, w
[1], vtn_value_type_block
)->block
= b
->block
;
363 if (b
->func
->start_block
== NULL
) {
364 /* This is the first block encountered for this function. In this
365 * case, we set the start block and add it to the list of
366 * implemented functions that we'll walk later.
368 b
->func
->start_block
= b
->block
;
369 list_addtail(&b
->func
->node
.link
, &b
->functions
);
374 case SpvOpSelectionMerge
:
376 vtn_assert(b
->block
&& b
->block
->merge
== NULL
);
381 case SpvOpBranchConditional
:
385 case SpvOpReturnValue
:
386 case SpvOpUnreachable
:
387 vtn_assert(b
->block
&& b
->block
->branch
== NULL
);
388 b
->block
->branch
= w
;
393 /* Continue on as per normal */
401 vtn_add_case(struct vtn_builder
*b
, struct vtn_switch
*swtch
,
402 struct vtn_block
*break_block
,
403 uint32_t block_id
, uint64_t val
, bool is_default
)
405 struct vtn_block
*case_block
= vtn_block(b
, block_id
);
407 /* Don't create dummy cases that just break */
408 if (case_block
== break_block
)
411 if (case_block
->switch_case
== NULL
) {
412 struct vtn_case
*c
= ralloc(b
, struct vtn_case
);
414 c
->node
.type
= vtn_cf_node_type_case
;
415 c
->node
.parent
= &swtch
->node
;
416 list_inithead(&c
->body
);
417 c
->start_block
= case_block
;
418 c
->fallthrough
= NULL
;
419 util_dynarray_init(&c
->values
, b
);
420 c
->is_default
= false;
423 list_addtail(&c
->node
.link
, &swtch
->cases
);
425 case_block
->switch_case
= c
;
429 case_block
->switch_case
->is_default
= true;
431 util_dynarray_append(&case_block
->switch_case
->values
, uint64_t, val
);
435 /* This function performs a depth-first search of the cases and puts them
436 * in fall-through order.
439 vtn_order_case(struct vtn_switch
*swtch
, struct vtn_case
*cse
)
446 list_del(&cse
->node
.link
);
448 if (cse
->fallthrough
) {
449 vtn_order_case(swtch
, cse
->fallthrough
);
451 /* If we have a fall-through, place this case right before the case it
452 * falls through to. This ensures that fallthroughs come one after
453 * the other. These two can never get separated because that would
454 * imply something else falling through to the same case. Also, this
455 * can't break ordering because the DFS ensures that this case is
456 * visited before anything that falls through to it.
458 list_addtail(&cse
->node
.link
, &cse
->fallthrough
->node
.link
);
460 list_add(&cse
->node
.link
, &swtch
->cases
);
464 static enum vtn_branch_type
465 vtn_get_branch_type(struct vtn_builder
*b
,
466 struct vtn_block
*block
,
467 struct vtn_case
*swcase
, struct vtn_block
*switch_break
,
468 struct vtn_block
*loop_break
, struct vtn_block
*loop_cont
)
470 if (block
->switch_case
) {
471 /* This branch is actually a fallthrough */
472 vtn_assert(swcase
->fallthrough
== NULL
||
473 swcase
->fallthrough
== block
->switch_case
);
474 swcase
->fallthrough
= block
->switch_case
;
475 return vtn_branch_type_switch_fallthrough
;
476 } else if (block
== loop_break
) {
477 return vtn_branch_type_loop_break
;
478 } else if (block
== loop_cont
) {
479 return vtn_branch_type_loop_continue
;
480 } else if (block
== switch_break
) {
481 return vtn_branch_type_switch_break
;
483 return vtn_branch_type_none
;
488 vtn_cfg_walk_blocks(struct vtn_builder
*b
,
489 struct vtn_cf_node
*cf_parent
,
490 struct list_head
*cf_list
,
491 struct vtn_block
*start
, struct vtn_case
*switch_case
,
492 struct vtn_block
*switch_break
,
493 struct vtn_block
*loop_break
, struct vtn_block
*loop_cont
,
494 struct vtn_block
*end
)
496 struct vtn_block
*block
= start
;
497 while (block
!= end
) {
498 if (block
->merge
&& (*block
->merge
& SpvOpCodeMask
) == SpvOpLoopMerge
&&
500 struct vtn_loop
*loop
= ralloc(b
, struct vtn_loop
);
502 loop
->node
.type
= vtn_cf_node_type_loop
;
503 loop
->node
.parent
= cf_parent
;
504 list_inithead(&loop
->body
);
505 list_inithead(&loop
->cont_body
);
506 loop
->control
= block
->merge
[3];
508 list_addtail(&loop
->node
.link
, cf_list
);
511 struct vtn_block
*new_loop_break
= vtn_block(b
, block
->merge
[1]);
512 struct vtn_block
*new_loop_cont
= vtn_block(b
, block
->merge
[2]);
514 /* Note: This recursive call will start with the current block as
515 * its start block. If we weren't careful, we would get here
516 * again and end up in infinite recursion. This is why we set
517 * block->loop above and check for it before creating one. This
518 * way, we only create the loop once and the second call that
519 * tries to handle this loop goes to the cases below and gets
520 * handled as a regular block.
522 * Note: When we make the recursive walk calls, we pass NULL for
523 * the switch break since you have to break out of the loop first.
524 * We do, however, still pass the current switch case because it's
525 * possible that the merge block for the loop is the start of
528 vtn_cfg_walk_blocks(b
, &loop
->node
, &loop
->body
,
529 block
, switch_case
, NULL
,
530 new_loop_break
, new_loop_cont
, NULL
);
531 vtn_cfg_walk_blocks(b
, &loop
->node
, &loop
->cont_body
,
532 new_loop_cont
, NULL
, NULL
,
533 new_loop_break
, NULL
, block
);
535 enum vtn_branch_type branch_type
=
536 vtn_get_branch_type(b
, new_loop_break
, switch_case
, switch_break
,
537 loop_break
, loop_cont
);
539 if (branch_type
!= vtn_branch_type_none
) {
540 /* Stop walking through the CFG when this inner loop's break block
541 * ends up as the same block as the outer loop's continue block
542 * because we are already going to visit it.
544 vtn_assert(branch_type
== vtn_branch_type_loop_continue
);
548 block
= new_loop_break
;
552 vtn_assert(block
->node
.link
.next
== NULL
);
553 block
->node
.parent
= cf_parent
;
554 list_addtail(&block
->node
.link
, cf_list
);
556 switch (*block
->branch
& SpvOpCodeMask
) {
558 struct vtn_block
*branch_block
= vtn_block(b
, block
->branch
[1]);
560 block
->branch_type
= vtn_get_branch_type(b
, branch_block
,
561 switch_case
, switch_break
,
562 loop_break
, loop_cont
);
564 if (block
->branch_type
!= vtn_branch_type_none
)
567 block
= branch_block
;
572 case SpvOpReturnValue
:
573 block
->branch_type
= vtn_branch_type_return
;
577 block
->branch_type
= vtn_branch_type_discard
;
580 case SpvOpBranchConditional
: {
581 struct vtn_block
*then_block
= vtn_block(b
, block
->branch
[2]);
582 struct vtn_block
*else_block
= vtn_block(b
, block
->branch
[3]);
584 struct vtn_if
*if_stmt
= ralloc(b
, struct vtn_if
);
586 if_stmt
->node
.type
= vtn_cf_node_type_if
;
587 if_stmt
->node
.parent
= cf_parent
;
588 if_stmt
->condition
= block
->branch
[1];
589 list_inithead(&if_stmt
->then_body
);
590 list_inithead(&if_stmt
->else_body
);
592 list_addtail(&if_stmt
->node
.link
, cf_list
);
595 (*block
->merge
& SpvOpCodeMask
) == SpvOpSelectionMerge
) {
596 if_stmt
->control
= block
->merge
[2];
598 if_stmt
->control
= SpvSelectionControlMaskNone
;
601 if_stmt
->then_type
= vtn_get_branch_type(b
, then_block
,
602 switch_case
, switch_break
,
603 loop_break
, loop_cont
);
604 if_stmt
->else_type
= vtn_get_branch_type(b
, else_block
,
605 switch_case
, switch_break
,
606 loop_break
, loop_cont
);
608 if (then_block
== else_block
) {
609 block
->branch_type
= if_stmt
->then_type
;
610 if (block
->branch_type
== vtn_branch_type_none
) {
616 } else if (if_stmt
->then_type
== vtn_branch_type_none
&&
617 if_stmt
->else_type
== vtn_branch_type_none
) {
618 /* Neither side of the if is something we can short-circuit. */
619 vtn_assert((*block
->merge
& SpvOpCodeMask
) == SpvOpSelectionMerge
);
620 struct vtn_block
*merge_block
= vtn_block(b
, block
->merge
[1]);
622 vtn_cfg_walk_blocks(b
, &if_stmt
->node
, &if_stmt
->then_body
,
623 then_block
, switch_case
, switch_break
,
624 loop_break
, loop_cont
, merge_block
);
625 vtn_cfg_walk_blocks(b
, &if_stmt
->node
, &if_stmt
->else_body
,
626 else_block
, switch_case
, switch_break
,
627 loop_break
, loop_cont
, merge_block
);
629 enum vtn_branch_type merge_type
=
630 vtn_get_branch_type(b
, merge_block
, switch_case
, switch_break
,
631 loop_break
, loop_cont
);
632 if (merge_type
== vtn_branch_type_none
) {
638 } else if (if_stmt
->then_type
!= vtn_branch_type_none
&&
639 if_stmt
->else_type
!= vtn_branch_type_none
) {
640 /* Both sides were short-circuited. We're done here. */
643 /* Exeactly one side of the branch could be short-circuited.
644 * We set the branch up as a predicated break/continue and we
645 * continue on with the other side as if it were what comes
648 if (if_stmt
->then_type
== vtn_branch_type_none
) {
655 vtn_fail("Should have returned or continued");
659 vtn_assert((*block
->merge
& SpvOpCodeMask
) == SpvOpSelectionMerge
);
660 struct vtn_block
*break_block
= vtn_block(b
, block
->merge
[1]);
662 struct vtn_switch
*swtch
= ralloc(b
, struct vtn_switch
);
664 swtch
->node
.type
= vtn_cf_node_type_switch
;
665 swtch
->node
.parent
= cf_parent
;
666 swtch
->selector
= block
->branch
[1];
667 list_inithead(&swtch
->cases
);
669 list_addtail(&swtch
->node
.link
, cf_list
);
671 /* First, we go through and record all of the cases. */
672 const uint32_t *branch_end
=
673 block
->branch
+ (block
->branch
[0] >> SpvWordCountShift
);
675 struct vtn_value
*cond_val
= vtn_untyped_value(b
, block
->branch
[1]);
676 vtn_fail_if(!cond_val
->type
||
677 cond_val
->type
->base_type
!= vtn_base_type_scalar
,
678 "Selector of OpSelect must have a type of OpTypeInt");
680 nir_alu_type cond_type
=
681 nir_get_nir_type_for_glsl_type(cond_val
->type
->type
);
682 vtn_fail_if(nir_alu_type_get_base_type(cond_type
) != nir_type_int
&&
683 nir_alu_type_get_base_type(cond_type
) != nir_type_uint
,
684 "Selector of OpSelect must have a type of OpTypeInt");
686 bool is_default
= true;
687 const unsigned bitsize
= nir_alu_type_get_type_size(cond_type
);
688 for (const uint32_t *w
= block
->branch
+ 2; w
< branch_end
;) {
689 uint64_t literal
= 0;
694 assert(bitsize
== 64);
695 literal
= vtn_u64_literal(w
);
700 uint32_t block_id
= *(w
++);
702 vtn_add_case(b
, swtch
, break_block
, block_id
, literal
, is_default
);
706 /* Now, we go through and walk the blocks. While we walk through
707 * the blocks, we also gather the much-needed fall-through
710 vtn_foreach_cf_node(case_node
, &swtch
->cases
) {
711 struct vtn_case
*cse
= vtn_cf_node_as_case(case_node
);
712 vtn_assert(cse
->start_block
!= break_block
);
713 vtn_cfg_walk_blocks(b
, &cse
->node
, &cse
->body
, cse
->start_block
,
714 cse
, break_block
, loop_break
, loop_cont
, NULL
);
717 /* Finally, we walk over all of the cases one more time and put
718 * them in fall-through order.
720 for (const uint32_t *w
= block
->branch
+ 2; w
< branch_end
;) {
721 struct vtn_block
*case_block
= vtn_block(b
, *w
);
726 assert(bitsize
== 64);
730 if (case_block
== break_block
)
733 vtn_assert(case_block
->switch_case
);
735 vtn_order_case(swtch
, case_block
->switch_case
);
738 enum vtn_branch_type branch_type
=
739 vtn_get_branch_type(b
, break_block
, switch_case
, NULL
,
740 loop_break
, loop_cont
);
742 if (branch_type
!= vtn_branch_type_none
) {
743 /* It is possible that the break is actually the continue block
744 * for the containing loop. In this case, we need to bail and let
745 * the loop parsing code handle the continue properly.
747 vtn_assert(branch_type
== vtn_branch_type_loop_continue
);
755 case SpvOpUnreachable
:
759 vtn_fail("Unhandled opcode");
765 vtn_build_cfg(struct vtn_builder
*b
, const uint32_t *words
, const uint32_t *end
)
767 vtn_foreach_instruction(b
, words
, end
,
768 vtn_cfg_handle_prepass_instruction
);
770 vtn_foreach_cf_node(node
, &b
->functions
) {
771 struct vtn_function
*func
= vtn_cf_node_as_function(node
);
772 vtn_cfg_walk_blocks(b
, &func
->node
, &func
->body
, func
->start_block
,
773 NULL
, NULL
, NULL
, NULL
, NULL
);
778 vtn_handle_phis_first_pass(struct vtn_builder
*b
, SpvOp opcode
,
779 const uint32_t *w
, unsigned count
)
781 if (opcode
== SpvOpLabel
)
782 return true; /* Nothing to do */
784 /* If this isn't a phi node, stop. */
785 if (opcode
!= SpvOpPhi
)
788 /* For handling phi nodes, we do a poor-man's out-of-ssa on the spot.
789 * For each phi, we create a variable with the appropreate type and
790 * do a load from that variable. Then, in a second pass, we add
791 * stores to that variable to each of the predecessor blocks.
793 * We could do something more intelligent here. However, in order to
794 * handle loops and things properly, we really need dominance
795 * information. It would end up basically being the into-SSA
796 * algorithm all over again. It's easier if we just let
797 * lower_vars_to_ssa do that for us instead of repeating it here.
799 struct vtn_type
*type
= vtn_value(b
, w
[1], vtn_value_type_type
)->type
;
800 nir_variable
*phi_var
=
801 nir_local_variable_create(b
->nb
.impl
, type
->type
, "phi");
802 _mesa_hash_table_insert(b
->phi_table
, w
, phi_var
);
804 vtn_push_ssa(b
, w
[2], type
,
805 vtn_local_load(b
, nir_build_deref_var(&b
->nb
, phi_var
), 0));
811 vtn_handle_phi_second_pass(struct vtn_builder
*b
, SpvOp opcode
,
812 const uint32_t *w
, unsigned count
)
814 if (opcode
!= SpvOpPhi
)
817 struct hash_entry
*phi_entry
= _mesa_hash_table_search(b
->phi_table
, w
);
818 vtn_assert(phi_entry
);
819 nir_variable
*phi_var
= phi_entry
->data
;
821 for (unsigned i
= 3; i
< count
; i
+= 2) {
822 struct vtn_block
*pred
= vtn_block(b
, w
[i
+ 1]);
824 /* If block does not have end_nop, that is because it is an unreacheable
825 * block, and hence it is not worth to handle it */
829 b
->nb
.cursor
= nir_after_instr(&pred
->end_nop
->instr
);
831 struct vtn_ssa_value
*src
= vtn_ssa_value(b
, w
[i
]);
833 vtn_local_store(b
, src
, nir_build_deref_var(&b
->nb
, phi_var
), 0);
840 vtn_emit_branch(struct vtn_builder
*b
, enum vtn_branch_type branch_type
,
841 nir_variable
*switch_fall_var
, bool *has_switch_break
)
843 switch (branch_type
) {
844 case vtn_branch_type_switch_break
:
845 nir_store_var(&b
->nb
, switch_fall_var
, nir_imm_false(&b
->nb
), 1);
846 *has_switch_break
= true;
848 case vtn_branch_type_switch_fallthrough
:
849 break; /* Nothing to do */
850 case vtn_branch_type_loop_break
:
851 nir_jump(&b
->nb
, nir_jump_break
);
853 case vtn_branch_type_loop_continue
:
854 nir_jump(&b
->nb
, nir_jump_continue
);
856 case vtn_branch_type_return
:
857 nir_jump(&b
->nb
, nir_jump_return
);
859 case vtn_branch_type_discard
: {
860 nir_intrinsic_instr
*discard
=
861 nir_intrinsic_instr_create(b
->nb
.shader
, nir_intrinsic_discard
);
862 nir_builder_instr_insert(&b
->nb
, &discard
->instr
);
866 vtn_fail("Invalid branch type");
871 vtn_switch_case_condition(struct vtn_builder
*b
, struct vtn_switch
*swtch
,
872 nir_ssa_def
*sel
, struct vtn_case
*cse
)
874 if (cse
->is_default
) {
875 nir_ssa_def
*any
= nir_imm_false(&b
->nb
);
876 vtn_foreach_cf_node(other_node
, &swtch
->cases
) {
877 struct vtn_case
*other
= vtn_cf_node_as_case(other_node
);
878 if (other
->is_default
)
881 any
= nir_ior(&b
->nb
, any
,
882 vtn_switch_case_condition(b
, swtch
, sel
, other
));
884 return nir_inot(&b
->nb
, any
);
886 nir_ssa_def
*cond
= nir_imm_false(&b
->nb
);
887 util_dynarray_foreach(&cse
->values
, uint64_t, val
) {
888 nir_ssa_def
*imm
= nir_imm_intN_t(&b
->nb
, *val
, sel
->bit_size
);
889 cond
= nir_ior(&b
->nb
, cond
, nir_ieq(&b
->nb
, sel
, imm
));
895 static nir_loop_control
896 vtn_loop_control(struct vtn_builder
*b
, struct vtn_loop
*vtn_loop
)
898 if (vtn_loop
->control
== SpvLoopControlMaskNone
)
899 return nir_loop_control_none
;
900 else if (vtn_loop
->control
& SpvLoopControlDontUnrollMask
)
901 return nir_loop_control_dont_unroll
;
902 else if (vtn_loop
->control
& SpvLoopControlUnrollMask
)
903 return nir_loop_control_unroll
;
904 else if (vtn_loop
->control
& SpvLoopControlDependencyInfiniteMask
||
905 vtn_loop
->control
& SpvLoopControlDependencyLengthMask
||
906 vtn_loop
->control
& SpvLoopControlMinIterationsMask
||
907 vtn_loop
->control
& SpvLoopControlMaxIterationsMask
||
908 vtn_loop
->control
& SpvLoopControlIterationMultipleMask
||
909 vtn_loop
->control
& SpvLoopControlPeelCountMask
||
910 vtn_loop
->control
& SpvLoopControlPartialCountMask
) {
911 /* We do not do anything special with these yet. */
912 return nir_loop_control_none
;
914 vtn_fail("Invalid loop control");
918 static nir_selection_control
919 vtn_selection_control(struct vtn_builder
*b
, struct vtn_if
*vtn_if
)
921 if (vtn_if
->control
== SpvSelectionControlMaskNone
)
922 return nir_selection_control_none
;
923 else if (vtn_if
->control
& SpvSelectionControlDontFlattenMask
)
924 return nir_selection_control_dont_flatten
;
925 else if (vtn_if
->control
& SpvSelectionControlFlattenMask
)
926 return nir_selection_control_flatten
;
928 vtn_fail("Invalid selection control");
932 vtn_emit_cf_list(struct vtn_builder
*b
, struct list_head
*cf_list
,
933 nir_variable
*switch_fall_var
, bool *has_switch_break
,
934 vtn_instruction_handler handler
)
936 vtn_foreach_cf_node(node
, cf_list
) {
937 switch (node
->type
) {
938 case vtn_cf_node_type_block
: {
939 struct vtn_block
*block
= vtn_cf_node_as_block(node
);
941 const uint32_t *block_start
= block
->label
;
942 const uint32_t *block_end
= block
->merge
? block
->merge
:
945 block_start
= vtn_foreach_instruction(b
, block_start
, block_end
,
946 vtn_handle_phis_first_pass
);
948 vtn_foreach_instruction(b
, block_start
, block_end
, handler
);
950 block
->end_nop
= nir_intrinsic_instr_create(b
->nb
.shader
,
952 nir_builder_instr_insert(&b
->nb
, &block
->end_nop
->instr
);
954 if ((*block
->branch
& SpvOpCodeMask
) == SpvOpReturnValue
) {
955 vtn_fail_if(b
->func
->type
->return_type
->base_type
==
957 "Return with a value from a function returning void");
958 struct vtn_ssa_value
*src
= vtn_ssa_value(b
, block
->branch
[1]);
959 const struct glsl_type
*ret_type
=
960 glsl_get_bare_type(b
->func
->type
->return_type
->type
);
961 nir_deref_instr
*ret_deref
=
962 nir_build_deref_cast(&b
->nb
, nir_load_param(&b
->nb
, 0),
963 nir_var_function_temp
, ret_type
, 0);
964 vtn_local_store(b
, src
, ret_deref
, 0);
967 if (block
->branch_type
!= vtn_branch_type_none
) {
968 vtn_emit_branch(b
, block
->branch_type
,
969 switch_fall_var
, has_switch_break
);
976 case vtn_cf_node_type_if
: {
977 struct vtn_if
*vtn_if
= vtn_cf_node_as_if(node
);
978 bool sw_break
= false;
981 nir_push_if(&b
->nb
, vtn_ssa_value(b
, vtn_if
->condition
)->def
);
983 nif
->control
= vtn_selection_control(b
, vtn_if
);
985 if (vtn_if
->then_type
== vtn_branch_type_none
) {
986 vtn_emit_cf_list(b
, &vtn_if
->then_body
,
987 switch_fall_var
, &sw_break
, handler
);
989 vtn_emit_branch(b
, vtn_if
->then_type
, switch_fall_var
, &sw_break
);
992 nir_push_else(&b
->nb
, nif
);
993 if (vtn_if
->else_type
== vtn_branch_type_none
) {
994 vtn_emit_cf_list(b
, &vtn_if
->else_body
,
995 switch_fall_var
, &sw_break
, handler
);
997 vtn_emit_branch(b
, vtn_if
->else_type
, switch_fall_var
, &sw_break
);
1000 nir_pop_if(&b
->nb
, nif
);
1002 /* If we encountered a switch break somewhere inside of the if,
1003 * then it would have been handled correctly by calling
1004 * emit_cf_list or emit_branch for the interrior. However, we
1005 * need to predicate everything following on wether or not we're
1009 *has_switch_break
= true;
1010 nir_push_if(&b
->nb
, nir_load_var(&b
->nb
, switch_fall_var
));
1015 case vtn_cf_node_type_loop
: {
1016 struct vtn_loop
*vtn_loop
= vtn_cf_node_as_loop(node
);
1018 nir_loop
*loop
= nir_push_loop(&b
->nb
);
1019 loop
->control
= vtn_loop_control(b
, vtn_loop
);
1021 vtn_emit_cf_list(b
, &vtn_loop
->body
, NULL
, NULL
, handler
);
1023 if (!list_is_empty(&vtn_loop
->cont_body
)) {
1024 /* If we have a non-trivial continue body then we need to put
1025 * it at the beginning of the loop with a flag to ensure that
1026 * it doesn't get executed in the first iteration.
1028 nir_variable
*do_cont
=
1029 nir_local_variable_create(b
->nb
.impl
, glsl_bool_type(), "cont");
1031 b
->nb
.cursor
= nir_before_cf_node(&loop
->cf_node
);
1032 nir_store_var(&b
->nb
, do_cont
, nir_imm_false(&b
->nb
), 1);
1034 b
->nb
.cursor
= nir_before_cf_list(&loop
->body
);
1037 nir_push_if(&b
->nb
, nir_load_var(&b
->nb
, do_cont
));
1039 vtn_emit_cf_list(b
, &vtn_loop
->cont_body
, NULL
, NULL
, handler
);
1041 nir_pop_if(&b
->nb
, cont_if
);
1043 nir_store_var(&b
->nb
, do_cont
, nir_imm_true(&b
->nb
), 1);
1045 b
->has_loop_continue
= true;
1048 nir_pop_loop(&b
->nb
, loop
);
1052 case vtn_cf_node_type_switch
: {
1053 struct vtn_switch
*vtn_switch
= vtn_cf_node_as_switch(node
);
1055 /* First, we create a variable to keep track of whether or not the
1056 * switch is still going at any given point. Any switch breaks
1057 * will set this variable to false.
1059 nir_variable
*fall_var
=
1060 nir_local_variable_create(b
->nb
.impl
, glsl_bool_type(), "fall");
1061 nir_store_var(&b
->nb
, fall_var
, nir_imm_false(&b
->nb
), 1);
1063 nir_ssa_def
*sel
= vtn_ssa_value(b
, vtn_switch
->selector
)->def
;
1065 /* Now we can walk the list of cases and actually emit code */
1066 vtn_foreach_cf_node(case_node
, &vtn_switch
->cases
) {
1067 struct vtn_case
*cse
= vtn_cf_node_as_case(case_node
);
1069 /* Figure out the condition */
1071 vtn_switch_case_condition(b
, vtn_switch
, sel
, cse
);
1072 /* Take fallthrough into account */
1073 cond
= nir_ior(&b
->nb
, cond
, nir_load_var(&b
->nb
, fall_var
));
1075 nir_if
*case_if
= nir_push_if(&b
->nb
, cond
);
1077 bool has_break
= false;
1078 nir_store_var(&b
->nb
, fall_var
, nir_imm_true(&b
->nb
), 1);
1079 vtn_emit_cf_list(b
, &cse
->body
, fall_var
, &has_break
, handler
);
1080 (void)has_break
; /* We don't care */
1082 nir_pop_if(&b
->nb
, case_if
);
1089 vtn_fail("Invalid CF node type");
1095 vtn_function_emit(struct vtn_builder
*b
, struct vtn_function
*func
,
1096 vtn_instruction_handler instruction_handler
)
1098 nir_builder_init(&b
->nb
, func
->impl
);
1100 b
->nb
.cursor
= nir_after_cf_list(&func
->impl
->body
);
1101 b
->nb
.exact
= b
->exact
;
1102 b
->has_loop_continue
= false;
1103 b
->phi_table
= _mesa_pointer_hash_table_create(b
);
1105 vtn_emit_cf_list(b
, &func
->body
, NULL
, NULL
, instruction_handler
);
1107 vtn_foreach_instruction(b
, func
->start_block
->label
, func
->end
,
1108 vtn_handle_phi_second_pass
);
1110 nir_rematerialize_derefs_in_use_blocks_impl(func
->impl
);
1112 /* Continue blocks for loops get inserted before the body of the loop
1113 * but instructions in the continue may use SSA defs in the loop body.
1114 * Therefore, we need to repair SSA to insert the needed phi nodes.
1116 if (b
->has_loop_continue
)
1117 nir_repair_ssa_impl(func
->impl
);
1119 func
->emitted
= true;