2 * Copyright © 2014-2015 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 #include "nir_control_flow.h"
28 #include "util/bitscan.h"
29 #include "util/half_float.h"
33 typedef struct nir_builder
{
36 /* Whether new ALU instructions will be marked "exact" */
40 nir_function_impl
*impl
;
44 nir_builder_init(nir_builder
*build
, nir_function_impl
*impl
)
46 memset(build
, 0, sizeof(*build
));
49 build
->shader
= impl
->function
->shader
;
53 nir_builder_init_simple_shader(nir_builder
*build
, void *mem_ctx
,
54 gl_shader_stage stage
,
55 const nir_shader_compiler_options
*options
)
57 build
->shader
= nir_shader_create(mem_ctx
, stage
, options
, NULL
);
58 nir_function
*func
= nir_function_create(build
->shader
, "main");
59 func
->is_entrypoint
= true;
61 build
->impl
= nir_function_impl_create(func
);
62 build
->cursor
= nir_after_cf_list(&build
->impl
->body
);
66 nir_builder_instr_insert(nir_builder
*build
, nir_instr
*instr
)
68 nir_instr_insert(build
->cursor
, instr
);
70 /* Move the cursor forward. */
71 build
->cursor
= nir_after_instr(instr
);
74 static inline nir_instr
*
75 nir_builder_last_instr(nir_builder
*build
)
77 assert(build
->cursor
.option
== nir_cursor_after_instr
);
78 return build
->cursor
.instr
;
82 nir_builder_cf_insert(nir_builder
*build
, nir_cf_node
*cf
)
84 nir_cf_node_insert(build
->cursor
, cf
);
88 nir_builder_is_inside_cf(nir_builder
*build
, nir_cf_node
*cf_node
)
90 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
91 for (nir_cf_node
*n
= &block
->cf_node
; n
; n
= n
->parent
) {
98 static inline nir_if
*
99 nir_push_if(nir_builder
*build
, nir_ssa_def
*condition
)
101 nir_if
*nif
= nir_if_create(build
->shader
);
102 nif
->condition
= nir_src_for_ssa(condition
);
103 nir_builder_cf_insert(build
, &nif
->cf_node
);
104 build
->cursor
= nir_before_cf_list(&nif
->then_list
);
108 static inline nir_if
*
109 nir_push_else(nir_builder
*build
, nir_if
*nif
)
112 assert(nir_builder_is_inside_cf(build
, &nif
->cf_node
));
114 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
115 nif
= nir_cf_node_as_if(block
->cf_node
.parent
);
117 build
->cursor
= nir_before_cf_list(&nif
->else_list
);
122 nir_pop_if(nir_builder
*build
, nir_if
*nif
)
125 assert(nir_builder_is_inside_cf(build
, &nif
->cf_node
));
127 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
128 nif
= nir_cf_node_as_if(block
->cf_node
.parent
);
130 build
->cursor
= nir_after_cf_node(&nif
->cf_node
);
133 static inline nir_ssa_def
*
134 nir_if_phi(nir_builder
*build
, nir_ssa_def
*then_def
, nir_ssa_def
*else_def
)
136 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
137 nir_if
*nif
= nir_cf_node_as_if(nir_cf_node_prev(&block
->cf_node
));
139 nir_phi_instr
*phi
= nir_phi_instr_create(build
->shader
);
141 nir_phi_src
*src
= ralloc(phi
, nir_phi_src
);
142 src
->pred
= nir_if_last_then_block(nif
);
143 src
->src
= nir_src_for_ssa(then_def
);
144 exec_list_push_tail(&phi
->srcs
, &src
->node
);
146 src
= ralloc(phi
, nir_phi_src
);
147 src
->pred
= nir_if_last_else_block(nif
);
148 src
->src
= nir_src_for_ssa(else_def
);
149 exec_list_push_tail(&phi
->srcs
, &src
->node
);
151 assert(then_def
->num_components
== else_def
->num_components
);
152 assert(then_def
->bit_size
== else_def
->bit_size
);
153 nir_ssa_dest_init(&phi
->instr
, &phi
->dest
,
154 then_def
->num_components
, then_def
->bit_size
, NULL
);
156 nir_builder_instr_insert(build
, &phi
->instr
);
158 return &phi
->dest
.ssa
;
161 static inline nir_loop
*
162 nir_push_loop(nir_builder
*build
)
164 nir_loop
*loop
= nir_loop_create(build
->shader
);
165 nir_builder_cf_insert(build
, &loop
->cf_node
);
166 build
->cursor
= nir_before_cf_list(&loop
->body
);
171 nir_pop_loop(nir_builder
*build
, nir_loop
*loop
)
174 assert(nir_builder_is_inside_cf(build
, &loop
->cf_node
));
176 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
177 loop
= nir_cf_node_as_loop(block
->cf_node
.parent
);
179 build
->cursor
= nir_after_cf_node(&loop
->cf_node
);
182 static inline nir_ssa_def
*
183 nir_ssa_undef(nir_builder
*build
, unsigned num_components
, unsigned bit_size
)
185 nir_ssa_undef_instr
*undef
=
186 nir_ssa_undef_instr_create(build
->shader
, num_components
, bit_size
);
190 nir_instr_insert(nir_before_cf_list(&build
->impl
->body
), &undef
->instr
);
195 static inline nir_ssa_def
*
196 nir_build_imm(nir_builder
*build
, unsigned num_components
,
197 unsigned bit_size
, const nir_const_value
*value
)
199 nir_load_const_instr
*load_const
=
200 nir_load_const_instr_create(build
->shader
, num_components
, bit_size
);
204 memcpy(load_const
->value
, value
, sizeof(nir_const_value
) * num_components
);
206 nir_builder_instr_insert(build
, &load_const
->instr
);
208 return &load_const
->def
;
211 static inline nir_ssa_def
*
212 nir_imm_zero(nir_builder
*build
, unsigned num_components
, unsigned bit_size
)
214 nir_load_const_instr
*load_const
=
215 nir_load_const_instr_create(build
->shader
, num_components
, bit_size
);
217 /* nir_load_const_instr_create uses rzalloc so it's already zero */
219 nir_builder_instr_insert(build
, &load_const
->instr
);
221 return &load_const
->def
;
224 static inline nir_ssa_def
*
225 nir_imm_bool(nir_builder
*build
, bool x
)
229 memset(&v
, 0, sizeof(v
));
232 return nir_build_imm(build
, 1, 1, &v
);
235 static inline nir_ssa_def
*
236 nir_imm_true(nir_builder
*build
)
238 return nir_imm_bool(build
, true);
241 static inline nir_ssa_def
*
242 nir_imm_false(nir_builder
*build
)
244 return nir_imm_bool(build
, false);
247 static inline nir_ssa_def
*
248 nir_imm_float16(nir_builder
*build
, float x
)
252 memset(&v
, 0, sizeof(v
));
253 v
.u16
= _mesa_float_to_half(x
);
255 return nir_build_imm(build
, 1, 16, &v
);
258 static inline nir_ssa_def
*
259 nir_imm_float(nir_builder
*build
, float x
)
263 memset(&v
, 0, sizeof(v
));
266 return nir_build_imm(build
, 1, 32, &v
);
269 static inline nir_ssa_def
*
270 nir_imm_double(nir_builder
*build
, double x
)
274 memset(&v
, 0, sizeof(v
));
277 return nir_build_imm(build
, 1, 64, &v
);
280 static inline nir_ssa_def
*
281 nir_imm_floatN_t(nir_builder
*build
, double x
, unsigned bit_size
)
285 return nir_imm_float16(build
, x
);
287 return nir_imm_float(build
, x
);
289 return nir_imm_double(build
, x
);
292 unreachable("unknown float immediate bit size");
295 static inline nir_ssa_def
*
296 nir_imm_vec2(nir_builder
*build
, float x
, float y
)
298 nir_const_value v
[2];
300 memset(v
, 0, sizeof(v
));
304 return nir_build_imm(build
, 2, 32, v
);
307 static inline nir_ssa_def
*
308 nir_imm_vec4(nir_builder
*build
, float x
, float y
, float z
, float w
)
310 nir_const_value v
[4];
312 memset(v
, 0, sizeof(v
));
318 return nir_build_imm(build
, 4, 32, v
);
321 static inline nir_ssa_def
*
322 nir_imm_ivec2(nir_builder
*build
, int x
, int y
)
324 nir_const_value v
[2];
326 memset(v
, 0, sizeof(v
));
330 return nir_build_imm(build
, 2, 32, v
);
333 static inline nir_ssa_def
*
334 nir_imm_int(nir_builder
*build
, int x
)
338 memset(&v
, 0, sizeof(v
));
341 return nir_build_imm(build
, 1, 32, &v
);
344 static inline nir_ssa_def
*
345 nir_imm_int64(nir_builder
*build
, int64_t x
)
349 memset(&v
, 0, sizeof(v
));
352 return nir_build_imm(build
, 1, 64, &v
);
355 static inline nir_ssa_def
*
356 nir_imm_intN_t(nir_builder
*build
, uint64_t x
, unsigned bit_size
)
360 memset(&v
, 0, sizeof(v
));
361 assert(bit_size
<= 64);
365 v
.i64
= x
& (~0ull >> (64 - bit_size
));
367 return nir_build_imm(build
, 1, bit_size
, &v
);
370 static inline nir_ssa_def
*
371 nir_imm_ivec4(nir_builder
*build
, int x
, int y
, int z
, int w
)
373 nir_const_value v
[4];
375 memset(v
, 0, sizeof(v
));
381 return nir_build_imm(build
, 4, 32, v
);
384 static inline nir_ssa_def
*
385 nir_imm_boolN_t(nir_builder
*build
, bool x
, unsigned bit_size
)
387 /* We use a 0/-1 convention for all booleans regardless of size */
388 return nir_imm_intN_t(build
, -(int)x
, bit_size
);
391 static inline nir_ssa_def
*
392 nir_builder_alu_instr_finish_and_insert(nir_builder
*build
, nir_alu_instr
*instr
)
394 const nir_op_info
*op_info
= &nir_op_infos
[instr
->op
];
396 instr
->exact
= build
->exact
;
398 /* Guess the number of components the destination temporary should have
399 * based on our input sizes, if it's not fixed for the op.
401 unsigned num_components
= op_info
->output_size
;
402 if (num_components
== 0) {
403 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
404 if (op_info
->input_sizes
[i
] == 0)
405 num_components
= MAX2(num_components
,
406 instr
->src
[i
].src
.ssa
->num_components
);
409 assert(num_components
!= 0);
411 /* Figure out the bitwidth based on the source bitwidth if the instruction
414 unsigned bit_size
= nir_alu_type_get_type_size(op_info
->output_type
);
416 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
417 unsigned src_bit_size
= instr
->src
[i
].src
.ssa
->bit_size
;
418 if (nir_alu_type_get_type_size(op_info
->input_types
[i
]) == 0) {
420 assert(src_bit_size
== bit_size
);
422 bit_size
= src_bit_size
;
424 assert(src_bit_size
==
425 nir_alu_type_get_type_size(op_info
->input_types
[i
]));
430 /* When in doubt, assume 32. */
434 /* Make sure we don't swizzle from outside of our source vector (like if a
435 * scalar value was passed into a multiply with a vector).
437 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
438 for (unsigned j
= instr
->src
[i
].src
.ssa
->num_components
;
439 j
< NIR_MAX_VEC_COMPONENTS
; j
++) {
440 instr
->src
[i
].swizzle
[j
] = instr
->src
[i
].src
.ssa
->num_components
- 1;
444 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
.dest
, num_components
,
446 instr
->dest
.write_mask
= (1 << num_components
) - 1;
448 nir_builder_instr_insert(build
, &instr
->instr
);
450 return &instr
->dest
.dest
.ssa
;
453 static inline nir_ssa_def
*
454 nir_build_alu(nir_builder
*build
, nir_op op
, nir_ssa_def
*src0
,
455 nir_ssa_def
*src1
, nir_ssa_def
*src2
, nir_ssa_def
*src3
)
457 nir_alu_instr
*instr
= nir_alu_instr_create(build
->shader
, op
);
461 instr
->src
[0].src
= nir_src_for_ssa(src0
);
463 instr
->src
[1].src
= nir_src_for_ssa(src1
);
465 instr
->src
[2].src
= nir_src_for_ssa(src2
);
467 instr
->src
[3].src
= nir_src_for_ssa(src3
);
469 return nir_builder_alu_instr_finish_and_insert(build
, instr
);
472 /* for the couple special cases with more than 4 src args: */
473 static inline nir_ssa_def
*
474 nir_build_alu_src_arr(nir_builder
*build
, nir_op op
, nir_ssa_def
**srcs
)
476 const nir_op_info
*op_info
= &nir_op_infos
[op
];
477 nir_alu_instr
*instr
= nir_alu_instr_create(build
->shader
, op
);
481 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++)
482 instr
->src
[i
].src
= nir_src_for_ssa(srcs
[i
]);
484 return nir_builder_alu_instr_finish_and_insert(build
, instr
);
487 #include "nir_builder_opcodes.h"
489 static inline nir_ssa_def
*
490 nir_vec(nir_builder
*build
, nir_ssa_def
**comp
, unsigned num_components
)
492 return nir_build_alu_src_arr(build
, nir_op_vec(num_components
), comp
);
496 * Similar to nir_fmov, but takes a nir_alu_src instead of a nir_ssa_def.
498 static inline nir_ssa_def
*
499 nir_fmov_alu(nir_builder
*build
, nir_alu_src src
, unsigned num_components
)
501 nir_alu_instr
*mov
= nir_alu_instr_create(build
->shader
, nir_op_fmov
);
502 nir_ssa_dest_init(&mov
->instr
, &mov
->dest
.dest
, num_components
,
503 nir_src_bit_size(src
.src
), NULL
);
504 mov
->exact
= build
->exact
;
505 mov
->dest
.write_mask
= (1 << num_components
) - 1;
507 nir_builder_instr_insert(build
, &mov
->instr
);
509 return &mov
->dest
.dest
.ssa
;
512 static inline nir_ssa_def
*
513 nir_imov_alu(nir_builder
*build
, nir_alu_src src
, unsigned num_components
)
515 nir_alu_instr
*mov
= nir_alu_instr_create(build
->shader
, nir_op_imov
);
516 nir_ssa_dest_init(&mov
->instr
, &mov
->dest
.dest
, num_components
,
517 nir_src_bit_size(src
.src
), NULL
);
518 mov
->exact
= build
->exact
;
519 mov
->dest
.write_mask
= (1 << num_components
) - 1;
521 nir_builder_instr_insert(build
, &mov
->instr
);
523 return &mov
->dest
.dest
.ssa
;
527 * Construct an fmov or imov that reswizzles the source's components.
529 static inline nir_ssa_def
*
530 nir_swizzle(nir_builder
*build
, nir_ssa_def
*src
, const unsigned *swiz
,
531 unsigned num_components
, bool use_fmov
)
533 assert(num_components
<= NIR_MAX_VEC_COMPONENTS
);
534 nir_alu_src alu_src
= { NIR_SRC_INIT
};
535 alu_src
.src
= nir_src_for_ssa(src
);
537 bool is_identity_swizzle
= true;
538 for (unsigned i
= 0; i
< num_components
&& i
< NIR_MAX_VEC_COMPONENTS
; i
++) {
540 is_identity_swizzle
= false;
541 alu_src
.swizzle
[i
] = swiz
[i
];
544 if (num_components
== src
->num_components
&& is_identity_swizzle
)
547 return use_fmov
? nir_fmov_alu(build
, alu_src
, num_components
) :
548 nir_imov_alu(build
, alu_src
, num_components
);
551 /* Selects the right fdot given the number of components in each source. */
552 static inline nir_ssa_def
*
553 nir_fdot(nir_builder
*build
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
555 assert(src0
->num_components
== src1
->num_components
);
556 switch (src0
->num_components
) {
557 case 1: return nir_fmul(build
, src0
, src1
);
558 case 2: return nir_fdot2(build
, src0
, src1
);
559 case 3: return nir_fdot3(build
, src0
, src1
);
560 case 4: return nir_fdot4(build
, src0
, src1
);
562 unreachable("bad component size");
568 static inline nir_ssa_def
*
569 nir_bany_inequal(nir_builder
*b
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
571 switch (src0
->num_components
) {
572 case 1: return nir_ine(b
, src0
, src1
);
573 case 2: return nir_bany_inequal2(b
, src0
, src1
);
574 case 3: return nir_bany_inequal3(b
, src0
, src1
);
575 case 4: return nir_bany_inequal4(b
, src0
, src1
);
577 unreachable("bad component size");
581 static inline nir_ssa_def
*
582 nir_bany(nir_builder
*b
, nir_ssa_def
*src
)
584 return nir_bany_inequal(b
, src
, nir_imm_false(b
));
587 static inline nir_ssa_def
*
588 nir_channel(nir_builder
*b
, nir_ssa_def
*def
, unsigned c
)
590 return nir_swizzle(b
, def
, &c
, 1, false);
593 static inline nir_ssa_def
*
594 nir_channels(nir_builder
*b
, nir_ssa_def
*def
, nir_component_mask_t mask
)
596 unsigned num_channels
= 0, swizzle
[NIR_MAX_VEC_COMPONENTS
] = { 0 };
598 for (unsigned i
= 0; i
< NIR_MAX_VEC_COMPONENTS
; i
++) {
599 if ((mask
& (1 << i
)) == 0)
601 swizzle
[num_channels
++] = i
;
604 return nir_swizzle(b
, def
, swizzle
, num_channels
, false);
607 static inline nir_ssa_def
*
608 _nir_vector_extract_helper(nir_builder
*b
, nir_ssa_def
*vec
, nir_ssa_def
*c
,
609 unsigned start
, unsigned end
)
611 if (start
== end
- 1) {
612 return nir_channel(b
, vec
, start
);
614 unsigned mid
= start
+ (end
- start
) / 2;
615 return nir_bcsel(b
, nir_ilt(b
, c
, nir_imm_int(b
, mid
)),
616 _nir_vector_extract_helper(b
, vec
, c
, start
, mid
),
617 _nir_vector_extract_helper(b
, vec
, c
, mid
, end
));
621 static inline nir_ssa_def
*
622 nir_vector_extract(nir_builder
*b
, nir_ssa_def
*vec
, nir_ssa_def
*c
)
624 nir_src c_src
= nir_src_for_ssa(c
);
625 if (nir_src_is_const(c_src
)) {
626 unsigned c_const
= nir_src_as_uint(c_src
);
627 if (c_const
< vec
->num_components
)
628 return nir_channel(b
, vec
, c_const
);
630 return nir_ssa_undef(b
, 1, vec
->bit_size
);
632 return _nir_vector_extract_helper(b
, vec
, c
, 0, vec
->num_components
);
636 static inline nir_ssa_def
*
637 nir_i2i(nir_builder
*build
, nir_ssa_def
*x
, unsigned dest_bit_size
)
639 if (x
->bit_size
== dest_bit_size
)
642 switch (dest_bit_size
) {
643 case 64: return nir_i2i64(build
, x
);
644 case 32: return nir_i2i32(build
, x
);
645 case 16: return nir_i2i16(build
, x
);
646 case 8: return nir_i2i8(build
, x
);
647 default: unreachable("Invalid bit size");
651 static inline nir_ssa_def
*
652 nir_u2u(nir_builder
*build
, nir_ssa_def
*x
, unsigned dest_bit_size
)
654 if (x
->bit_size
== dest_bit_size
)
657 switch (dest_bit_size
) {
658 case 64: return nir_u2u64(build
, x
);
659 case 32: return nir_u2u32(build
, x
);
660 case 16: return nir_u2u16(build
, x
);
661 case 8: return nir_u2u8(build
, x
);
662 default: unreachable("Invalid bit size");
666 static inline nir_ssa_def
*
667 nir_iadd_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
669 assert(x
->bit_size
<= 64);
670 if (x
->bit_size
< 64)
671 y
&= (1ull << x
->bit_size
) - 1;
676 return nir_iadd(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
680 static inline nir_ssa_def
*
681 nir_imul_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
683 assert(x
->bit_size
<= 64);
684 if (x
->bit_size
< 64)
685 y
&= (1ull << x
->bit_size
) - 1;
688 return nir_imm_intN_t(build
, 0, x
->bit_size
);
691 } else if (util_is_power_of_two_or_zero64(y
)) {
692 return nir_ishl(build
, x
, nir_imm_int(build
, ffsll(y
) - 1));
694 return nir_imul(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
698 static inline nir_ssa_def
*
699 nir_fadd_imm(nir_builder
*build
, nir_ssa_def
*x
, double y
)
701 return nir_fadd(build
, x
, nir_imm_floatN_t(build
, y
, x
->bit_size
));
704 static inline nir_ssa_def
*
705 nir_fmul_imm(nir_builder
*build
, nir_ssa_def
*x
, double y
)
707 return nir_fmul(build
, x
, nir_imm_floatN_t(build
, y
, x
->bit_size
));
710 static inline nir_ssa_def
*
711 nir_pack_bits(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
713 assert(src
->num_components
* src
->bit_size
== dest_bit_size
);
715 switch (dest_bit_size
) {
717 switch (src
->bit_size
) {
718 case 32: return nir_pack_64_2x32(b
, src
);
719 case 16: return nir_pack_64_4x16(b
, src
);
725 if (src
->bit_size
== 16)
726 return nir_pack_32_2x16(b
, src
);
733 /* If we got here, we have no dedicated unpack opcode. */
734 nir_ssa_def
*dest
= nir_imm_intN_t(b
, 0, dest_bit_size
);
735 for (unsigned i
= 0; i
< src
->num_components
; i
++) {
736 nir_ssa_def
*val
= nir_u2u(b
, nir_channel(b
, src
, i
), dest_bit_size
);
737 val
= nir_ishl(b
, val
, nir_imm_int(b
, i
* src
->bit_size
));
738 dest
= nir_ior(b
, dest
, val
);
743 static inline nir_ssa_def
*
744 nir_unpack_bits(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
746 assert(src
->num_components
== 1);
747 assert(src
->bit_size
> dest_bit_size
);
748 const unsigned dest_num_components
= src
->bit_size
/ dest_bit_size
;
749 assert(dest_num_components
<= NIR_MAX_VEC_COMPONENTS
);
751 switch (src
->bit_size
) {
753 switch (dest_bit_size
) {
754 case 32: return nir_unpack_64_2x32(b
, src
);
755 case 16: return nir_unpack_64_4x16(b
, src
);
761 if (dest_bit_size
== 16)
762 return nir_unpack_32_2x16(b
, src
);
769 /* If we got here, we have no dedicated unpack opcode. */
770 nir_ssa_def
*dest_comps
[NIR_MAX_VEC_COMPONENTS
];
771 for (unsigned i
= 0; i
< dest_num_components
; i
++) {
772 nir_ssa_def
*val
= nir_ushr(b
, src
, nir_imm_int(b
, i
* dest_bit_size
));
773 dest_comps
[i
] = nir_u2u(b
, val
, dest_bit_size
);
775 return nir_vec(b
, dest_comps
, dest_num_components
);
778 static inline nir_ssa_def
*
779 nir_bitcast_vector(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
781 assert((src
->bit_size
* src
->num_components
) % dest_bit_size
== 0);
782 const unsigned dest_num_components
=
783 (src
->bit_size
* src
->num_components
) / dest_bit_size
;
784 assert(dest_num_components
<= NIR_MAX_VEC_COMPONENTS
);
786 if (src
->bit_size
> dest_bit_size
) {
787 assert(src
->bit_size
% dest_bit_size
== 0);
788 if (src
->num_components
== 1) {
789 return nir_unpack_bits(b
, src
, dest_bit_size
);
791 const unsigned divisor
= src
->bit_size
/ dest_bit_size
;
792 assert(src
->num_components
* divisor
== dest_num_components
);
793 nir_ssa_def
*dest
[NIR_MAX_VEC_COMPONENTS
];
794 for (unsigned i
= 0; i
< src
->num_components
; i
++) {
795 nir_ssa_def
*unpacked
=
796 nir_unpack_bits(b
, nir_channel(b
, src
, i
), dest_bit_size
);
797 assert(unpacked
->num_components
== divisor
);
798 for (unsigned j
= 0; j
< divisor
; j
++)
799 dest
[i
* divisor
+ j
] = nir_channel(b
, unpacked
, j
);
801 return nir_vec(b
, dest
, dest_num_components
);
803 } else if (src
->bit_size
< dest_bit_size
) {
804 assert(dest_bit_size
% src
->bit_size
== 0);
805 if (dest_num_components
== 1) {
806 return nir_pack_bits(b
, src
, dest_bit_size
);
808 const unsigned divisor
= dest_bit_size
/ src
->bit_size
;
809 assert(src
->num_components
== dest_num_components
* divisor
);
810 nir_ssa_def
*dest
[NIR_MAX_VEC_COMPONENTS
];
811 for (unsigned i
= 0; i
< dest_num_components
; i
++) {
812 nir_component_mask_t src_mask
=
813 ((1 << divisor
) - 1) << (i
* divisor
);
814 dest
[i
] = nir_pack_bits(b
, nir_channels(b
, src
, src_mask
),
817 return nir_vec(b
, dest
, dest_num_components
);
820 assert(src
->bit_size
== dest_bit_size
);
826 * Turns a nir_src into a nir_ssa_def * so it can be passed to
827 * nir_build_alu()-based builder calls.
829 * See nir_ssa_for_alu_src() for alu instructions.
831 static inline nir_ssa_def
*
832 nir_ssa_for_src(nir_builder
*build
, nir_src src
, int num_components
)
834 if (src
.is_ssa
&& src
.ssa
->num_components
== num_components
)
837 nir_alu_src alu
= { NIR_SRC_INIT
};
839 for (int j
= 0; j
< 4; j
++)
842 return nir_imov_alu(build
, alu
, num_components
);
846 * Similar to nir_ssa_for_src(), but for alu srcs, respecting the
847 * nir_alu_src's swizzle.
849 static inline nir_ssa_def
*
850 nir_ssa_for_alu_src(nir_builder
*build
, nir_alu_instr
*instr
, unsigned srcn
)
852 static uint8_t trivial_swizzle
[NIR_MAX_VEC_COMPONENTS
];
853 for (int i
= 0; i
< NIR_MAX_VEC_COMPONENTS
; ++i
)
854 trivial_swizzle
[i
] = i
;
855 nir_alu_src
*src
= &instr
->src
[srcn
];
856 unsigned num_components
= nir_ssa_alu_instr_src_components(instr
, srcn
);
858 if (src
->src
.is_ssa
&& (src
->src
.ssa
->num_components
== num_components
) &&
859 !src
->abs
&& !src
->negate
&&
860 (memcmp(src
->swizzle
, trivial_swizzle
, num_components
) == 0))
863 return nir_imov_alu(build
, *src
, num_components
);
866 static inline unsigned
867 nir_get_ptr_bitsize(nir_builder
*build
)
869 if (build
->shader
->info
.stage
== MESA_SHADER_KERNEL
)
870 return build
->shader
->info
.cs
.ptr_size
;
874 static inline nir_deref_instr
*
875 nir_build_deref_var(nir_builder
*build
, nir_variable
*var
)
877 nir_deref_instr
*deref
=
878 nir_deref_instr_create(build
->shader
, nir_deref_type_var
);
880 deref
->mode
= var
->data
.mode
;
881 deref
->type
= var
->type
;
884 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
, 1,
885 nir_get_ptr_bitsize(build
), NULL
);
887 nir_builder_instr_insert(build
, &deref
->instr
);
892 static inline nir_deref_instr
*
893 nir_build_deref_array(nir_builder
*build
, nir_deref_instr
*parent
,
896 assert(glsl_type_is_array(parent
->type
) ||
897 glsl_type_is_matrix(parent
->type
) ||
898 glsl_type_is_vector(parent
->type
));
900 assert(index
->bit_size
== parent
->dest
.ssa
.bit_size
);
902 nir_deref_instr
*deref
=
903 nir_deref_instr_create(build
->shader
, nir_deref_type_array
);
905 deref
->mode
= parent
->mode
;
906 deref
->type
= glsl_get_array_element(parent
->type
);
907 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
908 deref
->arr
.index
= nir_src_for_ssa(index
);
910 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
911 parent
->dest
.ssa
.num_components
,
912 parent
->dest
.ssa
.bit_size
, NULL
);
914 nir_builder_instr_insert(build
, &deref
->instr
);
919 static inline nir_deref_instr
*
920 nir_build_deref_array_imm(nir_builder
*build
, nir_deref_instr
*parent
,
923 assert(parent
->dest
.is_ssa
);
924 nir_ssa_def
*idx_ssa
= nir_imm_intN_t(build
, index
,
925 parent
->dest
.ssa
.bit_size
);
927 return nir_build_deref_array(build
, parent
, idx_ssa
);
930 static inline nir_deref_instr
*
931 nir_build_deref_ptr_as_array(nir_builder
*build
, nir_deref_instr
*parent
,
934 assert(parent
->deref_type
== nir_deref_type_array
||
935 parent
->deref_type
== nir_deref_type_ptr_as_array
||
936 parent
->deref_type
== nir_deref_type_cast
);
938 assert(index
->bit_size
== parent
->dest
.ssa
.bit_size
);
940 nir_deref_instr
*deref
=
941 nir_deref_instr_create(build
->shader
, nir_deref_type_ptr_as_array
);
943 deref
->mode
= parent
->mode
;
944 deref
->type
= parent
->type
;
945 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
946 deref
->arr
.index
= nir_src_for_ssa(index
);
948 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
949 parent
->dest
.ssa
.num_components
,
950 parent
->dest
.ssa
.bit_size
, NULL
);
952 nir_builder_instr_insert(build
, &deref
->instr
);
957 static inline nir_deref_instr
*
958 nir_build_deref_array_wildcard(nir_builder
*build
, nir_deref_instr
*parent
)
960 assert(glsl_type_is_array(parent
->type
) ||
961 glsl_type_is_matrix(parent
->type
));
963 nir_deref_instr
*deref
=
964 nir_deref_instr_create(build
->shader
, nir_deref_type_array_wildcard
);
966 deref
->mode
= parent
->mode
;
967 deref
->type
= glsl_get_array_element(parent
->type
);
968 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
970 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
971 parent
->dest
.ssa
.num_components
,
972 parent
->dest
.ssa
.bit_size
, NULL
);
974 nir_builder_instr_insert(build
, &deref
->instr
);
979 static inline nir_deref_instr
*
980 nir_build_deref_struct(nir_builder
*build
, nir_deref_instr
*parent
,
983 assert(glsl_type_is_struct_or_ifc(parent
->type
));
985 nir_deref_instr
*deref
=
986 nir_deref_instr_create(build
->shader
, nir_deref_type_struct
);
988 deref
->mode
= parent
->mode
;
989 deref
->type
= glsl_get_struct_field(parent
->type
, index
);
990 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
991 deref
->strct
.index
= index
;
993 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
994 parent
->dest
.ssa
.num_components
,
995 parent
->dest
.ssa
.bit_size
, NULL
);
997 nir_builder_instr_insert(build
, &deref
->instr
);
1002 static inline nir_deref_instr
*
1003 nir_build_deref_cast(nir_builder
*build
, nir_ssa_def
*parent
,
1004 nir_variable_mode mode
, const struct glsl_type
*type
,
1005 unsigned ptr_stride
)
1007 nir_deref_instr
*deref
=
1008 nir_deref_instr_create(build
->shader
, nir_deref_type_cast
);
1012 deref
->parent
= nir_src_for_ssa(parent
);
1013 deref
->cast
.ptr_stride
= ptr_stride
;
1015 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
1016 parent
->num_components
, parent
->bit_size
, NULL
);
1018 nir_builder_instr_insert(build
, &deref
->instr
);
1023 /** Returns a deref that follows another but starting from the given parent
1025 * The new deref will be the same type and take the same array or struct index
1026 * as the leader deref but it may have a different parent. This is very
1027 * useful for walking deref paths.
1029 static inline nir_deref_instr
*
1030 nir_build_deref_follower(nir_builder
*b
, nir_deref_instr
*parent
,
1031 nir_deref_instr
*leader
)
1033 /* If the derefs would have the same parent, don't make a new one */
1034 assert(leader
->parent
.is_ssa
);
1035 if (leader
->parent
.ssa
== &parent
->dest
.ssa
)
1038 UNUSED nir_deref_instr
*leader_parent
= nir_src_as_deref(leader
->parent
);
1040 switch (leader
->deref_type
) {
1041 case nir_deref_type_var
:
1042 unreachable("A var dereference cannot have a parent");
1045 case nir_deref_type_array
:
1046 case nir_deref_type_array_wildcard
:
1047 assert(glsl_type_is_matrix(parent
->type
) ||
1048 glsl_type_is_array(parent
->type
) ||
1049 (leader
->deref_type
== nir_deref_type_array
&&
1050 glsl_type_is_vector(parent
->type
)));
1051 assert(glsl_get_length(parent
->type
) ==
1052 glsl_get_length(leader_parent
->type
));
1054 if (leader
->deref_type
== nir_deref_type_array
) {
1055 assert(leader
->arr
.index
.is_ssa
);
1056 nir_ssa_def
*index
= nir_i2i(b
, leader
->arr
.index
.ssa
,
1057 parent
->dest
.ssa
.bit_size
);
1058 return nir_build_deref_array(b
, parent
, index
);
1060 return nir_build_deref_array_wildcard(b
, parent
);
1063 case nir_deref_type_struct
:
1064 assert(glsl_type_is_struct_or_ifc(parent
->type
));
1065 assert(glsl_get_length(parent
->type
) ==
1066 glsl_get_length(leader_parent
->type
));
1068 return nir_build_deref_struct(b
, parent
, leader
->strct
.index
);
1071 unreachable("Invalid deref instruction type");
1075 static inline nir_ssa_def
*
1076 nir_load_reg(nir_builder
*build
, nir_register
*reg
)
1078 return nir_ssa_for_src(build
, nir_src_for_reg(reg
), reg
->num_components
);
1081 static inline nir_ssa_def
*
1082 nir_load_deref_with_access(nir_builder
*build
, nir_deref_instr
*deref
,
1083 enum gl_access_qualifier access
)
1085 nir_intrinsic_instr
*load
=
1086 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_load_deref
);
1087 load
->num_components
= glsl_get_vector_elements(deref
->type
);
1088 load
->src
[0] = nir_src_for_ssa(&deref
->dest
.ssa
);
1089 nir_ssa_dest_init(&load
->instr
, &load
->dest
, load
->num_components
,
1090 glsl_get_bit_size(deref
->type
), NULL
);
1091 nir_intrinsic_set_access(load
, access
);
1092 nir_builder_instr_insert(build
, &load
->instr
);
1093 return &load
->dest
.ssa
;
1096 static inline nir_ssa_def
*
1097 nir_load_deref(nir_builder
*build
, nir_deref_instr
*deref
)
1099 return nir_load_deref_with_access(build
, deref
, (enum gl_access_qualifier
)0);
1103 nir_store_deref_with_access(nir_builder
*build
, nir_deref_instr
*deref
,
1104 nir_ssa_def
*value
, unsigned writemask
,
1105 enum gl_access_qualifier access
)
1107 nir_intrinsic_instr
*store
=
1108 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_store_deref
);
1109 store
->num_components
= glsl_get_vector_elements(deref
->type
);
1110 store
->src
[0] = nir_src_for_ssa(&deref
->dest
.ssa
);
1111 store
->src
[1] = nir_src_for_ssa(value
);
1112 nir_intrinsic_set_write_mask(store
,
1113 writemask
& ((1 << store
->num_components
) - 1));
1114 nir_intrinsic_set_access(store
, access
);
1115 nir_builder_instr_insert(build
, &store
->instr
);
1119 nir_store_deref(nir_builder
*build
, nir_deref_instr
*deref
,
1120 nir_ssa_def
*value
, unsigned writemask
)
1122 nir_store_deref_with_access(build
, deref
, value
, writemask
,
1123 (enum gl_access_qualifier
)0);
1127 nir_copy_deref(nir_builder
*build
, nir_deref_instr
*dest
, nir_deref_instr
*src
)
1129 nir_intrinsic_instr
*copy
=
1130 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_copy_deref
);
1131 copy
->src
[0] = nir_src_for_ssa(&dest
->dest
.ssa
);
1132 copy
->src
[1] = nir_src_for_ssa(&src
->dest
.ssa
);
1133 nir_builder_instr_insert(build
, ©
->instr
);
1136 static inline nir_ssa_def
*
1137 nir_load_var(nir_builder
*build
, nir_variable
*var
)
1139 return nir_load_deref(build
, nir_build_deref_var(build
, var
));
1143 nir_store_var(nir_builder
*build
, nir_variable
*var
, nir_ssa_def
*value
,
1146 nir_store_deref(build
, nir_build_deref_var(build
, var
), value
, writemask
);
1150 nir_copy_var(nir_builder
*build
, nir_variable
*dest
, nir_variable
*src
)
1152 nir_copy_deref(build
, nir_build_deref_var(build
, dest
),
1153 nir_build_deref_var(build
, src
));
1156 static inline nir_ssa_def
*
1157 nir_load_param(nir_builder
*build
, uint32_t param_idx
)
1159 assert(param_idx
< build
->impl
->function
->num_params
);
1160 nir_parameter
*param
= &build
->impl
->function
->params
[param_idx
];
1162 nir_intrinsic_instr
*load
=
1163 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_load_param
);
1164 nir_intrinsic_set_param_idx(load
, param_idx
);
1165 load
->num_components
= param
->num_components
;
1166 nir_ssa_dest_init(&load
->instr
, &load
->dest
,
1167 param
->num_components
, param
->bit_size
, NULL
);
1168 nir_builder_instr_insert(build
, &load
->instr
);
1169 return &load
->dest
.ssa
;
1172 #include "nir_builder_opcodes.h"
1174 static inline nir_ssa_def
*
1175 nir_f2b(nir_builder
*build
, nir_ssa_def
*f
)
1177 return nir_f2b1(build
, f
);
1180 static inline nir_ssa_def
*
1181 nir_i2b(nir_builder
*build
, nir_ssa_def
*i
)
1183 return nir_i2b1(build
, i
);
1186 static inline nir_ssa_def
*
1187 nir_b2f(nir_builder
*build
, nir_ssa_def
*b
, uint32_t bit_size
)
1190 case 64: return nir_b2f64(build
, b
);
1191 case 32: return nir_b2f32(build
, b
);
1192 case 16: return nir_b2f16(build
, b
);
1194 unreachable("Invalid bit-size");
1198 static inline nir_ssa_def
*
1199 nir_load_barycentric(nir_builder
*build
, nir_intrinsic_op op
,
1200 unsigned interp_mode
)
1202 nir_intrinsic_instr
*bary
= nir_intrinsic_instr_create(build
->shader
, op
);
1203 nir_ssa_dest_init(&bary
->instr
, &bary
->dest
, 2, 32, NULL
);
1204 nir_intrinsic_set_interp_mode(bary
, interp_mode
);
1205 nir_builder_instr_insert(build
, &bary
->instr
);
1206 return &bary
->dest
.ssa
;
1210 nir_jump(nir_builder
*build
, nir_jump_type jump_type
)
1212 nir_jump_instr
*jump
= nir_jump_instr_create(build
->shader
, jump_type
);
1213 nir_builder_instr_insert(build
, &jump
->instr
);
1216 static inline nir_ssa_def
*
1217 nir_compare_func(nir_builder
*b
, enum compare_func func
,
1218 nir_ssa_def
*src0
, nir_ssa_def
*src1
)
1221 case COMPARE_FUNC_NEVER
:
1222 return nir_imm_int(b
, 0);
1223 case COMPARE_FUNC_ALWAYS
:
1224 return nir_imm_int(b
, ~0);
1225 case COMPARE_FUNC_EQUAL
:
1226 return nir_feq(b
, src0
, src1
);
1227 case COMPARE_FUNC_NOTEQUAL
:
1228 return nir_fne(b
, src0
, src1
);
1229 case COMPARE_FUNC_GREATER
:
1230 return nir_flt(b
, src1
, src0
);
1231 case COMPARE_FUNC_GEQUAL
:
1232 return nir_fge(b
, src0
, src1
);
1233 case COMPARE_FUNC_LESS
:
1234 return nir_flt(b
, src0
, src1
);
1235 case COMPARE_FUNC_LEQUAL
:
1236 return nir_fge(b
, src1
, src0
);
1238 unreachable("bad compare func");
1241 #endif /* NIR_BUILDER_H */